From 685a53243df3c081f0785248a366221143ed2c24 Mon Sep 17 00:00:00 2001 From: Matthias Wittgen Date: Tue, 6 Oct 2026 14:08:06 -0700 Subject: [PATCH 1/2] Resolve bare executable names to full paths for HTCondor After DM-51000 the bps default commands use bare executable names (e.g. "pipetask", "butler", "bps") instead of absolute paths. The HTCondor submit description's "executable" is resolved relative to the job sandbox rather than via PATH, so a bare name causes jobs to be held with "No such file or directory" on the compute node. Resolve a bare, non-transferred executable name to its full path via shutil.which() at submit time. Full paths and HTCondor env-var syntax are left untouched; an unresolvable name is left as-is with a warning. Generated with AI Co-Authored-By: SLAC AI --- .../lsst/ctrl/bps/htcondor/prepare_utils.py | 42 ++++++++++++++++++- tests/test_prepare_utils.py | 19 +++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/python/lsst/ctrl/bps/htcondor/prepare_utils.py b/python/lsst/ctrl/bps/htcondor/prepare_utils.py index dd8b456..19ae690 100644 --- a/python/lsst/ctrl/bps/htcondor/prepare_utils.py +++ b/python/lsst/ctrl/bps/htcondor/prepare_utils.py @@ -30,6 +30,7 @@ import logging import os import re +import shutil from collections import defaultdict from pathlib import Path from typing import Any, cast @@ -283,6 +284,45 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob): return jobcmds +def _resolve_executable(executable: str) -> str: + """Resolve a bare executable name to its full path via PATH. + + HTCondor resolves a relative ``executable`` in the submit description + relative to the job's sandbox directory rather than searching ``PATH``. + So a bare command name (e.g., ``pipetask``) must be turned into a full + path at submit time, otherwise the job is held with a "No such file or + directory" error on the compute node. + + Parameters + ---------- + executable : `str` + The executable value from the job description. + + Returns + ------- + executable : `str` + A full path if ``executable`` was a bare name found on ``PATH``, + otherwise the value unchanged (already absolute/relative paths and + values containing HTCondor env syntax are left as is). + """ + # Leave anything that already contains a path separator or HTCondor + # environment-variable syntax (e.g., $ENV(...), $(...)) untouched. + if "/" in executable or "$" in executable: + return executable + + resolved = shutil.which(executable) + if resolved: + _LOG.debug("Resolved bare executable %s to %s", executable, resolved) + return resolved + + _LOG.warning( + "Could not resolve executable '%s' on PATH; HTCondor resolves a bare " + "executable name relative to the job sandbox, so the job may be held.", + executable, + ) + return executable + + def _translate_command_line( cached_vals: dict[str, Any], generic_workflow: GenericWorkflow, gwjob: GenericWorkflowJob ) -> dict[str, Any]: @@ -353,7 +393,7 @@ def _translate_command_line( jobcmds["transfer_executable"] = "True" jobcmds["executable"] = gwjob.executable.src_uri else: - jobcmds["executable"] = _fix_env_var_syntax(gwjob.executable.src_uri) + jobcmds["executable"] = _resolve_executable(_fix_env_var_syntax(gwjob.executable.src_uri)) if arguments: arguments = _fix_env_var_syntax(arguments) diff --git a/tests/test_prepare_utils.py b/tests/test_prepare_utils.py index d30a69f..c968e4e 100644 --- a/tests/test_prepare_utils.py +++ b/tests/test_prepare_utils.py @@ -29,6 +29,7 @@ import logging import os +import shutil import unittest from copy import deepcopy @@ -222,6 +223,24 @@ def testMakeCommandExecutableEnvVar(self): jobcmds = prepare_utils._translate_command_line(self.cached_vals, gw, gwjob) self.assertEqual(jobcmds["executable"], "$ENV(CTRL_BPS_DIR)/bin/pipetask") + def testMakeCommandBareExecutableResolvedViaPath(self): + # A bare executable name (no path, no env var) is resolved to its + # full path via PATH because HTCondor resolves the executable + # relative to the job sandbox, not via PATH. + resolved = shutil.which("sh") + self.assertIsNotNone(resolved, "test requires 'sh' on PATH") + gw_exec = GenericWorkflowExec("sh", "sh") + gw, gwjob = self._make_job(executable=gw_exec) + jobcmds = prepare_utils._translate_command_line(self.cached_vals, gw, gwjob) + self.assertEqual(jobcmds["executable"], resolved) + + def testMakeCommandBareExecutableNotOnPath(self): + # A bare executable name that cannot be found on PATH is left as is. + gw_exec = GenericWorkflowExec("nosuchcmd", "nosuchcmd_xyzzy") + gw, gwjob = self._make_job(executable=gw_exec) + jobcmds = prepare_utils._translate_command_line(self.cached_vals, gw, gwjob) + self.assertEqual(jobcmds["executable"], "nosuchcmd_xyzzy") + def testMakeCommandArguments(self): # Arguments should have cmd, wms, file, and env placeholders replaced. gw, gwjob = self._make_job(arguments="run --attempt {opt}") From 5b171a4d262187b34231af9c3ae05320eaa2f9b4 Mon Sep 17 00:00:00 2001 From: Michelle Gower Date: Wed, 7 Oct 2026 14:02:02 -0700 Subject: [PATCH 2/2] Make improvements to AI generated code and tests. --- doc/changes/DM-51000.misc.rst | 1 + .../lsst/ctrl/bps/htcondor/prepare_utils.py | 27 +++++++++++-------- tests/test_prepare_utils.py | 4 ++- 3 files changed, 20 insertions(+), 12 deletions(-) create mode 100644 doc/changes/DM-51000.misc.rst diff --git a/doc/changes/DM-51000.misc.rst b/doc/changes/DM-51000.misc.rst new file mode 100644 index 0000000..016004d --- /dev/null +++ b/doc/changes/DM-51000.misc.rst @@ -0,0 +1 @@ +Updated the local submissions to handle the case when the given executable name is not an absolute path. It uses the submit-side location of the executable or prints a warning that the job may be held. This was triggered by ``ctrl_bps`` default yaml definitions no longer using ``*_DIR`` environment variables; instead they now rely on the PATH. diff --git a/python/lsst/ctrl/bps/htcondor/prepare_utils.py b/python/lsst/ctrl/bps/htcondor/prepare_utils.py index 19ae690..07fac74 100644 --- a/python/lsst/ctrl/bps/htcondor/prepare_utils.py +++ b/python/lsst/ctrl/bps/htcondor/prepare_utils.py @@ -285,13 +285,13 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob): def _resolve_executable(executable: str) -> str: - """Resolve a bare executable name to its full path via PATH. + """Resolve an executable name to its full path via PATH. HTCondor resolves a relative ``executable`` in the submit description relative to the job's sandbox directory rather than searching ``PATH``. - So a bare command name (e.g., ``pipetask``) must be turned into a full - path at submit time, otherwise the job is held with a "No such file or - directory" error on the compute node. + So a bare or relative command name (e.g., ``pipetask``) must be turned + into a full path at submit time, otherwise the job is held with a + "No such file or directory" error on the compute node. Parameters ---------- @@ -305,18 +305,23 @@ def _resolve_executable(executable: str) -> str: otherwise the value unchanged (already absolute/relative paths and values containing HTCondor env syntax are left as is). """ - # Leave anything that already contains a path separator or HTCondor + # Leave anything that already starts with a path separator or # environment-variable syntax (e.g., $ENV(...), $(...)) untouched. - if "/" in executable or "$" in executable: + if executable.startswith(("/", "$")): + _LOG.debug("Executable started with / or $. Using as-is (%s)", executable) return executable - resolved = shutil.which(executable) - if resolved: - _LOG.debug("Resolved bare executable %s to %s", executable, resolved) - return resolved + exec_path = shutil.which(executable) + if exec_path: + _LOG.debug("results of 'which %s' = %s", executable, exec_path) + # Still have to make sure it is not a relative path as + # shutil.which would skip checking PATH if executable + # contains '/' and then only check existence. + exec_path = os.path.abspath(exec_path) + return exec_path _LOG.warning( - "Could not resolve executable '%s' on PATH; HTCondor resolves a bare " + "Could not resolve executable '%s' on PATH; HTCondor resolves an " "executable name relative to the job sandbox, so the job may be held.", executable, ) diff --git a/tests/test_prepare_utils.py b/tests/test_prepare_utils.py index c968e4e..567d400 100644 --- a/tests/test_prepare_utils.py +++ b/tests/test_prepare_utils.py @@ -238,7 +238,9 @@ def testMakeCommandBareExecutableNotOnPath(self): # A bare executable name that cannot be found on PATH is left as is. gw_exec = GenericWorkflowExec("nosuchcmd", "nosuchcmd_xyzzy") gw, gwjob = self._make_job(executable=gw_exec) - jobcmds = prepare_utils._translate_command_line(self.cached_vals, gw, gwjob) + with self.assertLogs(level="WARNING") as cm_log: + jobcmds = prepare_utils._translate_command_line(self.cached_vals, gw, gwjob) + self.assertRegex(cm_log.output[0], "Could not resolve executable 'nosuchcmd_xyzzy' on PATH.*") self.assertEqual(jobcmds["executable"], "nosuchcmd_xyzzy") def testMakeCommandArguments(self):