Only add python host/run dependency when a package actually needs it

vinca/recipes.py

@@ -51,7 +51,24 @@
             "then": ["${{ stdlib('c') }}"],
         },
         "ninja",
-        "python",
+        # ament's own CMake/build tooling shells out to Python for boilerplate
+        # (environment hooks, package.xml parsing, index generation) regardless
+        # of whether the package being built has any Python content itself, so
+        # every package needs *some* interpreter present at build time. Pinned
+        # to the single python_min version (rather than left as a bare
+        # "python") so this doesn't drag a package that needs no interpreter
+        # into the full python-version build matrix: a fully-resolved version
+        # constraint like this is invisible to rattler-build's variant/used_vars
+        # scan, confirmed via `rattler-build build --render-only` producing one
+        # variant either way, vs. N variants (one per pinned python version)
+        # for a bare "python" -- see _package_needs_python for the actual
+        # per-package host/run python dependency this is distinct from.
+        # `default('3.11')` covers recipe.yaml consumers whose own
+        # conda_build_config.yaml doesn't define python_min at all (e.g. a
+        # minimal, hand-maintained pinning file rather than a full
+        # conda-forge-pinning merge) -- an undefined python_min renders to an
+        # empty string, producing the invalid match spec "python .*".
+        "python ${{ python_min | default('3.11') }}.*",
         "setuptools",
         "git",
         "git-lfs",
@@ -68,13 +85,90 @@
     ],
     "host": [
         {"if": "build_platform == target_platform", "then": ["pkg-config"]},
-        "python",
-        "numpy",
-        "pip",
     ],
     "run": [],
 }
 
+# Dependency names that indicate a package's own recipe genuinely needs a
+# Python interpreter/ABI in host/run (as opposed to Python merely being a
+# build-time tool for ament's own scripts, handled unconditionally above).
+#
+# Deliberately NOT included: rosidl_default_generators/rosidl_generator_py.
+# Real-world case found while testing on ros-humble: rclcpp (a pure C++
+# library, not a rosidl_interface_packages member) declares
+# <test_depend>rosidl_default_generators</test_depend> solely to generate
+# *test* message types (test_msgs) for its own test suite -- nothing to do
+# with rclcpp's own shipped artifact having Python content. The
+# member_of_groups check above is the precise, authoritative signal for "this
+# package itself has rosidl-generated Python bindings"; these two names would
+# only ever add noise on top of it.
+_PYTHON_DEPENDENCY_MARKERS = frozenset(
+    {
+        "rclpy",
+        "pybind11",
+        "python_cmake_module",
+        "ament_cmake_python",
+    }
+)
+
+
+def _package_needs_python(package: catkin_pkg.package.Package, build_type: str) -> bool:
+    """Decide whether a package's OWN artifact needs Python (host/run), ahead of
+    building anything.
+
+    This is deliberately conservative in the direction of false positives: a
+    package wrongly marked as needing Python just gets rebuilt once per Python
+    version for no benefit, whereas a package wrongly marked as NOT needing it
+    would silently be skipped when rebuilding for additional Python versions
+    and ship a stale/missing artifact for those versions. So every check here
+    is an "if in doubt, say yes":
+
+    * ``ament_python`` packages are pure Python by construction.
+    * Any ``rosidl_interface_packages`` member (i.e. it has .msg/.srv/.action
+      files) gets rosidl_generator_py-compiled Python bindings unconditionally,
+      independent of what build_type or explicit dependencies it declares.
+    * Anything that actually depends (build, exec, run, or test -- a test-only
+      dependency still means Python must be importable to run the test suite
+      during the build) on a known Python-flavored package name, or on any
+      rosdep key containing "python" or starting with "pybind" (catches the
+      python3-*/python-* rosdep naming conventions along with pybind11
+      variants), needs Python.
+
+    ``buildtool_depend``/``buildtool_export_depend`` are deliberately NOT
+    scanned: by ROS's own convention that tag means "a tool needed to invoke
+    the build system" (e.g. many ament_cmake packages declare a plain
+    ``<buildtool_depend>python3</buildtool_depend>`` purely so a codegen
+    script can run at build time), never "this package's shipped artifact
+    contains Python content" -- and build-time-only Python is already covered
+    unconditionally by the fixed ``python_min``-pinned entry every recipe
+    gets in ``_BASE_REQUIREMENTS``.
+
+    Everything else -- the vast majority of ROS packages, which are plain C/
+    C++ libraries and nodes -- does not, and skips the Python host/run
+    dependency (and therefore the per-Python-version rebuild) entirely.
+    """
+    if build_type == "ament_python":
+        return True
+    if any(
+        group.name == "rosidl_interface_packages" for group in package.member_of_groups
+    ):
+        return True
+    dependency_names = {
+        dependency.name
+        for dependency in (
+            *package.build_depends,
+            *package.build_export_depends,
+            *package.exec_depends,
+            *package.run_depends,
+            *package.test_depends,
+        )
+    }
+    if dependency_names & _PYTHON_DEPENDENCY_MARKERS:
+        return True
+    return any(
+        "python" in name or name.startswith("pybind") for name in dependency_names
+    )
+
 
 def get_depmods(
     vinca_conf: dict[str, Any], package_name: str, distro: Distro
@@ -347,12 +441,15 @@ def generate_output(
     package = catkin_pkg.package.parse_package_string(xml)
     package.evaluate_conditions(os.environ)
 
-    python_dependencies = resolve_pkgname("python", vinca_conf, distro)
-    output["requirements"]["run"].extend(python_dependencies)
-    output["requirements"]["host"].extend(python_dependencies)
-
     is_dummy = is_dummy_metapackage(shortname, vinca_conf)
     build_type = package.get_build_type()
+
+    if not is_dummy and _package_needs_python(package, build_type):
+        python_dependencies = resolve_pkgname("python", vinca_conf, distro)
+        output["requirements"]["run"].extend(python_dependencies)
+        output["requirements"]["host"].extend(python_dependencies)
+        output["requirements"]["host"].extend(["numpy", "pip"])
+
     if not is_dummy:
         try:
             output["build"]["script"] = _BUILD_SCRIPTS[build_type]
@@ -412,12 +509,20 @@ def generate_output(
     # Build tools normally belong in `host`, but git has to be in `build` so that it
     # is runnable on the build machine when cross-compiling. cmake is already part of
     # the base build requirements, so re-adding it would only create a duplicate.
+    # Likewise python3/python (a common buildtool_depend for packages that just need
+    # an interpreter present to run a codegen script, e.g. rclcpp's
+    # <buildtool_depend>python3</buildtool_depend> for ament_cmake_gen_version_h):
+    # re-adding a bare "python" here would drag the package back into the full
+    # python-version build matrix that _package_needs_python's build-time-only,
+    # python_min-pinned base entry exists specifically to avoid.
     for dependency in build_tools:
         resolved = resolve_pkgname(dependency, vinca_conf, distro)
         if not resolved:
             unsatisfied.add(dependency)
         elif "git" in resolved:
             output["requirements"]["build"].extend(resolved)
+        elif resolved == ["python"]:
+            pass
         elif dependency != "cmake":
             build_dependencies.append(dependency)
 

vinca/templates/build_ament_cmake.sh.in

@@ -14,6 +14,24 @@ cd build
 # necessary for correctly linking SIP files (from python_qt_bindings)
 export LINK=$CXX
 
+# Speed up repeated local builds (e.g. rebuilding the same packages across
+# several Python-version passes) by routing the C/C++ compiler through
+# sccache -- opt-in via VINCA_USE_SCCACHE=1, rather than auto-detecting
+# sccache on PATH, so this never silently changes compiler invocations
+# just because sccache happens to be installed for some unrelated reason.
+# Requires the top-level `rattler-build build` invocation to pass
+# --no-build-id, since both ccache and sccache are sensitive to the
+# timestamped build-directory paths rattler-build uses by default.
+SCCACHE_CMAKE_ARGS=""
+if [[ "${VINCA_USE_SCCACHE:-}" == "1" ]]; then
+  if command -v sccache >/dev/null 2>&1; then
+    SCCACHE_CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"
+  else
+    echo "VINCA_USE_SCCACHE=1 but sccache was not found on PATH" >&2
+    exit 1
+  fi
+fi
+
 if [[ "$CONDA_BUILD_CROSS_COMPILATION" != "1" ]]; then
   PYTHON_EXECUTABLE=$PREFIX/bin/python
   PKG_CONFIG_EXECUTABLE=$PREFIX/bin/pkg-config
@@ -58,8 +76,22 @@ fi;
 # PYTHON_INSTALL_DIR should be a relative path, see
 # https://github.com/ament/ament_cmake/blob/2.3.2/ament_cmake_python/README.md
 # So we compute the relative path of $SP_DIR w.r.t. to $PREFIX,
-# but it is not trivial to do this in bash scripting, so let's do it via python
-export PYTHON_INSTALL_DIR=`python -c "import os;print(os.path.relpath(os.environ['SP_DIR'],os.environ['PREFIX']))"`
+# but it is not trivial to do this in bash scripting, so let's do it via python.
+# rattler-build only sets $SP_DIR when python is one of the recipe's own host
+# dependencies (vinca now only adds that for packages it detects actually need
+# Python) -- but a handful of packages call ament_python_install_package() in
+# their own CMakeLists for some small internal helper without declaring a
+# Python-flavored dependency in package.xml at all (e.g. ament_cmake_test's
+# test-utility module), so $SP_DIR can be legitimately unset even though this
+# specific build still needs a real PYTHON_INSTALL_DIR. Fall back to computing
+# the standard site-packages layout from whatever python is on PATH (the
+# build-time-only interpreter every recipe gets, at minimum) instead of
+# failing outright.
+if [[ -n "${SP_DIR:-}" ]]; then
+  export PYTHON_INSTALL_DIR=`python -c "import os;print(os.path.relpath(os.environ['SP_DIR'],os.environ['PREFIX']))"`
+else
+  export PYTHON_INSTALL_DIR=`python -c "import sys;print('lib/python%d.%d/site-packages' % sys.version_info[:2])"`
+fi
 echo "Using PYTHON_INSTALL_DIR: $PYTHON_INSTALL_DIR"
 
 if [[ $target_platform =~ emscripten.* ]]; then
@@ -132,6 +164,7 @@ $CMAKE_GEN \
     -DCMAKE_OSX_DEPLOYMENT_TARGET=$OSX_DEPLOYMENT_TARGET \
     --compile-no-warning-as-error \
     $EXTRA_CMAKE_ARGS \
+    $SCCACHE_CMAKE_ARGS \
     @(additional_cmake_args) \
     $WORK_DIR
 

vinca/templates/build_catkin.sh.in

@@ -19,6 +19,24 @@ cd build
 # necessary for correctly linking SIP files (from python_qt_bindings)
 export LINK=$CXX
 
+# Speed up repeated local builds (e.g. rebuilding the same packages across
+# several Python-version passes) by routing the C/C++ compiler through
+# sccache -- opt-in via VINCA_USE_SCCACHE=1, rather than auto-detecting
+# sccache on PATH, so this never silently changes compiler invocations
+# just because sccache happens to be installed for some unrelated reason.
+# Requires the top-level `rattler-build build` invocation to pass
+# --no-build-id, since both ccache and sccache are sensitive to the
+# timestamped build-directory paths rattler-build uses by default.
+SCCACHE_CMAKE_ARGS=""
+if [[ "${VINCA_USE_SCCACHE:-}" == "1" ]]; then
+  if command -v sccache >/dev/null 2>&1; then
+    SCCACHE_CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"
+  else
+    echo "VINCA_USE_SCCACHE=1 but sccache was not found on PATH" >&2
+    exit 1
+  fi
+fi
+
 if [[ "$CONDA_BUILD_CROSS_COMPILATION" != "1" ]]; then
   PYTHON_EXECUTABLE=$PREFIX/bin/python
   PKG_CONFIG_EXECUTABLE=$PREFIX/bin/pkg-config
@@ -109,6 +127,7 @@ cmake ${CMAKE_ARGS} --compile-no-warning-as-error \
          -DCATKIN_BUILD_BINARY_PACKAGE=$CATKIN_BUILD_BINARY_PACKAGE \
          -DCMAKE_OSX_DEPLOYMENT_TARGET=$OSX_DEPLOYMENT_TARGET \
          $EXTRA_CMAKE_ARGS \
+         $SCCACHE_CMAKE_ARGS \
          @(additional_cmake_args) \
          -G "$GENERATOR" \
          $SRC_DIR/$PKG_NAME/src/work/@(additional_folder)