From c62e75224ce2ed341758137c2bb98fdbe995ae33 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Tue, 8 Sep 2026 10:05:20 +1000 Subject: [PATCH 01/10] feat: only add python host/run dependency when a package actually needs it Every non-dummy recipe unconditionally got python (plus numpy/pip) in host and run, regardless of whether the package itself has any Python content. Since host/run python participates in the pinned python variant, this drags every single ROS package into a full rebuild any time the build wants to support more than one Python version -- even pure C++ libraries and nodes, which are the vast majority. Add _package_needs_python(): a conservative (false-positives-ok, false-negatives-not) ahead-of-time heuristic based on data vinca already parses from package.xml -- ament_python build type, rosidl interface package membership (msg/srv/action always compile Python bindings via rosidl_generator_py), or an actual python-flavored dependency (rclpy, pybind11, any python3-*/python-* rosdep key). Only packages matching one of these get python/numpy/pip in host and python in run. The build-time-only python entry every ament_cmake package still needs (ament's own CMake tooling shells out to Python for boilerplate regardless of the package's own content) is kept, but pinned to the single python_min version via Jinja rather than left as a bare "python" -- confirmed via `rattler-build build --render-only` that a fully-resolved version constraint like this doesn't participate in the variant/used_vars scan (one build variant either way), while a bare "python" produces one variant per pinned python version. --- vinca/recipes.py | 93 +++++++++++++++++++++++++++++++++++++++---- vinca/test_recipes.py | 86 +++++++++++++++++++++++++++++++++++---- 2 files changed, 163 insertions(+), 16 deletions(-) diff --git a/vinca/recipes.py b/vinca/recipes.py index dbba622..e7f3499 100644 --- a/vinca/recipes.py +++ b/vinca/recipes.py @@ -51,7 +51,19 @@ "then": ["${{ stdlib('c') }}"], }, "ninja", - "python", + # ament's own CMake/build tooling shells out to Python for boilerplate + # (environment hooks, package.xml parsing, index generation) regardless + # of whether the package being built has any Python content itself, so + # every package needs *some* interpreter present at build time. Pinned + # to the single python_min version (rather than left as a bare + # "python") so this doesn't drag a package that needs no interpreter + # into the full python-version build matrix: a fully-resolved version + # constraint like this is invisible to rattler-build's variant/used_vars + # scan, confirmed via `rattler-build build --render-only` producing one + # variant either way, vs. N variants (one per pinned python version) + # for a bare "python" -- see _package_needs_python for the actual + # per-package host/run python dependency this is distinct from. + "python ${{ python_min }}.*", "setuptools", "git", "git-lfs", @@ -68,13 +80,75 @@ ], "host": [ {"if": "build_platform == target_platform", "then": ["pkg-config"]}, - "python", - "numpy", - "pip", ], "run": [], } +# Dependency names that indicate a package's own recipe genuinely needs a +# Python interpreter/ABI in host/run (as opposed to Python merely being a +# build-time tool for ament's own scripts, handled unconditionally above). +_PYTHON_DEPENDENCY_MARKERS = frozenset( + { + "rclpy", + "pybind11", + "python_cmake_module", + "ament_cmake_python", + "rosidl_default_generators", + "rosidl_generator_py", + } +) + + +def _package_needs_python(package: catkin_pkg.package.Package, build_type: str) -> bool: + """Decide whether a package's OWN artifact needs Python (host/run), ahead of + building anything. + + This is deliberately conservative in the direction of false positives: a + package wrongly marked as needing Python just gets rebuilt once per Python + version for no benefit, whereas a package wrongly marked as NOT needing it + would silently be skipped when rebuilding for additional Python versions + and ship a stale/missing artifact for those versions. So every check here + is an "if in doubt, say yes": + + * ``ament_python`` packages are pure Python by construction. + * Any ``rosidl_interface_packages`` member (i.e. it has .msg/.srv/.action + files) gets rosidl_generator_py-compiled Python bindings unconditionally, + independent of what build_type or explicit dependencies it declares. + * Anything that actually depends (build, buildtool, exec, run, or test -- + a test-only dependency still means Python must be importable to run the + test suite during the build) on a known Python-flavored package name, or + on any rosdep key containing "python" or starting with "pybind" + (catches the python3-*/python-* rosdep naming conventions along with + pybind11 variants), needs Python. + + Everything else -- the vast majority of ROS packages, which are plain C/ + C++ libraries and nodes -- does not, and skips the Python host/run + dependency (and therefore the per-Python-version rebuild) entirely. + """ + if build_type == "ament_python": + return True + if any( + group.name == "rosidl_interface_packages" for group in package.member_of_groups + ): + return True + dependency_names = { + dependency.name + for dependency in ( + *package.build_depends, + *package.build_export_depends, + *package.buildtool_depends, + *package.buildtool_export_depends, + *package.exec_depends, + *package.run_depends, + *package.test_depends, + ) + } + if dependency_names & _PYTHON_DEPENDENCY_MARKERS: + return True + return any( + "python" in name or name.startswith("pybind") for name in dependency_names + ) + def get_depmods( vinca_conf: dict[str, Any], package_name: str, distro: Distro @@ -347,12 +421,15 @@ def generate_output( package = catkin_pkg.package.parse_package_string(xml) package.evaluate_conditions(os.environ) - python_dependencies = resolve_pkgname("python", vinca_conf, distro) - output["requirements"]["run"].extend(python_dependencies) - output["requirements"]["host"].extend(python_dependencies) - is_dummy = is_dummy_metapackage(shortname, vinca_conf) build_type = package.get_build_type() + + if not is_dummy and _package_needs_python(package, build_type): + python_dependencies = resolve_pkgname("python", vinca_conf, distro) + output["requirements"]["run"].extend(python_dependencies) + output["requirements"]["host"].extend(python_dependencies) + output["requirements"]["host"].extend(["numpy", "pip"]) + if not is_dummy: try: output["build"]["script"] = _BUILD_SCRIPTS[build_type] diff --git a/vinca/test_recipes.py b/vinca/test_recipes.py index 2809d87..b9b1f89 100644 --- a/vinca/test_recipes.py +++ b/vinca/test_recipes.py @@ -14,6 +14,7 @@ https://github.com/example/{name} {depends} {build_type} +{member_of_group} """ @@ -32,9 +33,12 @@ } -def package_xml(name, build_type="ament_cmake", depends=()): +def package_xml(name, build_type="ament_cmake", depends=(), member_of_group=None): body = "\n".join(f" <{tag}>{value}" for tag, value in depends) - return PACKAGE_XML.format(name=name, depends=body, build_type=build_type) + group = f" {member_of_group}" if member_of_group else "" + return PACKAGE_XML.format( + name=name, depends=body, build_type=build_type, member_of_group=group + ) class FakeDistro: @@ -95,12 +99,13 @@ def build( name, depends=(), build_type="ament_cmake", + member_of_group=None, unsatisfied=None, dependencies_only=False, **overrides, ): distro = FakeDistro( - {name: package_xml(name, build_type, depends)}, + {name: package_xml(name, build_type, depends, member_of_group)}, repository_by_name={name: f"https://github.com/ros2/{name}.git"}, ) return generate_output( @@ -127,7 +132,7 @@ def test_generate_output_produces_a_complete_recipe(): "then": ["${{ stdlib('c') }}"], }, "ninja", - "python", + "python ${{ python_min }}.*", "setuptools", "git", "git-lfs", @@ -149,16 +154,18 @@ def test_generate_output_produces_a_complete_recipe(): ], }, ], + # A plain build_depend on rclcpp (the C++ client library) carries no + # Python-flavored dependency and isn't a rosidl interface package, so + # this recipe correctly gets no python/numpy/pip host dependency and + # no python run dependency -- see test_needs_python_* below for the + # cases that do. "host": [ {"if": "build_platform == target_platform", "then": ["pkg-config"]}, - "numpy", - "pip", - "python", "ros2-rclcpp", "ros2-ros-environment", "ros2-ros-workspace", ], - "run": ["python", "ros2-ros-workspace"], + "run": ["ros2-ros-workspace"], }, "build": { "script": "${{ '$RECIPE_DIR/build_ament_cmake.sh' if unix or wasm32 " @@ -222,6 +229,69 @@ def test_every_duplicate_cmake_is_replaced_by_the_selector(): ) +def _host_run_python_markers(output): + host = output["requirements"]["host"] + run = output["requirements"]["run"] + return "python" in host, "numpy" in host, "pip" in host, "python" in run + + +def test_needs_python_plain_cpp_package_gets_no_python_dependency(): + # A pure C++ library/node (no rosidl interfaces, no python-flavored + # dependency) must not be pulled into the python host/run dependency -- + # and therefore not into a per-python-version rebuild -- at all. + output = build("demo", depends=[("build_depend", "rclcpp")]) + + assert _host_run_python_markers(output) == (False, False, False, False) + # The build-time-only interpreter ament's own tooling needs is still + # present, but pinned to a single fixed version rather than the bare + # "python" that would drag this recipe into the python variant matrix. + assert "python ${{ python_min }}.*" in output["requirements"]["build"] + assert "python" not in output["requirements"]["build"] + + +def test_needs_python_rosidl_interface_package_gets_python_dependency(): + # msg/srv/action packages declare membership in rosidl_interface_packages; + # rosidl_generator_py always compiles Python bindings for these, + # independent of whatever build_type or explicit depends they declare. + output = build( + "demo", + depends=[("build_depend", "rosidl_default_generators")], + member_of_group="rosidl_interface_packages", + ) + + assert _host_run_python_markers(output) == (True, True, True, True) + + +def test_needs_python_rclpy_dependency_gets_python_dependency(): + output = build("demo", depends=[("exec_depend", "rclpy")]) + + assert _host_run_python_markers(output) == (True, True, True, True) + + +def test_needs_python_rosdep_python3_key_gets_python_dependency(): + # Broad, deliberately permissive catch-all for the python3-*/python-* + # rosdep naming convention: false positives here are cheap (one extra + # per-python-version rebuild), false negatives are not (a silently + # missing artifact for the versions it wasn't rebuilt for). + output = build("demo", depends=[("exec_depend", "python3-yaml")]) + + assert _host_run_python_markers(output) == (True, True, True, True) + + +def test_needs_python_ament_python_build_type_gets_python_dependency(): + output = build("demo", build_type="ament_python") + + assert _host_run_python_markers(output) == (True, True, True, True) + + +def test_needs_python_test_only_dependency_still_counts(): + # A test-only dependency still means Python must be importable to run + # the test suite during the build, so it counts too. + output = build("demo", depends=[("test_depend", "rclpy")]) + + assert _host_run_python_markers(output) == (True, True, True, True) + + def test_mimick_vendor_moves_from_host_to_build(): output = build("demo", depends=[("build_depend", "mimick_vendor")]) From 1af23bd35e13e464f78e0cda65a45ce7452648f0 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Tue, 8 Sep 2026 10:07:05 +1000 Subject: [PATCH 02/10] feat: route the ament_cmake build script's C/C++ compiler through sccache Rebuilding the same package across several Python-version passes (the whole point of the conditional-python-dependency change) recompiles a lot of identical C/C++ source, since only the final Python-linking bits actually differ between versions. Route CMAKE_C/CXX_COMPILER_LAUNCHER through sccache when it's present in the build environment (a no-op otherwise, so this is safe for anyone without sccache installed) to turn that redundant recompilation into cache hits. Per rattler-build's own sccache/ccache guidance, the top-level `rattler-build build` invocation needs --no-build-id alongside this, since both tools are sensitive to the timestamped build-directory path rattler-build uses by default. --- vinca/templates/build_ament_cmake.sh.in | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/vinca/templates/build_ament_cmake.sh.in b/vinca/templates/build_ament_cmake.sh.in index cc25a25..d87b554 100644 --- a/vinca/templates/build_ament_cmake.sh.in +++ b/vinca/templates/build_ament_cmake.sh.in @@ -14,6 +14,17 @@ cd build # necessary for correctly linking SIP files (from python_qt_bindings) export LINK=$CXX +# Speed up repeated local builds (e.g. rebuilding the same packages across +# several Python-version passes) by routing the C/C++ compiler through +# sccache when it's present in the build environment; a no-op otherwise. +# Requires the top-level `rattler-build build` invocation to pass +# --no-build-id, since both ccache and sccache are sensitive to the +# timestamped build-directory paths rattler-build uses by default. +SCCACHE_CMAKE_ARGS="" +if command -v sccache >/dev/null 2>&1; then + SCCACHE_CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache" +fi + if [[ "$CONDA_BUILD_CROSS_COMPILATION" != "1" ]]; then PYTHON_EXECUTABLE=$PREFIX/bin/python PKG_CONFIG_EXECUTABLE=$PREFIX/bin/pkg-config @@ -132,6 +143,7 @@ $CMAKE_GEN \ -DCMAKE_OSX_DEPLOYMENT_TARGET=$OSX_DEPLOYMENT_TARGET \ --compile-no-warning-as-error \ $EXTRA_CMAKE_ARGS \ + $SCCACHE_CMAKE_ARGS \ @(additional_cmake_args) \ $WORK_DIR From 1a12e9b377c2851a736ad7b787339f571242385e Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Tue, 8 Sep 2026 10:12:42 +1000 Subject: [PATCH 03/10] fix: don't scan buildtool_depend for the needs-python heuristic Real-world case found while testing on ros-humble: rclcpp (a pure C++ library with no Python content) declares python3 purely so ament_cmake_gen_version_h can run a codegen script at build time. That was matching the "python" substring catch-all and marking rclcpp as needing a per-Python-version rebuild for no actual benefit. buildtool_depend means "a tool needed to invoke this package's build system" by ROS's own convention, never "this package's shipped artifact has Python content" -- and build-time-only Python is already covered unconditionally by the fixed python_min-pinned entry every recipe gets. Excluding buildtool_depend/buildtool_export_depend from the scan is a precision improvement, not a risk: every other dependency type (build, exec, run, test) is still scanned, so this doesn't reopen any false-negative risk. --- vinca/recipes.py | 23 +++++++++++++++-------- vinca/test_recipes.py | 13 +++++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/vinca/recipes.py b/vinca/recipes.py index e7f3499..e815ff0 100644 --- a/vinca/recipes.py +++ b/vinca/recipes.py @@ -114,12 +114,21 @@ def _package_needs_python(package: catkin_pkg.package.Package, build_type: str) * Any ``rosidl_interface_packages`` member (i.e. it has .msg/.srv/.action files) gets rosidl_generator_py-compiled Python bindings unconditionally, independent of what build_type or explicit dependencies it declares. - * Anything that actually depends (build, buildtool, exec, run, or test -- - a test-only dependency still means Python must be importable to run the - test suite during the build) on a known Python-flavored package name, or - on any rosdep key containing "python" or starting with "pybind" - (catches the python3-*/python-* rosdep naming conventions along with - pybind11 variants), needs Python. + * Anything that actually depends (build, exec, run, or test -- a test-only + dependency still means Python must be importable to run the test suite + during the build) on a known Python-flavored package name, or on any + rosdep key containing "python" or starting with "pybind" (catches the + python3-*/python-* rosdep naming conventions along with pybind11 + variants), needs Python. + + ``buildtool_depend``/``buildtool_export_depend`` are deliberately NOT + scanned: by ROS's own convention that tag means "a tool needed to invoke + the build system" (e.g. many ament_cmake packages declare a plain + ``python3`` purely so a codegen + script can run at build time), never "this package's shipped artifact + contains Python content" -- and build-time-only Python is already covered + unconditionally by the fixed ``python_min``-pinned entry every recipe + gets in ``_BASE_REQUIREMENTS``. Everything else -- the vast majority of ROS packages, which are plain C/ C++ libraries and nodes -- does not, and skips the Python host/run @@ -136,8 +145,6 @@ def _package_needs_python(package: catkin_pkg.package.Package, build_type: str) for dependency in ( *package.build_depends, *package.build_export_depends, - *package.buildtool_depends, - *package.buildtool_export_depends, *package.exec_depends, *package.run_depends, *package.test_depends, diff --git a/vinca/test_recipes.py b/vinca/test_recipes.py index b9b1f89..2dc5758 100644 --- a/vinca/test_recipes.py +++ b/vinca/test_recipes.py @@ -292,6 +292,19 @@ def test_needs_python_test_only_dependency_still_counts(): assert _host_run_python_markers(output) == (True, True, True, True) +def test_needs_python_ignores_buildtool_depend_on_python3(): + # Real-world case: rclcpp (a pure C++ library with no Python content of + # its own) declares python3 purely + # so ament_cmake_gen_version_h can run a codegen script at build time. + # buildtool_depend means "a tool needed to invoke the build", never + # "this package's shipped artifact has Python content", so it must not + # trigger the python host/run dependency (build-time-only Python is + # already covered unconditionally elsewhere). + output = build("demo", depends=[("buildtool_depend", "python3")]) + + assert _host_run_python_markers(output) == (False, False, False, False) + + def test_mimick_vendor_moves_from_host_to_build(): output = build("demo", depends=[("build_depend", "mimick_vendor")]) From 85489cd2bb2ef5c2dd03dd45ec012b959c95f112 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Tue, 8 Sep 2026 10:13:56 +1000 Subject: [PATCH 04/10] fix: drop rosidl_default_generators/rosidl_generator_py from the python markers Another real-world false trigger found on ros-humble: rclcpp (not a rosidl_interface_packages member) declares rosidl_default_generators solely to generate test_msgs for its own test suite, unrelated to rclcpp's own shipped artifact having Python content. The member_of_groups check is the precise, authoritative signal for "this package itself has rosidl-generated Python bindings" (every rosidl interface package declares that membership by ROS convention); these two marker names only ever added noise on top of it, with a demonstrated collision. Dropping them from the curated set doesn't reopen any false-negative risk -- the group-membership check still catches every genuine rosidl interface package unconditionally. --- vinca/recipes.py | 12 ++++++++++-- vinca/test_recipes.py | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/vinca/recipes.py b/vinca/recipes.py index e815ff0..9d4a7d6 100644 --- a/vinca/recipes.py +++ b/vinca/recipes.py @@ -87,14 +87,22 @@ # Dependency names that indicate a package's own recipe genuinely needs a # Python interpreter/ABI in host/run (as opposed to Python merely being a # build-time tool for ament's own scripts, handled unconditionally above). +# +# Deliberately NOT included: rosidl_default_generators/rosidl_generator_py. +# Real-world case found while testing on ros-humble: rclcpp (a pure C++ +# library, not a rosidl_interface_packages member) declares +# rosidl_default_generators solely to generate +# *test* message types (test_msgs) for its own test suite -- nothing to do +# with rclcpp's own shipped artifact having Python content. The +# member_of_groups check above is the precise, authoritative signal for "this +# package itself has rosidl-generated Python bindings"; these two names would +# only ever add noise on top of it. _PYTHON_DEPENDENCY_MARKERS = frozenset( { "rclpy", "pybind11", "python_cmake_module", "ament_cmake_python", - "rosidl_default_generators", - "rosidl_generator_py", } ) diff --git a/vinca/test_recipes.py b/vinca/test_recipes.py index 2dc5758..3d148ba 100644 --- a/vinca/test_recipes.py +++ b/vinca/test_recipes.py @@ -305,6 +305,20 @@ def test_needs_python_ignores_buildtool_depend_on_python3(): assert _host_run_python_markers(output) == (False, False, False, False) +def test_needs_python_ignores_test_only_rosidl_default_generators(): + # Real-world case: rclcpp (not a rosidl_interface_packages member) has + # rosidl_default_generators solely to + # generate test_msgs for its own test suite -- nothing to do with + # rclcpp's own shipped artifact having Python content. The + # member_of_groups check is the precise signal for "this package itself + # has rosidl-generated Python bindings" (see the positive case above); + # rosidl_default_generators/rosidl_generator_py depended on for any other + # reason must not, by itself, trigger the python host/run dependency. + output = build("demo", depends=[("test_depend", "rosidl_default_generators")]) + + assert _host_run_python_markers(output) == (False, False, False, False) + + def test_mimick_vendor_moves_from_host_to_build(): output = build("demo", depends=[("build_depend", "mimick_vendor")]) From c3322d0b63f79f12ff74a56a97548c67aac78b7c Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Tue, 8 Sep 2026 10:17:43 +1000 Subject: [PATCH 05/10] fix: don't re-add a bare python host dep for a buildtool_depend that resolves to it Found by actually generating recipes against ros-humble, not just unit tests: rclcpp's python3 takes a SEPARATE code path from _package_needs_python entirely -- vinca's existing build_tools loop resolves every buildtool_depend and folds whatever isn't "git" or "cmake" into build_dependencies, which then gets resolved into requirements.host. python3 resolves (via robostack.yaml) to the conda "python" package, so it was landing back in host as a bare, un-pinned "python" -- reopening exactly the variant-matrix problem the python_min-pinned base entry exists to avoid, regardless of what _package_needs_python decided. Skip re-adding it when a buildtool_depend resolves to exactly ["python"]: the base entry already provides a working build-time interpreter for every package, buildtool_depend or not, so this is never a loss of function -- purely removing a redundant, un-pinned duplicate. Also fixes the test double: SYSTEM_PACKAGES lacked a "python3" entry, so test_needs_python_ignores_buildtool_depend_on_python3 was passing for the wrong reason (the fake resolver never produced "python" from "python3" at all, real robostack.yaml does) rather than exercising this fix. --- vinca/recipes.py | 8 ++++++++ vinca/test_recipes.py | 3 +++ 2 files changed, 11 insertions(+) diff --git a/vinca/recipes.py b/vinca/recipes.py index 9d4a7d6..8fcb0a5 100644 --- a/vinca/recipes.py +++ b/vinca/recipes.py @@ -504,12 +504,20 @@ def generate_output( # Build tools normally belong in `host`, but git has to be in `build` so that it # is runnable on the build machine when cross-compiling. cmake is already part of # the base build requirements, so re-adding it would only create a duplicate. + # Likewise python3/python (a common buildtool_depend for packages that just need + # an interpreter present to run a codegen script, e.g. rclcpp's + # python3 for ament_cmake_gen_version_h): + # re-adding a bare "python" here would drag the package back into the full + # python-version build matrix that _package_needs_python's build-time-only, + # python_min-pinned base entry exists specifically to avoid. for dependency in build_tools: resolved = resolve_pkgname(dependency, vinca_conf, distro) if not resolved: unsatisfied.add(dependency) elif "git" in resolved: output["requirements"]["build"].extend(resolved) + elif resolved == ["python"]: + pass elif dependency != "cmake": build_dependencies.append(dependency) diff --git a/vinca/test_recipes.py b/vinca/test_recipes.py index 3d148ba..5a7377f 100644 --- a/vinca/test_recipes.py +++ b/vinca/test_recipes.py @@ -22,6 +22,9 @@ SYSTEM_PACKAGES = { "python": ["python"], + # matches robostack.yaml's real rosdep mapping: python3 -> the same + # conda "python" package as the plain "python" rosdep key. + "python3": ["python"], "python-setuptools": ["setuptools"], "cmake": ["cmake"], "git": ["git"], From fd16e65791da545b9c2c69aa1de1715414055a17 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Tue, 8 Sep 2026 10:40:13 +1000 Subject: [PATCH 06/10] fix: fall back to a computed site-packages path when SP_DIR is unset Found by building ros-humble with the new conditional python dependency: ament_cmake_test's own CMakeLists calls ament_python_install_package() for a small internal test-utility module, without declaring any Python-flavored dependency in its package.xml -- so _package_needs_python correctly doesn't add python to its host requirements (nothing in its own metadata says it needs it), but that also means rattler-build never sets $SP_DIR for its build (SP_DIR is only populated when python is a host dependency), and the unconditional `os.environ['SP_DIR']` lookup in build_ament_cmake.sh.in raised a KeyError. Under `set -e`, a failing command substitution inside a plain assignment doesn't itself abort the script, so this silently produced an EMPTY PYTHON_INSTALL_DIR for every affected package -- harmless for most (they never call ament_python_install_package), but a hard CMake configure error for the ones that do. Fall back to computing the standard lib/pythonX.Y/site-packages layout from whatever python is on PATH (the build-time-only interpreter every recipe gets at minimum, per the python_min-pinned base requirement) when $SP_DIR isn't set, instead of failing outright. --- vinca/templates/build_ament_cmake.sh.in | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/vinca/templates/build_ament_cmake.sh.in b/vinca/templates/build_ament_cmake.sh.in index d87b554..4653d6b 100644 --- a/vinca/templates/build_ament_cmake.sh.in +++ b/vinca/templates/build_ament_cmake.sh.in @@ -69,8 +69,22 @@ fi; # PYTHON_INSTALL_DIR should be a relative path, see # https://github.com/ament/ament_cmake/blob/2.3.2/ament_cmake_python/README.md # So we compute the relative path of $SP_DIR w.r.t. to $PREFIX, -# but it is not trivial to do this in bash scripting, so let's do it via python -export PYTHON_INSTALL_DIR=`python -c "import os;print(os.path.relpath(os.environ['SP_DIR'],os.environ['PREFIX']))"` +# but it is not trivial to do this in bash scripting, so let's do it via python. +# rattler-build only sets $SP_DIR when python is one of the recipe's own host +# dependencies (vinca now only adds that for packages it detects actually need +# Python) -- but a handful of packages call ament_python_install_package() in +# their own CMakeLists for some small internal helper without declaring a +# Python-flavored dependency in package.xml at all (e.g. ament_cmake_test's +# test-utility module), so $SP_DIR can be legitimately unset even though this +# specific build still needs a real PYTHON_INSTALL_DIR. Fall back to computing +# the standard site-packages layout from whatever python is on PATH (the +# build-time-only interpreter every recipe gets, at minimum) instead of +# failing outright. +if [[ -n "${SP_DIR:-}" ]]; then + export PYTHON_INSTALL_DIR=`python -c "import os;print(os.path.relpath(os.environ['SP_DIR'],os.environ['PREFIX']))"` +else + export PYTHON_INSTALL_DIR=`python -c "import sys;print('lib/python%d.%d/site-packages' % sys.version_info[:2])"` +fi echo "Using PYTHON_INSTALL_DIR: $PYTHON_INSTALL_DIR" if [[ $target_platform =~ emscripten.* ]]; then From 471bdc901b10cbf498f4043bb587cfa6cac52f78 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Tue, 8 Sep 2026 14:09:39 +1000 Subject: [PATCH 07/10] style: ruff format test_recipes.py CI's fmt-check caught this -- package_xml()'s member_of_group ternary needed ruff's multi-line wrapping. Co-Authored-By: Claude Sonnet 5 --- vinca/test_recipes.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vinca/test_recipes.py b/vinca/test_recipes.py index 5a7377f..52d632f 100644 --- a/vinca/test_recipes.py +++ b/vinca/test_recipes.py @@ -38,7 +38,11 @@ def package_xml(name, build_type="ament_cmake", depends=(), member_of_group=None): body = "\n".join(f" <{tag}>{value}" for tag, value in depends) - group = f" {member_of_group}" if member_of_group else "" + group = ( + f" {member_of_group}" + if member_of_group + else "" + ) return PACKAGE_XML.format( name=name, depends=body, build_type=build_type, member_of_group=group ) From 7265c3856dff901378042550704f3f226f02c9d6 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Tue, 8 Sep 2026 14:19:08 +1000 Subject: [PATCH 08/10] fix: also route build_catkin.sh's compiler through sccache build_ament_cmake.sh.in got sccache routing, but build_catkin.sh.in (used by vendored third-party CMake projects that aren't native ament packages -- e.g. fastrtps, fastcdr, the "cmake"/"catkin" build types) didn't. Found while trying to demonstrate sccache's benefit on fastrtps specifically: sccache reported 0 compile requests across a full rebuild, because this template's cmake invocation never set CMAKE_C/CXX_COMPILER_LAUNCHER at all. Same pattern as the ament_cmake template: no-op if sccache isn't installed. Co-Authored-By: Claude Sonnet 5 --- vinca/templates/build_catkin.sh.in | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/vinca/templates/build_catkin.sh.in b/vinca/templates/build_catkin.sh.in index 157fccf..422ae9b 100644 --- a/vinca/templates/build_catkin.sh.in +++ b/vinca/templates/build_catkin.sh.in @@ -19,6 +19,17 @@ cd build # necessary for correctly linking SIP files (from python_qt_bindings) export LINK=$CXX +# Speed up repeated local builds (e.g. rebuilding the same packages across +# several Python-version passes) by routing the C/C++ compiler through +# sccache when it's present in the build environment; a no-op otherwise. +# Requires the top-level `rattler-build build` invocation to pass +# --no-build-id, since both ccache and sccache are sensitive to the +# timestamped build-directory paths rattler-build uses by default. +SCCACHE_CMAKE_ARGS="" +if command -v sccache >/dev/null 2>&1; then + SCCACHE_CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache" +fi + if [[ "$CONDA_BUILD_CROSS_COMPILATION" != "1" ]]; then PYTHON_EXECUTABLE=$PREFIX/bin/python PKG_CONFIG_EXECUTABLE=$PREFIX/bin/pkg-config @@ -109,6 +120,7 @@ cmake ${CMAKE_ARGS} --compile-no-warning-as-error \ -DCATKIN_BUILD_BINARY_PACKAGE=$CATKIN_BUILD_BINARY_PACKAGE \ -DCMAKE_OSX_DEPLOYMENT_TARGET=$OSX_DEPLOYMENT_TARGET \ $EXTRA_CMAKE_ARGS \ + $SCCACHE_CMAKE_ARGS \ @(additional_cmake_args) \ -G "$GENERATOR" \ $SRC_DIR/$PKG_NAME/src/work/@(additional_folder) From 2cfe1323cb806246186de88534724fe7f5c0a05e Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Tue, 8 Sep 2026 14:34:57 +1000 Subject: [PATCH 09/10] fix: default python_min to 3.11 when a repo's pinning doesn't define it Found trying to apply this to RoboStack/ros-humble's actual main branch: unlike the (unmerged) branch this was originally validated against, main's conda_build_config.yaml is a minimal, hand-maintained file that doesn't define python_min at all (no full conda-forge-pinning merge). An undefined python_min renders to an empty string, producing the invalid match spec "python .*" and a hard recipe-parse failure -- not just for this recipe, for every single one, since every non-dummy recipe gets this base requirement. `${{ python_min | default('3.11') }}` degrades gracefully for any vinca consumer whose pinning file doesn't define it, instead of silently assuming a full conda-forge-pinning merge is always present. Co-Authored-By: Claude Sonnet 5 --- vinca/recipes.py | 7 ++++++- vinca/test_recipes.py | 7 +++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/vinca/recipes.py b/vinca/recipes.py index 8fcb0a5..f0eb75f 100644 --- a/vinca/recipes.py +++ b/vinca/recipes.py @@ -63,7 +63,12 @@ # variant either way, vs. N variants (one per pinned python version) # for a bare "python" -- see _package_needs_python for the actual # per-package host/run python dependency this is distinct from. - "python ${{ python_min }}.*", + # `default('3.11')` covers recipe.yaml consumers whose own + # conda_build_config.yaml doesn't define python_min at all (e.g. a + # minimal, hand-maintained pinning file rather than a full + # conda-forge-pinning merge) -- an undefined python_min renders to an + # empty string, producing the invalid match spec "python .*". + "python ${{ python_min | default('3.11') }}.*", "setuptools", "git", "git-lfs", diff --git a/vinca/test_recipes.py b/vinca/test_recipes.py index 52d632f..dee01d7 100644 --- a/vinca/test_recipes.py +++ b/vinca/test_recipes.py @@ -139,7 +139,7 @@ def test_generate_output_produces_a_complete_recipe(): "then": ["${{ stdlib('c') }}"], }, "ninja", - "python ${{ python_min }}.*", + "python ${{ python_min | default('3.11') }}.*", "setuptools", "git", "git-lfs", @@ -252,7 +252,10 @@ def test_needs_python_plain_cpp_package_gets_no_python_dependency(): # The build-time-only interpreter ament's own tooling needs is still # present, but pinned to a single fixed version rather than the bare # "python" that would drag this recipe into the python variant matrix. - assert "python ${{ python_min }}.*" in output["requirements"]["build"] + assert ( + "python ${{ python_min | default('3.11') }}.*" + in output["requirements"]["build"] + ) assert "python" not in output["requirements"]["build"] From 1bc88d1e074f03c34655b765bc257efc91e68ee6 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Tue, 8 Sep 2026 19:43:25 +1000 Subject: [PATCH 10/10] fix: make sccache routing opt-in via VINCA_USE_SCCACHE=1 Per @traversaro's review on RoboStack/vinca#153: auto-detecting sccache on PATH meant compiler invocations silently changed depending on whether sccache happened to be installed for some unrelated reason -- confusing to debug (e.g. two otherwise-identical machines producing different build commands with no visible cause). Now requires an explicit VINCA_USE_SCCACHE=1 in the build environment; sccache being on PATH alone does nothing. Also hard-fails with a clear message if VINCA_USE_SCCACHE=1 is set but sccache isn't actually findable, rather than silently falling back to no caching (which would be its own kind of confusing "it didn't error, but did it actually use the cache?" situation). Co-Authored-By: Claude Sonnet 5 --- vinca/templates/build_ament_cmake.sh.in | 13 ++++++++++--- vinca/templates/build_catkin.sh.in | 13 ++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/vinca/templates/build_ament_cmake.sh.in b/vinca/templates/build_ament_cmake.sh.in index 4653d6b..0d9e914 100644 --- a/vinca/templates/build_ament_cmake.sh.in +++ b/vinca/templates/build_ament_cmake.sh.in @@ -16,13 +16,20 @@ export LINK=$CXX # Speed up repeated local builds (e.g. rebuilding the same packages across # several Python-version passes) by routing the C/C++ compiler through -# sccache when it's present in the build environment; a no-op otherwise. +# sccache -- opt-in via VINCA_USE_SCCACHE=1, rather than auto-detecting +# sccache on PATH, so this never silently changes compiler invocations +# just because sccache happens to be installed for some unrelated reason. # Requires the top-level `rattler-build build` invocation to pass # --no-build-id, since both ccache and sccache are sensitive to the # timestamped build-directory paths rattler-build uses by default. SCCACHE_CMAKE_ARGS="" -if command -v sccache >/dev/null 2>&1; then - SCCACHE_CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache" +if [[ "${VINCA_USE_SCCACHE:-}" == "1" ]]; then + if command -v sccache >/dev/null 2>&1; then + SCCACHE_CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache" + else + echo "VINCA_USE_SCCACHE=1 but sccache was not found on PATH" >&2 + exit 1 + fi fi if [[ "$CONDA_BUILD_CROSS_COMPILATION" != "1" ]]; then diff --git a/vinca/templates/build_catkin.sh.in b/vinca/templates/build_catkin.sh.in index 422ae9b..f83ccf4 100644 --- a/vinca/templates/build_catkin.sh.in +++ b/vinca/templates/build_catkin.sh.in @@ -21,13 +21,20 @@ export LINK=$CXX # Speed up repeated local builds (e.g. rebuilding the same packages across # several Python-version passes) by routing the C/C++ compiler through -# sccache when it's present in the build environment; a no-op otherwise. +# sccache -- opt-in via VINCA_USE_SCCACHE=1, rather than auto-detecting +# sccache on PATH, so this never silently changes compiler invocations +# just because sccache happens to be installed for some unrelated reason. # Requires the top-level `rattler-build build` invocation to pass # --no-build-id, since both ccache and sccache are sensitive to the # timestamped build-directory paths rattler-build uses by default. SCCACHE_CMAKE_ARGS="" -if command -v sccache >/dev/null 2>&1; then - SCCACHE_CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache" +if [[ "${VINCA_USE_SCCACHE:-}" == "1" ]]; then + if command -v sccache >/dev/null 2>&1; then + SCCACHE_CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache" + else + echo "VINCA_USE_SCCACHE=1 but sccache was not found on PATH" >&2 + exit 1 + fi fi if [[ "$CONDA_BUILD_CROSS_COMPILATION" != "1" ]]; then