diff --git a/.bazelrc.deleted_packages b/.bazelrc.deleted_packages index d0c8d405e4..456ea639df 100644 --- a/.bazelrc.deleted_packages +++ b/.bazelrc.deleted_packages @@ -40,6 +40,9 @@ common --deleted_packages=tests/integration/pip_parse/empty common --deleted_packages=tests/integration/pip_parse_isolated common --deleted_packages=tests/integration/prebuilt_toolchain common --deleted_packages=tests/integration/py_cc_toolchain_registered +common --deleted_packages=tests/integration/python_version_scope +common --deleted_packages=tests/integration/python_version_scope/bindings +common --deleted_packages=tests/integration/python_version_scope/native common --deleted_packages=tests/integration/runtime_manifests common --deleted_packages=tests/integration/toolchain_target_settings common --deleted_packages=tests/integration/unified_pypi diff --git a/docs/index.md b/docs/index.md index 0204f38ec7..c1d30478d4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -97,6 +97,7 @@ self getting-started pypi/index Toolchains +Python version project scoping coverage precompiling gazelle/docs/index diff --git a/docs/python-version-project-scope.md b/docs/python-version-project-scope.md new file mode 100644 index 0000000000..c329224575 --- /dev/null +++ b/docs/python-version-project-scope.md @@ -0,0 +1,87 @@ +# Python version project scoping + +A Python binding and its build tools need the selected interpreter version; a +C++ or Rust library underneath the binding usually does not. Bazel's project +scope lets the root module name the packages that own the +`//python/config_settings:python_version` flag. Targets outside them see the +version given on the command line, so binaries transitioned to different Python +versions share one configuration of their Python-independent native +dependencies. + +Project scope is off by default, and only the root module can enable it. Tags in +dependency modules are ignored, so a library can use it in its own builds +without imposing a policy on its consumers. + +```starlark +# MODULE.bazel +config = use_extension("@rules_python//python/extensions:config.bzl", "config") +config.python_version_scope(scope = "project") +``` + +`scope` accepts: + +* `"universal"` (the default): the version reaches every target, including exec + tools such as the precompiler. +* `"project"`: the root module's `//:PROJECT.scl` decides which targets keep the + version. + +Project scope requires Bazel 9.2 or later. Older releases drop project-scoped +flags on exec transitions, so even owned exec tools would lose the version +([bazelbuild/bazel#28320](https://github.com/bazelbuild/bazel/issues/28320)). +The extension fails on older versions. + +## Ownership + +Under project scope, `//python/config_settings:python_version` is an alias to +a flag in rules_python's generated configuration repository. That repository's +`PROJECT.scl` forwards to the root module's `//:PROJECT.scl` with Bazel's +`actual` field, which may itself redirect with `actual`. Without the tag nothing +forwards there, so other builds don't need a root `PROJECT.scl`. List every +package that must see the selected version in `active_directories`: + +```starlark +# PROJECT.scl +project = { + "active_directories": { + "default": [ + "//bindings", + "@@rules_python+", + ], + }, +} +``` + +Repositories are listed by canonical name. `bazel mod dump_repo_mapping ""` +prints the canonical names visible to the root module. + +The example owns only `//bindings` and rules_python itself. A real project must +also own every other Python-sensitive target: interpreter and header +repositories, toolchains, version-sensitive generators, and any alias or +wrapper in between. Every dependency path from a transitioned target must stay +inside the owned packages until it reaches Python-independent code. Owned exec +tools keep the selected version. Bazel does not infer ownership from +dependencies and cannot detect an omitted consumer: such a target sees the +command-line version and can select the wrong interpreter. + +If you also use aspect_rules_py, its terminals set this flag as well as their +own `@aspect_rules_py//py:python_version`. Scope both flags and own both +rulesets' repositories, or native dependencies still get one configuration per +Python version. + +## Command-line baseline + +Pass a fixed `--@rules_python//python/config_settings:python_version=` +and let binaries select their versions with transitions, for example through +`python_version` attributes. Outside the owned packages, Bazel restores that +command-line value, not the flag's declared default. Invocations with different +command-line versions therefore still build native dependencies in different +configurations. + +## Limitations + +Only the Python version flag is scoped. ABI, platform, free-threading, +build-script environment, and other settings are unaffected. + +See Bazel's +[project-scoped flags example](https://github.com/gregestren/snippets/tree/master/project_scoped_flags) +and the [configurability roadmap](https://bazel.build/about/roadmap#configurability). diff --git a/news/4375.added.md b/news/4375.added.md new file mode 100644 index 0000000000..2fd6110162 --- /dev/null +++ b/news/4375.added.md @@ -0,0 +1 @@ +The `config.python_version_scope(scope = "project")` extension tag lets the root module opt into Python version project scoping using its `//:PROJECT.scl`. The default remains universal. diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index f519a38c23..a79a1fbcb7 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -44,6 +44,7 @@ construct_config_settings( ":py_pbs_distribution", ], minor_mapping = MINOR_MAPPING, + python_version_scope = config.python_version_scope, versions = PYTHON_VERSIONS, ) diff --git a/python/extensions/config.bzl b/python/extensions/config.bzl index 63d0a6eefb..c2807eb3ce 100644 --- a/python/extensions/config.bzl +++ b/python/extensions/config.bzl @@ -68,12 +68,34 @@ Internal: value of TOOL_DEPS_DEV_ONLY from rules_python's MODULE.bazel. }, ) +_python_version_scope = tag_class( + doc = """Opt into project scoping of the Python version flag (root module only). + +With `scope = "project"`, the root module's `//:PROJECT.scl` owns the scope. +Targets outside its active directories reset to the invocation's Python version, +allowing native dependencies to be shared between version-transitioned binaries. +Tags from dependency modules are ignored. The default remains `"universal"`. See {doc}`/python-version-project-scope`. +""", + attrs = { + "scope": attr.string(mandatory = True, values = ["universal", "project"]), + }, +) + def _config_impl(module_ctx): transition_setting_generators = {} transition_settings = [] explicit_init_py_modules = {} tool_deps_dev_only = True + python_version_scope = "universal" + scope_configured = False for mod in module_ctx.modules: + for tag in mod.tags.python_version_scope: + if not mod.is_root: + continue + if scope_configured: + fail("python_version_scope may only be configured once") + scope_configured = True + python_version_scope = tag.scope if mod.name == "rules_python": for tag in mod.tags.internal_configure: tool_deps_dev_only = tag.tool_deps_dev_only @@ -95,6 +117,7 @@ def _config_impl(module_ctx): transition_settings = transition_settings, explicit_init_py_modules = explicit_init_py_modules, tool_deps_dev_only = tool_deps_dev_only, + python_version_scope = python_version_scope, ) pypi_deps() @@ -115,5 +138,6 @@ config = module_extension( "add_transition_setting": _add_transition_setting, "explicit_init_py": _explicit_init_py, "internal_configure": _internal_configure, + "python_version_scope": _python_version_scope, }, ) diff --git a/python/private/config_settings.bzl b/python/private/config_settings.bzl index 8711f204eb..6908883885 100644 --- a/python/private/config_settings.bzl +++ b/python/private/config_settings.bzl @@ -40,7 +40,8 @@ def construct_config_settings( versions, minor_mapping, compat_lowest_version = "3.8", - documented_flags): # buildifier: disable=function-docstring + documented_flags, + python_version_scope = "universal"): # buildifier: disable=function-docstring """Create a 'python_version' config flag and construct all config settings used in rules_python. This mainly includes the targets that are used in the toolchain and pip hub @@ -55,21 +56,28 @@ def construct_config_settings( version for `is_python_3.X` flags. documented_flags: {type}`list[str]` The labels of the documented settings that affect build configuration. + python_version_scope: Scope of the Python version flag. Defaults to universal. """ _ = name # @unused - _python_version_flag( - name = _PYTHON_VERSION_FLAG.name, - build_setting_default = default_version, - # Propagate the flag into exec configurations so exec tools (e.g. the - # precompiler) use the target's Python version. Bazel 10 resets - # Starlark flags in exec configurations unless they have universal - # scope (https://github.com/bazelbuild/bazel/issues/26909). It's set - # here, not as an attr default, because Bazel ignores scope attr - # defaults for flags set on the command line - # (https://github.com/bazelbuild/bazel/issues/31806). - scope = "universal", - visibility = ["//visibility:public"], - ) + if python_version_scope == "project": + # Bazel finds a scoped flag's project from the flag's package, so the + # project-scoped flag lives in the generated repository, which ships + # the PROJECT.scl that forwards to the root module's definition. + native.alias( + name = _PYTHON_VERSION_FLAG.name, + actual = "@rules_python_internal//:python_version", + visibility = ["//visibility:public"], + ) + else: + python_version_flag( + name = _PYTHON_VERSION_FLAG.name, + build_setting_default = default_version, + # Universal scope preserves exec-tool version selection. Set scope + # here: Bazel ignores attr defaults for command-line flags + # (bazel#31806). + scope = python_version_scope, + visibility = ["//visibility:public"], + ) _python_version_major_minor_flag( name = _PYTHON_VERSION_MAJOR_MINOR_FLAG.name, @@ -211,7 +219,7 @@ def _python_version_flag_impl(ctx): config_common.FeatureFlagInfo(value = value), ] -_python_version_flag = rule( +python_version_flag = rule( implementation = _python_version_flag_impl, build_setting = config.string(flag = True), attrs = { diff --git a/python/private/internal_config_repo.bzl b/python/private/internal_config_repo.bzl index 8f203bb89e..558e4878d3 100644 --- a/python/private/internal_config_repo.bzl +++ b/python/private/internal_config_repo.bzl @@ -29,6 +29,7 @@ config = struct( build_python_zip_default = {build_python_zip_default}, supports_whl_extraction = {supports_whl_extraction}, enable_pystar = True, + python_version_scope = {python_version_scope}, enable_deprecation_warnings = {enable_deprecation_warnings}, extract_needs_chmod = {extract_needs_chmod}, bazel_8_or_later = {bazel_8_or_later}, @@ -71,6 +72,30 @@ _EXTRA_TRANSITIONS_TEMPLATE = """ EXTRA_TRANSITION_SETTINGS = {labels} """ +# Under project scope, //python/config_settings:python_version aliases this +# flag. Bazel looks for its project from this package, so only this mode ships +# a PROJECT.scl and other builds don't need one in the root module. +_PYTHON_VERSION_FLAG_LOADS = """ +load("@pythons_hub//:versions.bzl", "DEFAULT_PYTHON_VERSION") +load("@rules_python//python/private:config_settings.bzl", "python_version_flag") # buildifier: disable=bzl-visibility +""" + +_PYTHON_VERSION_FLAG_TARGET = """ +python_version_flag( + name = "python_version", + build_setting_default = DEFAULT_PYTHON_VERSION, + scope = "project", + visibility = ["//visibility:public"], +) +""" + +_PYTHON_VERSION_PROJECT = """\ +# Generated by @rules_python//python/private:internal_config_repo.bzl +# Bazel's project alias: the root module's PROJECT.scl owns the version flag. + +project = {"actual": "//:PROJECT.scl"} +""" + _TRANSITION_SETTINGS_DEBUG_TEMPLATE = """ # Generated by @rules_python//python/private:internal_config_repo.bzl @@ -88,6 +113,9 @@ def _internal_config_repo_impl(rctx): bazel_major_version = 99999 bazel_minor_version = 99999 + if rctx.attr.python_version_scope == "project" and (bazel_major_version, bazel_minor_version) < (9, 2): + fail("Python version project scope requires Bazel 9.2 or later") + supports_whl_extraction = False extract_needs_chmod = bazel_major_version < 8 or (bazel_major_version == 8 and bazel_minor_version < 6) if bazel_major_version >= 8: @@ -105,6 +133,7 @@ def _internal_config_repo_impl(rctx): explicit_init_py_modules = {k: str(v) == "True" for k, v in rctx.attr.explicit_init_py_modules.items()} rctx.file("rules_python_config.bzl", _CONFIG_TEMPLATE.format( + python_version_scope = repr(rctx.attr.python_version_scope), build_python_zip_default = repo_utils.get_platforms_os_name(rctx) == "windows", enable_deprecation_warnings = _bool_from_environ(rctx, _ENABLE_DEPRECATION_WARNINGS_ENVVAR_NAME, _ENABLE_DEPRECATION_WARNINGS_DEFAULT), builtin_py_info_symbol = builtin_py_info_symbol, @@ -119,9 +148,13 @@ def _internal_config_repo_impl(rctx): bazel_10_or_later = str(bazel_major_version > 9), )) - rctx.file("BUILD", ROOT_BUILD_TEMPLATE.format( + build_content = ROOT_BUILD_TEMPLATE.format( visibility = "@rules_python//:__subpackages__", - )) + ) + if rctx.attr.python_version_scope == "project": + build_content = _PYTHON_VERSION_FLAG_LOADS + build_content + _PYTHON_VERSION_FLAG_TARGET + rctx.file("PROJECT.scl", _PYTHON_VERSION_PROJECT) + rctx.file("BUILD", build_content) rctx.file( "extra_transition_settings.bzl", @@ -146,6 +179,7 @@ internal_config_repo = repository_rule( environ = [], attrs = { "explicit_init_py_modules": attr.string_dict(), + "python_version_scope": attr.string(default = "universal", values = ["universal", "project"]), "tool_deps_dev_only": attr.bool(default = True), "transition_setting_generators": attr.string_list_dict(), "transition_settings": attr.string_list(), diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index 9e909dc45b..8e16bf59cd 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -198,3 +198,9 @@ py_binary( srcs = ["uv_lock_pypi_server.py"], deps = [":uv_lock_pypi_server_lib"], ) + +rules_python_integration_test( + name = "python_version_scope_test", + bazel_versions = ["10.0.0rc2"], + py_main = "python_version_scope_test.py", +) diff --git a/tests/integration/python_version_scope/BUILD.bazel b/tests/integration/python_version_scope/BUILD.bazel new file mode 100644 index 0000000000..f77e32aa93 --- /dev/null +++ b/tests/integration/python_version_scope/BUILD.bazel @@ -0,0 +1,4 @@ +exports_files([ + "PROJECT.scl", + "probe.bzl", +]) diff --git a/tests/integration/python_version_scope/MODULE.bazel b/tests/integration/python_version_scope/MODULE.bazel new file mode 100644 index 0000000000..8ca4d6c765 --- /dev/null +++ b/tests/integration/python_version_scope/MODULE.bazel @@ -0,0 +1,10 @@ +module(name = "python_version_scope_test") + +bazel_dep(name = "rules_python", version = "0.0.0") +local_path_override( + module_name = "rules_python", + path = "../../..", +) + +config = use_extension("@rules_python//python/extensions:config.bzl", "config") +config.python_version_scope(scope = "project") diff --git a/tests/integration/python_version_scope/PROJECT.scl b/tests/integration/python_version_scope/PROJECT.scl new file mode 100644 index 0000000000..6678eb557c --- /dev/null +++ b/tests/integration/python_version_scope/PROJECT.scl @@ -0,0 +1,3 @@ +"""Own bindings and the flag definition; leave the native core outside.""" + +project = {"active_directories": {"default": ["//bindings", "@@rules_python+"]}} diff --git a/tests/integration/python_version_scope/WORKSPACE b/tests/integration/python_version_scope/WORKSPACE new file mode 100644 index 0000000000..d483adc318 --- /dev/null +++ b/tests/integration/python_version_scope/WORKSPACE @@ -0,0 +1 @@ +# Workspace marker required by the Bazel integration test harness. diff --git a/tests/integration/python_version_scope/bindings/BUILD.bazel b/tests/integration/python_version_scope/bindings/BUILD.bazel new file mode 100644 index 0000000000..382b288573 --- /dev/null +++ b/tests/integration/python_version_scope/bindings/BUILD.bazel @@ -0,0 +1,52 @@ +load("//:probe.bzl", "check", "probe") + +config_setting( + name = "py311", + flag_values = {"@rules_python//python/config_settings:python_version": "3.11"}, +) + +config_setting( + name = "py312", + flag_values = {"@rules_python//python/config_settings:python_version": "3.12"}, +) + +probe( + name = "tool", + value = select({ + ":py310": "3.10", + ":py311": "3.11", + ":py312": "3.12", + "//conditions:default": "unexpected", + }), +) + +probe( + name = "binding", + tool = ":tool", + value = select({ + ":py310": "3.10", + ":py311": "3.11", + ":py312": "3.12", + "//conditions:default": "unexpected", + }), + deps = ["//native:core"], +) + +check( + name = "check311", + native_version = "3.10", + target = ":binding", + version = "3.11", +) + +check( + name = "check312", + native_version = "3.10", + target = ":binding", + version = "3.12", +) + +config_setting( + name = "py310", + flag_values = {"@rules_python//python/config_settings:python_version": "3.10"}, +) diff --git a/tests/integration/python_version_scope/native/BUILD.bazel b/tests/integration/python_version_scope/native/BUILD.bazel new file mode 100644 index 0000000000..a89311e28c --- /dev/null +++ b/tests/integration/python_version_scope/native/BUILD.bazel @@ -0,0 +1,27 @@ +load("//:probe.bzl", "probe") + +config_setting( + name = "py311", + flag_values = {"@rules_python//python/config_settings:python_version": "3.11"}, +) + +config_setting( + name = "py312", + flag_values = {"@rules_python//python/config_settings:python_version": "3.12"}, +) + +probe( + name = "core", + value = select({ + ":py310": "3.10", + ":py311": "3.11", + ":py312": "3.12", + "//conditions:default": "unexpected", + }), + visibility = ["//visibility:public"], +) + +config_setting( + name = "py310", + flag_values = {"@rules_python//python/config_settings:python_version": "3.10"}, +) diff --git a/tests/integration/python_version_scope/probe.bzl b/tests/integration/python_version_scope/probe.bzl new file mode 100644 index 0000000000..697072b91b --- /dev/null +++ b/tests/integration/python_version_scope/probe.bzl @@ -0,0 +1,35 @@ +"""Analysis assertions for target and exec configuration propagation.""" + +VersionInfo = provider("Observed version propagation through the fixture graph.", fields = ["value", "native", "tool"]) + +def _probe_impl(ctx): + return [VersionInfo( + value = ctx.attr.value, + native = [dep[VersionInfo].value for dep in ctx.attr.deps], + tool = ctx.attr.tool[VersionInfo].value if ctx.attr.tool else None, + )] + +probe = rule(implementation = _probe_impl, attrs = { + "deps": attr.label_list(), + "tool": attr.label(cfg = "exec"), + "value": attr.string(), +}) + +def _transition_impl(_settings, attr): + return {"@rules_python//python/config_settings:python_version": attr.version} + +_version_transition = transition(implementation = _transition_impl, inputs = [], outputs = ["@rules_python//python/config_settings:python_version"]) + +def _check_impl(ctx): + info = ctx.attr.target[0][VersionInfo] + expected_native = ctx.attr.version if ctx.attr.native_version == "inherit" else ctx.attr.native_version + if info.value != ctx.attr.version or info.native != [expected_native] or info.tool != ctx.attr.version: + fail("Expected binding/tool %s, native %s; got %s" % (ctx.attr.version, expected_native, info)) + return [DefaultInfo()] + +check = rule(implementation = _check_impl, attrs = { + "native_version": attr.string(default = "inherit"), + "target": attr.label(cfg = _version_transition), + "version": attr.string(), + "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"), +}) diff --git a/tests/integration/python_version_scope_test.py b/tests/integration/python_version_scope_test.py new file mode 100644 index 0000000000..bc89552249 --- /dev/null +++ b/tests/integration/python_version_scope_test.py @@ -0,0 +1,110 @@ +"""Test the consumer opt-in, including baseline reset and exec propagation.""" + +import unittest + +from tests.integration import runner + + +class PythonVersionScopeTest(runner.TestCase): + def test_project_scope(self): + self.run_bazel( + "build", + "--@rules_python//python/config_settings:python_version=3.10", + "//bindings:check311", + "//bindings:check312", + ) + + result = self.run_bazel( + "cquery", + "--@rules_python//python/config_settings:python_version=3.10", + 'filter("native:core", deps(set(//bindings:check311 //bindings:check312)))', + "--output=label", + ) + self.assertEqual(len(result.stdout.strip().splitlines()), 1) + + def test_default_scope(self): + module = self.repo_root / "MODULE.bazel" + bindings = self.repo_root / "bindings/BUILD.bazel" + project = self.repo_root / "PROJECT.scl" + original_project = project.read_text() + original_module = module.read_text() + original_bindings = bindings.read_text() + try: + project.unlink() + module.write_text( + original_module.replace( + 'config.python_version_scope(scope = "project")', "" + ) + ) + bindings.write_text( + original_bindings.replace( + 'native_version = "3.10"', 'native_version = "inherit"' + ) + ) + # Building the flag's own package must not need a root PROJECT.scl. + self.run_bazel( + "build", + "--@rules_python//python/config_settings:python_version=3.10", + "//bindings:check311", + "//bindings:check312", + "@rules_python//python/config_settings:python_version", + ) + finally: + module.write_text(original_module) + bindings.write_text(original_bindings) + project.write_text(original_project) + + def test_dependency_cannot_configure_scope(self): + module = self.repo_root / "MODULE.bazel" + bindings = self.repo_root / "bindings/BUILD.bazel" + original = module.read_text() + original_bindings = bindings.read_text() + dependency = self.repo_root / "dependency" + dependency.mkdir() + (dependency / "MODULE.bazel").write_text( + 'module(name = "dependency")\n' + 'bazel_dep(name = "rules_python", version = "0.0.0")\n' + 'config = use_extension("@rules_python//python/extensions:config.bzl", "config")\n' + 'config.python_version_scope(scope = "project")\n' + ) + try: + module.write_text( + original.replace('config.python_version_scope(scope = "project")', "") + + '\nbazel_dep(name = "dependency")\n' + 'local_path_override(module_name = "dependency", path = "dependency")\n' + ) + bindings.write_text( + original_bindings.replace( + 'native_version = "3.10"', 'native_version = "inherit"' + ) + ) + self.run_bazel( + "build", + "--@rules_python//python/config_settings:python_version=3.10", + "//bindings:check311", + "//bindings:check312", + ) + finally: + module.write_text(original) + bindings.write_text(original_bindings) + (dependency / "MODULE.bazel").unlink() + dependency.rmdir() + + def test_duplicate_configuration(self): + module = self.repo_root / "MODULE.bazel" + original = module.read_text() + try: + module.write_text( + original + '\nconfig.python_version_scope(scope = "project")\n' + ) + result = self.run_bazel("build", "//bindings:check311", check=False) + self.assertNotEqual(result.exit_code, 0) + self.assert_result_matches( + result, "python_version_scope may only be configured once" + ) + finally: + module.write_text(original) + + +if __name__ == "__main__": + unittest.main()