diff --git a/news/3514.added.md b/news/3514.added.md new file mode 100644 index 0000000000..8f950b2b16 --- /dev/null +++ b/news/3514.added.md @@ -0,0 +1,2 @@ +(pip,python) Added `pyproject_toml` attribute to `pip.default()` and `python.defaults()` + to read the default Python version from the `requires-python` field of `pyproject.toml`. diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index c3d60ffaf1..9cf0b195a9 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -709,6 +709,7 @@ bzl_library( ":full_version", ":pbs_manifest", ":platform_info", + ":pyproject_utils", ":python_register_toolchains", ":pythons_hub", ":repo_utils", @@ -876,6 +877,15 @@ bzl_library( ], ) +bzl_library( + name = "pyproject_utils", + srcs = ["pyproject_utils.bzl"], + deps = [ + ":version", + "@toml.bzl//:toml", + ], +) + bzl_library( name = "bzlmod_enabled", srcs = ["bzlmod_enabled.bzl"], diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 677154ee46..8dd32afb9c 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -125,6 +125,7 @@ bzl_library( ":whl_library", "//python/private:auth", "//python/private:normalize_name", + "//python/private:pyproject_utils", "//python/private:repo_utils", "@pythons_hub//:interpreters", "@pythons_hub//:versions", diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 02c5ad0c34..f1f6b1bbf2 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -20,6 +20,7 @@ load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config") load("@toml.bzl", "toml") load("//python/private:auth.bzl", "AUTH_ATTRS") load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_requires_python") load("//python/private:repo_utils.bzl", "repo_utils") load(":hub_builder.bzl", "hub_builder") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") @@ -208,6 +209,7 @@ def build_config( default_hub = None defaults = { "platforms": default_platforms(), + "python_version": None, } for mod in module_ctx.modules: if not (mod.is_root or mod.name == "rules_python"): @@ -219,6 +221,11 @@ def build_config( if default_hub: fail("Duplicate pip.default tag: only one explicit default PyPI hub is allowed.") default_hub = tag.default_hub + pyproject_toml = tag.pyproject_toml + if pyproject_toml: + pyproject = read_pyproject(module_ctx, pyproject_toml) + if pyproject.requires_python: + defaults["python_version"] = version_from_requires_python(pyproject.requires_python) platform = tag.platform if platform: @@ -256,6 +263,7 @@ def build_config( default_hub = default_hub, index_url = defaults.get("index_url", "https://pypi.org/simple").rstrip("/"), netrc = defaults.get("netrc", None), + python_version = defaults.get("python_version", None), platforms = { name: _plat(**values) for name, values in defaults["platforms"].items() @@ -359,6 +367,15 @@ You cannot use both the additive_build_content and additive_build_content_file a for mod in module_ctx.modules: for pip_attr in mod.tags.parse: + python_version = pip_attr.python_version + if not python_version and pip_attr.pyproject_toml: + pyproject = read_pyproject(module_ctx, pip_attr.pyproject_toml) + if pyproject.requires_python: + python_version = version_from_requires_python(pyproject.requires_python) + python_version = python_version or config.python_version + if not python_version: + _fail("pip.parse() requires one of `python_version`, `pyproject_toml`, or `pip.default(pyproject_toml=...)` to be set") + hub_name = pip_attr.hub_name if hub_name == "pypi": if is_pypi_hub_reserved: @@ -422,6 +439,7 @@ You cannot use both the additive_build_content and additive_build_content_file a builder.pip_parse( module_ctx, pip_attr = pip_attr, + python_version = python_version, ) # Keeps track of all the hub's whl repos across the different versions. @@ -699,6 +717,23 @@ If you are defining custom platforms in your project and don't want things to cl [isolation] feature. [isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate +""", + ), + "pyproject_toml": attr.label( + mandatory = False, + doc = """\ +Label pointing to pyproject.toml file to read the default Python version from. +When specified, reads the `requires-python` field from pyproject.toml and uses +it as the default python_version for all `pip.parse()` calls that don't +explicitly specify one. + +:::{note} +The version must be specified as `==X.Y.Z` (exact version with full semver). +This is designed to work with dependency management tools like Renovate. +::: + +:::{versionadded} VERSION_NEXT_FEATURE +::: """, ), "whl_abi_tags": attr.string_list( @@ -869,8 +904,23 @@ find in case extra indexes are specified. """, default = True, ), + "pyproject_toml": attr.label( + mandatory = False, + doc = """\ +Label pointing to a pyproject.toml file to read the Python version from. +When specified, the `requires-python` field is used as the `python_version` +for this `pip.parse()` call, unless `python_version` is set explicitly. + +:::{note} +The version must be specified as `==X.Y.Z` (exact version with full semver). +::: + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), "python_version": attr.string( - mandatory = True, + mandatory = False, doc = """ The Python version the dependencies are targetting, in Major.Minor format (e.g., "3.11") or patch level granularity (e.g. "3.11.1"). @@ -878,6 +928,15 @@ The Python version the dependencies are targetting, in Major.Minor format If an interpreter isn't explicitly provided (using `python_interpreter` or `python_interpreter_target`), then the version specified here must have a corresponding `python.toolchain()` configured. + +:::{seealso} +The {obj}`pyproject_toml` attribute for getting the version from a project file. +::: + +:::{versionchanged} VERSION_NEXT_FEATURE +No longer mandatory if the {obj}`pyproject_toml` attribute or +{obj}`pip.default.pyproject_toml` is specified. +::: """, ), "simpleapi_skip": attr.string_list( diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 7717f36731..01dc8494d6 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -144,8 +144,8 @@ def _build(self): whl_libraries = self._whl_libraries, ) -def _pip_parse(self, module_ctx, pip_attr): - python_version = pip_attr.python_version +def _pip_parse(self, module_ctx, pip_attr, python_version = None): + python_version = python_version or pip_attr.python_version if python_version in self._platforms: fail(( "Duplicate pip python version '{version}' for hub " + @@ -191,7 +191,8 @@ def _pip_parse(self, module_ctx, pip_attr): self, module_ctx, pip_attr = pip_attr, - enable_pipstar_extract = bool(self._config.enable_pipstar_extract or self._get_index_urls.get(pip_attr.python_version)), + python_version = python_version, + enable_pipstar_extract = bool(self._config.enable_pipstar_extract or self._get_index_urls.get(python_version)), ) ### end of PUBLIC methods @@ -393,11 +394,11 @@ def _set_get_index_urls(self, mctx, pip_attr): ) return True -def _detect_interpreter(self, pip_attr): +def _detect_interpreter(self, pip_attr, python_version): python_interpreter_target = pip_attr.python_interpreter_target if python_interpreter_target == None and not pip_attr.python_interpreter: python_name = "python_{}_host".format( - pip_attr.python_version.replace(".", "_"), + python_version.replace(".", "_"), ) if python_name not in self._available_interpreters: fail(( @@ -407,7 +408,7 @@ def _detect_interpreter(self, pip_attr): "Expected to find {python_name} among registered versions:\n {labels}" ).format( hub_name = self.name, - version = pip_attr.python_version, + version = python_version, python_name = python_name, labels = " \n".join(self._available_interpreters), )) @@ -476,6 +477,7 @@ def _create_whl_repos( module_ctx, *, pip_attr, + python_version, enable_pipstar_extract = False): """create all of the whl repositories @@ -483,10 +485,11 @@ def _create_whl_repos( self: the builder. module_ctx: {type}`module_ctx`. pip_attr: {type}`struct` - the struct that comes from the tag class iteration. + python_version: {type}`str` - the resolved python version for this pip.parse call. enable_pipstar_extract: {type}`bool` - enable the pipstar extraction or not. """ logger = self._logger - platforms = self._platforms[pip_attr.python_version] + platforms = self._platforms[python_version] requirements_by_platform = parse_requirements( module_ctx, requirements_by_platform = requirements_files_by_platform( @@ -498,7 +501,7 @@ def _create_whl_repos( extra_pip_args = pip_attr.extra_pip_args, platforms = sorted(platforms), # here we only need keys python_version = full_version( - version = pip_attr.python_version, + version = python_version, minor_mapping = self._minor_mapping, ), logger = logger, @@ -528,7 +531,7 @@ def _create_whl_repos( pip_attr = pip_attr, ) - interpreter = _detect_interpreter(self, pip_attr) + interpreter = _detect_interpreter(self, pip_attr, python_version) for whl in requirements_by_platform: whl_library_args = common_args | _whl_library_args( @@ -543,16 +546,16 @@ def _create_whl_repos( whl_library_args = whl_library_args, download_only = pip_attr.download_only, netrc = self._config.netrc or pip_attr.netrc, - use_downloader = src.url and _use_downloader(self, pip_attr.python_version, whl.name), + use_downloader = src.url and _use_downloader(self, python_version, whl.name), auth_patterns = self._config.auth_patterns or pip_attr.auth_patterns, - python_version = _major_minor_version(pip_attr.python_version), + python_version = _major_minor_version(python_version), is_multiple_versions = whl.is_multiple_versions, interpreter = interpreter, enable_pipstar_extract = enable_pipstar_extract, ) _add_whl_library( self, - python_version = pip_attr.python_version, + python_version = python_version, whl = whl, repo = repo, ) diff --git a/python/private/pyproject_utils.bzl b/python/private/pyproject_utils.bzl new file mode 100644 index 0000000000..e96ffd2023 --- /dev/null +++ b/python/private/pyproject_utils.bzl @@ -0,0 +1,51 @@ +"""Utilities for reading values from pyproject.toml.""" + +load("@toml.bzl", "toml") +load(":version.bzl", "version") + +def read_pyproject(module_ctx, pyproject): + """Read a pyproject.toml file and return the relevant fields. + + The file is parsed with a pure-Starlark TOML decoder; no Python + interpreter is required. The raw `requires-python` value is returned + as-is so that callers can decide how to interpret it. + + Args: + module_ctx: the module extension context (needs `path` and `read`). + pyproject: {type}`Label` pointing at the pyproject.toml file. + + Returns: + {type}`struct` with the attributes: + * `requires_python`: {type}`str | None` the raw `requires-python` + value (e.g. `"==3.13.9"`), or `None` if it is not set. + """ + data = toml.decode(module_ctx.read(module_ctx.path(pyproject), watch = "yes")) + return struct( + requires_python = data.get("project", {}).get("requires-python"), + ) + +def version_from_requires_python(requires_python): + """Derive a concrete Python version from a `requires-python` value. + + Currently only an exact `==X.Y.Z` specifier is supported. The value is + validated and normalized via {obj}`//python/private:version.bzl` so that + malformed input fails in a consistent way. Broader specifier support + (e.g. `>=`, `X.Y`) can be layered on here in the future. + + Args: + requires_python: {type}`str` the raw `requires-python` value. + + Returns: + {type}`str` the normalized version string (e.g. `"3.13.9"`). + """ + if not requires_python: + fail("`requires-python` must be specified") + + if not requires_python.startswith("=="): + fail("`requires-python` must pin an exact version with `==`, got: {}".format(requires_python)) + + bare_version = requires_python[len("=="):].strip() + + # Parse strictly so malformed versions fail cleanly, then normalize. + version.parse(bare_version, strict = True) + return version.normalize(bare_version) diff --git a/python/private/python.bzl b/python/private/python.bzl index 23bae5d341..f098a626fb 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -20,6 +20,7 @@ load(":auth.bzl", "AUTH_ATTRS") load(":full_version.bzl", "full_version") load(":pbs_manifest.bzl", "parse_runtime_manifest") load(":platform_info.bzl", "platform_info") +load(":pyproject_utils.bzl", "read_pyproject", "version_from_requires_python") load(":python_register_toolchains.bzl", "python_register_toolchains") load(":pythons_hub.bzl", "hub_repo") load(":repo_utils.bzl", "repo_utils") @@ -88,6 +89,7 @@ def parse_modules(*, module_ctx, logger = None, _fail = fail): mod = mod, seen_versions = seen_versions, config = config, + default_python_version = default_python_version, ) for toolchain_attr in toolchain_attr_structs: @@ -971,8 +973,15 @@ def _compute_default_python_version(mctx): defaults_attr_structs = _create_defaults_attr_structs(mod = mod) default_python_version_env = None default_python_version_file = None + pyproject_toml_label = None for defaults_attr in defaults_attr_structs: + pyproject_toml_label = _one_or_the_same( + pyproject_toml_label, + defaults_attr.pyproject_toml, + onerror = lambda: fail("Multiple pyproject.toml files specified in defaults"), + ) + default_python_version = _one_or_the_same( default_python_version, defaults_attr.python_version, @@ -988,11 +997,17 @@ def _compute_default_python_version(mctx): defaults_attr.python_version_file, onerror = _fail_multiple_defaults_python_version_file, ) + + # Priority order: ENV > pyproject_toml > python_version_file > python_version if default_python_version_file: default_python_version = _one_or_the_same( default_python_version, mctx.read(default_python_version_file, watch = "yes").strip(), ) + if pyproject_toml_label: + pyproject = read_pyproject(mctx, pyproject_toml_label) + if pyproject.requires_python: + default_python_version = version_from_requires_python(pyproject.requires_python) if default_python_version_env: default_python_version = mctx.getenv( default_python_version_env, @@ -1034,11 +1049,29 @@ def _create_defaults_attr_struct(*, tag): python_version = getattr(tag, "python_version", None), python_version_env = getattr(tag, "python_version_env", None), python_version_file = getattr(tag, "python_version_file", None), + pyproject_toml = getattr(tag, "pyproject_toml", None), ) -def _create_toolchain_attr_structs(*, mod, config, seen_versions): +def _create_toolchain_attr_structs(*, mod, config, seen_versions, default_python_version): arg_structs = [] + # Auto-register a toolchain for the default version if not already + # registered via an explicit python.toolchain() call. + # This works for any default source: pyproject_toml, python_version_file, + # python_version_env, or python_version. + has_explicit_toolchain = default_python_version and any([ + tag.python_version == default_python_version + for tag in mod.tags.toolchain + ]) + if (default_python_version and + default_python_version not in seen_versions and + mod.is_root and not has_explicit_toolchain): + arg_structs.append(_create_toolchain_attrs_struct( + python_version = default_python_version, + toolchain_tag_count = 1, + )) + seen_versions[default_python_version] = True + for tag in mod.tags.toolchain: arg_structs.append(_create_toolchain_attrs_struct( tag = tag, @@ -1078,6 +1111,17 @@ def _create_toolchain_attrs_struct( _defaults = tag_class( doc = """Tag class to specify the default Python version.""", attrs = { + "pyproject_toml": attr.label( + mandatory = False, + doc = """\ +Label pointing to pyproject.toml file to read the default Python version from. +When specified, reads the `requires-python` field from pyproject.toml. +The version must be specified as `==X.Y.Z` (exact version with full semver). + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), "python_version": attr.string( mandatory = False, doc = """\ diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index bc4c0bcb5b..c708939ba3 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -24,7 +24,7 @@ load(":pip_parse.bzl", _parse = "pip_parse") _tests = [] -def _pypi_mock_mctx(*modules, os_name = "unittest", arch_name = "exotic", environ = {}, read = None): +def _pypi_mock_mctx(*modules, os_name = "unittest", arch_name = "exotic", environ = {}, read = None, mock_files = {}): _ = read # @unused return mocks.mctx( modules = list(modules), @@ -36,7 +36,7 @@ def _pypi_mock_mctx(*modules, os_name = "unittest", arch_name = "exotic", enviro simple==0.0.1 \ --hash=sha256:deadbeef \ --hash=sha256:deadbaaf""", - }, + } | mock_files, ) def _default( @@ -51,6 +51,7 @@ def _default( netrc = None, os_name = None, platform = None, + pyproject_toml = None, whl_platform_tags = None, whl_abi_tags = None): return struct( @@ -64,6 +65,7 @@ def _default( netrc = netrc, os_name = os_name, platform = platform, + pyproject_toml = pyproject_toml, whl_abi_tags = whl_abi_tags or [], whl_platform_tags = whl_platform_tags or [], ) @@ -175,6 +177,58 @@ def _test_simple(env): _tests.append(_test_simple) +def _test_pip_parse_pyproject_toml(env): + # pip.parse() reads the version from pyproject.toml's requires-python when + # python_version is not set explicitly. + pypi = _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + _mod( + name = "rules_python", + parse = [ + _parse( + hub_name = "pypi", + pyproject_toml = "pyproject.toml", + simpleapi_skip = ["simple"], + requirements_lock = "requirements.txt", + ), + ], + ), + os_name = "linux", + arch_name = "x86_64", + mock_files = { + "pyproject.toml": "[project]\nrequires-python = \"==3.15.19\"\n", + }, + ), + available_interpreters = { + "python_3_15_19_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + + # Resolves identically to passing python_version = "3.15.19" explicitly: + # the full version drives interpreter selection, hub naming uses major.minor. + pypi.exposed_packages().contains_exactly({"pypi": ["simple"]}) + pypi.hub_whl_map().contains_exactly({"pypi": { + "simple": { + "pypi_315_simple": [ + whl_config_setting( + version = "3.15", + ), + ], + }, + }}) + pypi.whl_libraries().contains_exactly({ + "pypi_315_simple": { + "config_load": "@pypi//:config.bzl", + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadbaaf", + }, + }) + +_tests.append(_test_pip_parse_pyproject_toml) + def _test_simple_isolated(env): """Simulate `isolate = True` with parse_modules. diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index 939639f2c5..7b5bdfdfd6 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -3,7 +3,7 @@ def pip_parse( *, hub_name, - python_version, + python_version = None, add_libdir_to_library_search_path = False, auth_patterns = {}, download_only = False, @@ -19,6 +19,7 @@ def pip_parse( netrc = None, parse_all_requirements_files = True, pip_data_exclude = None, + pyproject_toml = None, python_interpreter = None, python_interpreter_target = None, quiet = True, @@ -52,6 +53,7 @@ def pip_parse( netrc = netrc, parse_all_requirements_files = parse_all_requirements_files, pip_data_exclude = pip_data_exclude, + pyproject_toml = pyproject_toml, python_interpreter = python_interpreter, python_interpreter_target = python_interpreter_target, python_version = python_version,