diff --git a/docs/pypi/download-workspace.md b/docs/pypi/download-workspace.md index 4912f71103..10e86d252e 100644 --- a/docs/pypi/download-workspace.md +++ b/docs/pypi/download-workspace.md @@ -5,14 +5,16 @@ This documentation page covers how to download PyPI dependencies in the legacy `WORKSPACE` setup. -To add pip dependencies to your `WORKSPACE`, load the `pip_parse` function and -call it to create the central external repo and individual wheel external repos. +To add pip dependencies to your `WORKSPACE`, load the {obj}`pip_parse` or +{obj}`uv_lock` function and call it to create the central external repo and +individual wheel external repos. ```starlark load("@rules_python//python:pip.bzl", "pip_parse") # Create a central repo that knows about the dependencies needed from -# requirements_lock.txt. +# requirements_lock.txt (or use uv_lock(name = "my_deps", uv_lock = "//path/to:uv.lock") +# for a uv.lock file). pip_parse( name = "my_deps", requirements_lock = "//path/to:requirements_lock.txt", diff --git a/docs/pypi/lock.md b/docs/pypi/lock.md index 727795c1ef..86ee6d4133 100644 --- a/docs/pypi/lock.md +++ b/docs/pypi/lock.md @@ -4,8 +4,7 @@ # Lock :::{note} -`rules_python` supports `requirements.txt` and `uv.lock` (bzlmod only) -formats. +`rules_python` supports `requirements.txt` and `uv.lock` formats. #{gh-issue}`2787` tracks `pylock.toml` support. ::: @@ -231,7 +230,7 @@ lock( `diff_test` from `bazel_skylib`. ::: -## uv.lock (bzlmod only) +## uv.lock The {obj}`lock` rule also supports generating cross-platform `uv.lock` files using `uv lock` when {arg}`lock.out` ends with `.lock` (such as `uv.lock`). @@ -251,7 +250,8 @@ lock( ``` Run `bazel run //:uv_lock.update` to generate or update `uv.lock`, then pass it -to {obj}`pip.parse` in `MODULE.bazel` via {attr}`pip.parse.uv_lock`: +to {obj}`pip.parse` in `MODULE.bazel` via {attr}`pip.parse.uv_lock` (or to +{obj}`uv_lock` in `WORKSPACE`): ```starlark pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") diff --git a/news/uv-lock-workspace.added.md b/news/uv-lock-workspace.added.md new file mode 100644 index 0000000000..ceb9594e6c --- /dev/null +++ b/news/uv-lock-workspace.added.md @@ -0,0 +1,2 @@ +(pypi) Added the {obj}`uv_lock` `WORKSPACE` macro in `//python:pip.bzl` to +install PyPI dependencies from a resolved `uv.lock` file. diff --git a/python/BUILD.bazel b/python/BUILD.bazel index e0e32c5d72..a473df123f 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -229,6 +229,7 @@ bzl_library( "//python/private/pypi:package_annotation", "//python/private/pypi:pip_compile", "//python/private/pypi:pip_repository", + "//python/private/pypi:uv_lock", "//python/private/pypi:whl_library_alias", "//python/private/whl_filegroup", ], diff --git a/python/pip.bzl b/python/pip.bzl index 44ee69d65b..6d260ce24f 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -14,9 +14,9 @@ """Rules for pip integration. This contains a set of rules that are used to support inclusion of third-party -dependencies via fully locked `requirements.txt` files. Some of the exported -symbols should not be used and they are either undocumented here or marked as -for internal use only. +dependencies via fully locked `requirements.txt` or `uv.lock` files. Some of the +exported symbols should not be used and they are either undocumented here or +marked as for internal use only. If you are using a bazel version 7 or above with `bzlmod`, you should only care about the {bzl:obj}`compile_pip_requirements` macro exposed in this file. The @@ -28,12 +28,14 @@ load("//python/private/pypi:multi_pip_parse.bzl", _multi_pip_parse = "multi_pip_ load("//python/private/pypi:package_annotation.bzl", _package_annotation = "package_annotation") load("//python/private/pypi:pip_compile.bzl", "pip_compile") load("//python/private/pypi:pip_repository.bzl", "pip_repository") +load("//python/private/pypi:uv_lock.bzl", _uv_lock = "uv_lock") load("//python/private/pypi:whl_library_alias.bzl", _whl_library_alias = "whl_library_alias") load("//python/private/whl_filegroup:whl_filegroup.bzl", _whl_filegroup = "whl_filegroup") compile_pip_requirements = pip_compile package_annotation = _package_annotation pip_parse = pip_repository +uv_lock = _uv_lock whl_filegroup = _whl_filegroup # Extra utilities visible to rules_python users. diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 149074dd3f..784f96e848 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -243,9 +243,9 @@ bzl_library( ":pep508_evaluate", ":pep508_requirement", ":select_whl", + ":uv_lock_to_requirements", "//python/private:normalize_name", "//python/private:repo_utils", - "//python/uv/private:uv_lock_to_requirements", ], ) @@ -435,6 +435,30 @@ bzl_library( ], ) +bzl_library( + name = "uv_lock", + srcs = ["uv_lock.bzl"], + deps = [ + ":multi_pip_parse", + ":uv_lock_repo", + "//python:versions", + "@bazel_skylib//lib:types", + ], +) + +bzl_library( + name = "uv_lock_repo", + srcs = ["uv_lock_repo.bzl"], + deps = [ + ":attrs", + ":pip_repository", + ":pip_repository_attrs", + ":uv_lock_to_requirements", + "//python/private:repo_utils", + "@toml.bzl//:toml", + ], +) + bzl_library( name = "venv_entry_point", srcs = ["venv_entry_point.bzl"], @@ -672,6 +696,11 @@ bzl_library( srcs = ["platform.bzl"], ) +bzl_library( + name = "uv_lock_to_requirements", + srcs = ["uv_lock_to_requirements.bzl"], +) + bzl_library( name = "version_from_filename", srcs = ["version_from_filename.bzl"], diff --git a/python/private/pypi/multi_pip_parse.bzl b/python/private/pypi/multi_pip_parse.bzl index 60496c2eca..cbf2b2b76f 100644 --- a/python/private/pypi/multi_pip_parse.bzl +++ b/python/private/pypi/multi_pip_parse.bzl @@ -114,7 +114,7 @@ def install_deps(**whl_library_kwargs): rctx.file("requirements.bzl", requirements_bzl) rctx.file("BUILD.bazel", "exports_files(['requirements.bzl'])") -_multi_pip_parse = repository_rule( +multi_pip_parse_hub_repo = repository_rule( _multi_pip_parse_impl, attrs = { "default_version": attr.string(), @@ -158,7 +158,7 @@ def multi_pip_parse(name, default_version, python_versions, python_interpreter_t ) pip_parses[python_version] = pip_parse_name - return _multi_pip_parse( + return multi_pip_parse_hub_repo( name = name, default_version = default_version, pip_parses = pip_parses, diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 2ac8b3724f..fe83ac2acc 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -28,7 +28,6 @@ behavior. load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:repo_utils.bzl", "repo_utils") -load("//python/uv/private:uv_lock_to_requirements.bzl", "uv_lock_extras_map") # buildifier: disable=bzl-visibility load(":argparse.bzl", "argparse") load(":hash.bzl", "hash") load(":index_sources.bzl", "index_sources") @@ -36,6 +35,7 @@ load(":parse_requirements_txt.bzl", "parse_requirements_txt") load(":pep508_evaluate.bzl", "evaluate") load(":pep508_requirement.bzl", "requirement") load(":select_whl.bzl", "select_whl") +load(":uv_lock_to_requirements.bzl", "uv_lock_extras_map") def parse_requirements( ctx, diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index 436af73c8d..c293b1e4cb 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -82,7 +82,14 @@ package(default_visibility = ["//visibility:public"]) exports_files(["requirements.bzl"]) """ -def _pip_repository_impl(rctx): +def render_pip_repository(rctx, *, requirements_txt = None): + """Render a PyPI hub repository for WORKSPACE from requirements or uv.lock. + + Args: + rctx: {type}`repository_ctx` Handle to the rule repository context. + requirements_txt: {type}`str | None` Optional pre-rendered requirements.txt + string (e.g. converted from `uv.lock`). + """ logger = repo_utils.logger(rctx) python_interpreter = pypi_repo_utils.resolve_python_interpreter( rctx, @@ -109,10 +116,25 @@ def _pip_repository_impl(rctx): "windows_x86_64", ] - marker_env = env(os = rctx.os.name, arch = rctx.os.arch, python_version = python_version) - requirements_by_platform = parse_requirements( - rctx, - requirements_by_platform = requirements_files_by_platform( + if requirements_txt != None: + # Convert uv.lock to requirements.txt text and feed it through the + # requirements_lock parser via a stub context instead of passing uv_lock + # directly to parse_requirements(). The native uv_lock path in + # parse_requirements() requires populated whl_platform_tags/whl_abi_tags + # to select wheels and omits --hash flags from requirement_line, whereas + # WORKSPACE whl_library delegates resolution and hash verification to + # pip using requirement_line. + parse_ctx = struct( + read = lambda _: requirements_txt, + ) + req_files_by_platform = requirements_files_by_platform( + requirements_lock = rctx.attr.uv_lock, + extra_pip_args = rctx.attr.extra_pip_args, + platforms = platforms, + ) + else: + parse_ctx = rctx + req_files_by_platform = requirements_files_by_platform( requirements_by_platform = rctx.attr.requirements_by_platform, requirements_linux = rctx.attr.requirements_linux, requirements_lock = rctx.attr.requirements_lock, @@ -120,7 +142,12 @@ def _pip_repository_impl(rctx): requirements_windows = rctx.attr.requirements_windows, extra_pip_args = rctx.attr.extra_pip_args, platforms = platforms, - ), + ) + + marker_env = env(os = rctx.os.name, arch = rctx.os.arch, python_version = python_version) + requirements_by_platform = parse_requirements( + parse_ctx, + requirements_by_platform = req_files_by_platform, extra_pip_args = rctx.attr.extra_pip_args, platforms = { p: struct( @@ -271,6 +298,9 @@ def _pip_repository_impl(rctx): return +def _pip_repository_impl(rctx): + render_pip_repository(rctx) + pip_repository = repository_rule( attrs = dict( annotations = attr.string_dict( diff --git a/python/private/pypi/uv_lock.bzl b/python/private/pypi/uv_lock.bzl new file mode 100644 index 0000000000..85b49417fb --- /dev/null +++ b/python/private/pypi/uv_lock.bzl @@ -0,0 +1,83 @@ +"""A WORKSPACE macro for installing PyPI dependencies from a uv.lock file.""" + +load("@bazel_skylib//lib:types.bzl", "types") +load("//python:versions.bzl", "MINOR_MAPPING") +load(":multi_pip_parse.bzl", "multi_pip_parse_hub_repo") +load(":uv_lock_repo.bzl", "uv_lock_repo") + +def uv_lock( + name, + uv_lock, + default_version = None, + python_versions = None, + python_interpreter_target = None, + minor_mapping = None, + **kwargs): + """Accepts a resolved `uv.lock` file and installs the dependencies listed within. + + Supports both single-version and multi-version Python setups in `WORKSPACE`. + + :::{versionadded} VERSION_NEXT_FEATURE + ::: + + Args: + name: {type}`str` The name of the generated PyPI hub repository. + uv_lock: {type}`Label` A resolved `uv.lock` file. + default_version: {type}`str | None` Optional default Python version when + configuring multiple Python versions. + python_versions: {type}`list[str] | None` Optional list of Python versions + when `python_interpreter_target` is a dict. Defaults to the keys of + `python_interpreter_target`. + python_interpreter_target: {type}`Label | str | dict[str, Label] | None` + Either a single target label for the host Python interpreter, or a + dict mapping Python `X.Y` versions to host interpreter targets. + minor_mapping: {type}`dict[str, str] | None` Optional mapping from `X.Y` + to `X.Y.Z` Python versions for multi-version hub aliases. Defaults + to `MINOR_MAPPING` from `//python:versions.bzl`. + **kwargs: Additional repository rule arguments forwarded to each + underlying PyPI repository (see {obj}`pip_parse` for supported + attributes). + """ + _uv_lock_repo = kwargs.pop("_uv_lock_repo", uv_lock_repo) + _multi_pip_parse_hub_repo = kwargs.pop( + "_multi_pip_parse_hub_repo", + multi_pip_parse_hub_repo, + ) + + if not types.is_dict(python_interpreter_target): + if python_interpreter_target != None: + kwargs["python_interpreter_target"] = python_interpreter_target + _uv_lock_repo( + name = name, + uv_lock = uv_lock, + **kwargs + ) + return + + python_versions = python_versions or sorted(python_interpreter_target.keys()) + minor_mapping = minor_mapping or MINOR_MAPPING + pip_parses = {} + for python_version in python_versions: + if python_version not in python_interpreter_target: + fail( + "Missing python_interpreter_target for Python version %s in '%s'" % ( + python_version, + name, + ), + ) + + spoke_name = name + "_" + python_version.replace(".", "_") + _uv_lock_repo( + name = spoke_name, + python_interpreter_target = python_interpreter_target[python_version], + uv_lock = uv_lock, + **kwargs + ) + pip_parses[python_version] = spoke_name + + _multi_pip_parse_hub_repo( + name = name, + default_version = default_version, + pip_parses = pip_parses, + minor_mapping = minor_mapping, + ) diff --git a/python/private/pypi/uv_lock_repo.bzl b/python/private/pypi/uv_lock_repo.bzl new file mode 100644 index 0000000000..363a0c03c3 --- /dev/null +++ b/python/private/pypi/uv_lock_repo.bzl @@ -0,0 +1,53 @@ +"""A WORKSPACE repository rule for installing PyPI dependencies from uv.lock.""" + +load("@toml.bzl", "toml") +load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR") +load(":attrs.bzl", COMMON_ATTRS = "ATTRS") +load(":pip_repository.bzl", "render_pip_repository") +load(":pip_repository_attrs.bzl", "ATTRS") +load(":uv_lock_to_requirements.bzl", "uv_lock_to_requirements") + +def _uv_lock_repo_impl(rctx): + uv_lock_content = toml.decode(rctx.read(rctx.attr.uv_lock)) + requirements_txt = uv_lock_to_requirements(uv_lock_content) + render_pip_repository( + rctx, + requirements_txt = requirements_txt, + ) + +uv_lock_repo = repository_rule( + attrs = dict( + annotations = attr.string_dict( + doc = """ +Optional annotations to apply to packages. Keys should be package names, with +capitalization matching the input lockfile, and values should be generated +using the `package_annotation` macro. +""", + ), + experimental_requirement_cycles = ATTRS["experimental_requirement_cycles"], + extra_hub_aliases = ATTRS["extra_hub_aliases"], + use_hub_alias_dependencies = ATTRS["use_hub_alias_dependencies"], + uv_lock = attr.label( + allow_single_file = True, + mandatory = True, + doc = """ +A resolved `uv.lock` file containing the transitive set of dependencies. +""", + ), + _config_template = attr.label( + default = ":config.bzl.tmpl", + ), + _template = attr.label( + default = ":requirements.bzl.tmpl.workspace", + ), + **COMMON_ATTRS + ), + doc = """ +Accepts a resolved `uv.lock` file and installs the dependencies listed within. +""", + implementation = _uv_lock_repo_impl, + environ = [ + "RULES_PYTHON_PIP_ISOLATED", + REPO_DEBUG_ENV_VAR, + ], +) diff --git a/python/private/pypi/uv_lock_to_requirements.bzl b/python/private/pypi/uv_lock_to_requirements.bzl new file mode 100644 index 0000000000..af2014de19 --- /dev/null +++ b/python/private/pypi/uv_lock_to_requirements.bzl @@ -0,0 +1,263 @@ +"""Convert a parsed uv.lock to requirements.txt format.""" + +def uv_lock_extras_map(uv_lock): + """Compute extras for each package from uv.lock data. + + Args: + uv_lock: a decoded JSON struct from a uv.lock file. + + Returns: + A dict of {package_name: [extra1, extra2, ...]} for packages with extras. + """ + extras_map = {} + for pkg in uv_lock.get("package", []): + pkg_name = pkg.get("name", "") + + for extra in pkg.get("provides-extras", pkg.get("extras", [])): + _add_extras(extras_map, pkg_name, [extra]) + + opt_deps = pkg.get("optional-dependencies", {}) + if opt_deps: + _add_extras(extras_map, pkg_name, _sorted(opt_deps.keys())) + + deps = pkg.get("dependencies", []) + for dep in deps: + dep_name = dep.get("name", "") + dep_extras_raw = dep.get("extra", []) + dep_extras = [dep_extras_raw] if type(dep_extras_raw) == "string" else dep_extras_raw + if dep_extras and dep_name != pkg_name: + _add_extras(extras_map, dep_name, dep_extras) + + metadata = pkg.get("metadata", {}) + for rd in metadata.get("requires-dist", []): + rd_name = rd.get("name", "") + rd_extras_raw = rd.get("extras", []) + rd_extras = [rd_extras_raw] if type(rd_extras_raw) == "string" else rd_extras_raw + if rd_extras and rd_name != pkg_name: + _add_extras(extras_map, rd_name, rd_extras) + + return extras_map + +def _uv_lock_markers_map(uv_lock): + """Compute effective environment markers for each package in uv.lock. + + Propagates dependency-edge markers from root packages through the + dependency graph and combines them with package-level resolution-markers. + + Args: + uv_lock: a decoded JSON struct from a uv.lock file. + + Returns: + A dict mapping (package_name, version) to a PEP 508 marker string ("" if + unconditional). + """ + packages = uv_lock.get("package", []) + if not packages: + return {} + + by_name = {} + by_name_version = {} + for idx, pkg in enumerate(packages): + pkg_name = pkg.get("name", "") + pkg_version = pkg.get("version", "") + by_name.setdefault(pkg_name, []).append(idx) + by_name_version[(pkg_name, pkg_version)] = idx + + outgoing = [[] for _ in range(len(packages))] + has_incoming = [False for _ in range(len(packages))] + for idx, pkg in enumerate(packages): + pkg_name = pkg.get("name", "") + all_deps = list(pkg.get("dependencies", [])) + for _extra, deps in pkg.get("optional-dependencies", {}).items(): + all_deps.extend(deps) + for _group, deps in pkg.get("dev-dependencies", {}).items(): + all_deps.extend(deps) + + for dep in all_deps: + dep_name = dep.get("name", "") + if not dep_name or dep_name == pkg_name: + continue + dep_version = dep.get("version", "") + if dep_version and (dep_name, dep_version) in by_name_version: + targets = [by_name_version[(dep_name, dep_version)]] + else: + targets = by_name.get(dep_name, []) + edge_marker = dep.get("marker", "") + for target_idx in targets: + has_incoming[target_idx] = True + outgoing[idx].append((target_idx, edge_marker)) + + # reach[idx] is True (unconditionally reachable from a root) or a list of + # disjunctive path marker strings. + reach = [] + for idx, pkg in enumerate(packages): + source = pkg.get("source", {}) + is_root = ( + not has_incoming[idx] or + bool(source.get("virtual")) or + bool(source.get("editable")) + ) + reach.append(True if is_root else []) + + for _ in range(len(packages)): + changed = False + for u in range(len(packages)): + u_reach = reach[u] + if u_reach == []: + continue + for v, edge_marker in outgoing[u]: + if reach[v] == True: + continue + if u_reach == True: + if not edge_marker: + reach[v] = True + changed = True + elif edge_marker not in reach[v]: + reach[v].append(edge_marker) + changed = True + else: + for d in u_reach: + combined = _and_combine(d, edge_marker) if edge_marker else d + if combined not in reach[v]: + reach[v].append(combined) + changed = True + if not changed: + break + + markers_map = {} + for idx, pkg in enumerate(packages): + pkg_name = pkg.get("name", "") + pkg_version = pkg.get("version", "") + v_reach = reach[idx] + if v_reach == True or v_reach == []: + reach_expr = "" + else: + reach_expr = " or ".join(v_reach) + + res_markers = pkg.get("resolution-markers", []) + res_expr = " or ".join(res_markers) if res_markers else "" + + if reach_expr and res_expr: + final_expr = _and_combine(reach_expr, res_expr) + else: + final_expr = reach_expr or res_expr + + markers_map[(pkg_name, pkg_version)] = final_expr + + return markers_map + +def uv_lock_to_requirements(uv_lock): + """Convert a parsed uv.lock JSON struct to a requirements.txt formatted string. + + Args: + uv_lock: a decoded JSON struct from a uv.lock file. + + Returns: + A requirements.txt formatted string. + """ + packages = uv_lock.get("package", []) + extras_map = uv_lock_extras_map(uv_lock) + markers_map = _uv_lock_markers_map(uv_lock) + + dependents = {} + for pkg in packages: + pkg_name = pkg.get("name", "") + deps = pkg.get("dependencies", []) + for dep in deps: + dep_name = dep.get("name", "") + if dep_name != pkg_name: + _add_dependent(dependents, dep_name, pkg_name) + opt_deps = pkg.get("optional-dependencies", {}) + for _extra, deps in opt_deps.items(): + for dep in deps: + dep_name = dep.get("name", "") + if dep_name != pkg_name: + _add_dependent(dependents, dep_name, pkg_name) + + lines = [] + for pkg in packages: + source = pkg.get("source", {}) + if not source.get("registry"): + continue + + pkg_name = pkg.get("name", "") + version = pkg.get("version", "") + + marker_expr = markers_map.get((pkg_name, version), "") + hashes = _collect_hashes(pkg) + + pkg_extras = extras_map.get(pkg_name, []) + if pkg_extras: + req = "{}[{}]=={}".format(pkg_name, ",".join(pkg_extras), version) + else: + req = "{}=={}".format(pkg_name, version) + if marker_expr: + req += " ; " + marker_expr + + if hashes: + req += " \\" + lines.append(req) + + _emit_hashes(lines, hashes) + + dep_vias = _sorted(dependents.get(pkg_name, [])) + if dep_vias: + if len(dep_vias) == 1: + lines.append(" # via " + dep_vias[0]) + else: + lines.append(" # via") + for via in dep_vias: + lines.append(" # " + via) + + lines.append("") + + return "\n".join(lines) + +def _and_combine(a, b): + if not a: + return b + if not b or a == b: + return a + a_clauses = a.split(" and ") if " or " not in a else [a] + b_clauses = b.split(" and ") if " or " not in b else [b] + if all([c in a_clauses for c in b_clauses]): + return a + if all([c in b_clauses for c in a_clauses]): + return b + a_str = "({})".format(a) if " or " in a else a + b_str = "({})".format(b) if " or " in b else b + return "{} and {}".format(a_str, b_str) + +def _add_dependent(dependents, dep_name, dependent_name): + if dep_name not in dependents: + dependents[dep_name] = [] + if dependent_name not in dependents[dep_name]: + dependents[dep_name].append(dependent_name) + +def _add_extras(extras_map, pkg_name, extras): + existing = extras_map.get(pkg_name, []) + for extra in extras: + if extra not in existing: + existing.append(extra) + extras_map[pkg_name] = _sorted(existing) + +def _collect_hashes(pkg): + hashes = [] + for wheel in pkg.get("wheels", []): + whash = wheel.get("hash", "") + if whash.startswith("sha256:"): + hashes.append(whash[len("sha256:"):]) + sdist = pkg.get("sdist") + if sdist: + shash = sdist.get("hash", "") + if shash.startswith("sha256:"): + hashes.append(shash[len("sha256:"):]) + return _sorted(hashes) + +def _sorted(items): + return sorted(items) + +def _emit_hashes(lines, hashes): + for i, h in enumerate(hashes): + suffix = " \\" if i < len(hashes) - 1 else "" + lines.append(" --hash=sha256:{}{}".format(h, suffix)) diff --git a/python/uv/private/BUILD.bazel b/python/uv/private/BUILD.bazel index e4e652ae78..1eb5c46a52 100644 --- a/python/uv/private/BUILD.bazel +++ b/python/uv/private/BUILD.bazel @@ -146,16 +146,6 @@ bzl_library( deps = ["//python/private:text_util"], ) -# keep -bzl_library( - name = "uv_lock_to_requirements", - srcs = ["uv_lock_to_requirements.bzl"], - visibility = [ - "//python/private:__subpackages__", - "//python/uv:__subpackages__", - ], -) - bzl_library( name = "uv_toolchain_config", srcs = ["uv_toolchain_config.bzl"], diff --git a/python/uv/private/uv_lock_to_requirements.bzl b/python/uv/private/uv_lock_to_requirements.bzl deleted file mode 100644 index 73d912bbfd..0000000000 --- a/python/uv/private/uv_lock_to_requirements.bzl +++ /dev/null @@ -1,139 +0,0 @@ -"""Convert a parsed uv.lock to requirements.txt format.""" - -def uv_lock_extras_map(uv_lock): - """Compute extras for each package from uv.lock data. - - Args: - uv_lock: a decoded JSON struct from a uv.lock file. - - Returns: - A dict of {package_name: [extra1, extra2, ...]} for packages with extras. - """ - extras_map = {} - for pkg in uv_lock.get("package", []): - pkg_name = pkg.get("name", "") - - for extra in pkg.get("provides-extras", pkg.get("extras", [])): - _add_extras(extras_map, pkg_name, [extra]) - - opt_deps = pkg.get("optional-dependencies", {}) - if opt_deps: - _add_extras(extras_map, pkg_name, _sorted(opt_deps.keys())) - - deps = pkg.get("dependencies", []) - for dep in deps: - dep_name = dep.get("name", "") - dep_extras_raw = dep.get("extra", []) - dep_extras = [dep_extras_raw] if type(dep_extras_raw) == "string" else dep_extras_raw - if dep_extras and dep_name != pkg_name: - _add_extras(extras_map, dep_name, dep_extras) - - metadata = pkg.get("metadata", {}) - for rd in metadata.get("requires-dist", []): - rd_name = rd.get("name", "") - rd_extras_raw = rd.get("extras", []) - rd_extras = [rd_extras_raw] if type(rd_extras_raw) == "string" else rd_extras_raw - if rd_extras and rd_name != pkg_name: - _add_extras(extras_map, rd_name, rd_extras) - - return extras_map - -def uv_lock_to_requirements(uv_lock): - """Convert a parsed uv.lock JSON struct to a requirements.txt formatted string. - - Args: - uv_lock: a decoded JSON struct from a uv.lock file. - - Returns: - A requirements.txt formatted string. - """ - packages = uv_lock.get("package", []) - extras_map = uv_lock_extras_map(uv_lock) - - dependents = {} - for pkg in packages: - pkg_name = pkg.get("name", "") - deps = pkg.get("dependencies", []) - for dep in deps: - dep_name = dep.get("name", "") - if dep_name != pkg_name: - _add_dependent(dependents, dep_name, pkg_name) - opt_deps = pkg.get("optional-dependencies", {}) - for _extra, deps in opt_deps.items(): - for dep in deps: - dep_name = dep.get("name", "") - if dep_name != pkg_name: - _add_dependent(dependents, dep_name, pkg_name) - - lines = [] - for pkg in packages: - source = pkg.get("source", {}) - if not source.get("registry"): - continue - - pkg_name = pkg.get("name", "") - version = pkg.get("version", "") - - markers = pkg.get("resolution-markers", []) - hashes = _collect_hashes(pkg) - - pkg_extras = extras_map.get(pkg_name, []) - if pkg_extras: - req = "{}[{}]=={}".format(pkg_name, ",".join(pkg_extras), version) - else: - req = "{}=={}".format(pkg_name, version) - if markers: - req += " ; " + " or ".join(markers) - - if hashes: - req += " \\" - lines.append(req) - - _emit_hashes(lines, hashes) - - dep_vias = _sorted(dependents.get(pkg_name, [])) - if dep_vias: - if len(dep_vias) == 1: - lines.append(" # via " + dep_vias[0]) - else: - lines.append(" # via") - for via in dep_vias: - lines.append(" # " + via) - - lines.append("") - - return "\n".join(lines) - -def _add_dependent(dependents, dep_name, dependent_name): - if dep_name not in dependents: - dependents[dep_name] = [] - if dependent_name not in dependents[dep_name]: - dependents[dep_name].append(dependent_name) - -def _add_extras(extras_map, pkg_name, extras): - existing = extras_map.get(pkg_name, []) - for extra in extras: - if extra not in existing: - existing.append(extra) - extras_map[pkg_name] = _sorted(existing) - -def _collect_hashes(pkg): - hashes = [] - for wheel in pkg.get("wheels", []): - whash = wheel.get("hash", "") - if whash.startswith("sha256:"): - hashes.append(whash[len("sha256:"):]) - sdist = pkg.get("sdist") - if sdist: - shash = sdist.get("hash", "") - if shash.startswith("sha256:"): - hashes.append(shash[len("sha256:"):]) - return _sorted(hashes) - -def _sorted(items): - return sorted(items) - -def _emit_hashes(lines, hashes): - for i, h in enumerate(hashes): - suffix = " \\" if i < len(hashes) - 1 else "" - lines.append(" --hash=sha256:{}{}".format(h, suffix)) diff --git a/tests/pypi/uv_lock/BUILD.bazel b/tests/pypi/uv_lock/BUILD.bazel new file mode 100644 index 0000000000..c41a97f9c9 --- /dev/null +++ b/tests/pypi/uv_lock/BUILD.bazel @@ -0,0 +1,3 @@ +load(":uv_lock_tests.bzl", "uv_lock_test_suite") + +uv_lock_test_suite(name = "uv_lock_tests") diff --git a/tests/pypi/uv_lock/uv_lock_tests.bzl b/tests/pypi/uv_lock/uv_lock_tests.bzl new file mode 100644 index 0000000000..4bf3d01625 --- /dev/null +++ b/tests/pypi/uv_lock/uv_lock_tests.bzl @@ -0,0 +1,89 @@ +"""Unit tests for the WORKSPACE uv_lock macro.""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python:pip.bzl", "uv_lock") +load("//tests/support/mocks:mocks.bzl", "mocks") + +_tests = [] + +def _test_single_version(env): + mock_repo = mocks.repo() + mock_hub = mocks.repo() + + uv_lock( + name = "pypi", + python_interpreter_target = "@python3_14_host//:python", + uv_lock = "//:uv.lock", + extra_pip_args = ["--retries=5"], + _uv_lock_repo = mock_repo.rule, + _multi_pip_parse_hub_repo = mock_hub.rule, + ) + + env.expect.that_collection(mock_hub.calls).contains_exactly([]) + env.expect.that_collection(mock_repo.calls).contains_exactly([ + { + "extra_pip_args": ["--retries=5"], + "name": "pypi", + "python_interpreter_target": "@python3_14_host//:python", + "uv_lock": "//:uv.lock", + }, + ]) + +_tests.append(_test_single_version) + +def _test_multi_version(env): + mock_repo = mocks.repo() + mock_hub = mocks.repo() + + uv_lock( + name = "pypi", + default_version = "3.10", + minor_mapping = { + "3.10": "3.10.20", + "3.11": "3.11.15", + }, + python_interpreter_target = { + "3.10": "@python_3_10_host//:python", + "3.11": "@python_3_11_host//:python", + }, + uv_lock = "//requirements:uv.lock", + _uv_lock_repo = mock_repo.rule, + _multi_pip_parse_hub_repo = mock_hub.rule, + ) + + env.expect.that_collection(mock_repo.calls).contains_exactly([ + { + "name": "pypi_3_10", + "python_interpreter_target": "@python_3_10_host//:python", + "uv_lock": "//requirements:uv.lock", + }, + { + "name": "pypi_3_11", + "python_interpreter_target": "@python_3_11_host//:python", + "uv_lock": "//requirements:uv.lock", + }, + ]) + env.expect.that_collection(mock_hub.calls).contains_exactly([ + { + "default_version": "3.10", + "minor_mapping": { + "3.10": "3.10.20", + "3.11": "3.11.15", + }, + "name": "pypi", + "pip_parses": { + "3.10": "pypi_3_10", + "3.11": "pypi_3_11", + }, + }, + ]) + +_tests.append(_test_multi_version) + +def uv_lock_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/uv/lock/uv_lock_to_requirements_tests.bzl b/tests/uv/lock/uv_lock_to_requirements_tests.bzl index 1b947877c9..65a8c2e30f 100644 --- a/tests/uv/lock/uv_lock_to_requirements_tests.bzl +++ b/tests/uv/lock/uv_lock_to_requirements_tests.bzl @@ -1,7 +1,7 @@ """Tests for uv_lock_to_requirements.""" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/uv/private:uv_lock_to_requirements.bzl", "uv_lock_to_requirements") # buildifier: disable=bzl-visibility +load("//python/private/pypi:uv_lock_to_requirements.bzl", "uv_lock_to_requirements") # buildifier: disable=bzl-visibility _tests = [] @@ -285,5 +285,62 @@ pytest-bazel[all]==0.1.6 \\ _tests.append(_test_requires_dist_extras) +def _test_transitive_dependency_edge_markers(env): + """Test that markers on dependency edges propagate transitively.""" + got = uv_lock_to_requirements(json.decode("""{"package":[ + {"name":"cryptography","version":"50.0.2","source":{"registry":"https://pypi.org/simple"},"wheels":[{"hash":"sha256:cccc","url":"https://example.org/cryptography.whl"}]}, + {"name":"keyring","version":"25.7.0","source":{"registry":"https://pypi.org/simple"},"dependencies":[{"name":"secretstorage","marker":"sys_platform == 'linux'"}],"wheels":[{"hash":"sha256:aaaa","url":"https://example.org/keyring.whl"}]}, + {"name":"root-pkg","version":"0.0.0","source":{"virtual":"."},"dependencies":[{"name":"keyring","marker":"platform_machine != 's390x'"}]}, + {"name":"secretstorage","version":"3.5.0","source":{"registry":"https://pypi.org/simple"},"dependencies":[{"name":"cryptography"}],"wheels":[{"hash":"sha256:bbbb","url":"https://example.org/secretstorage.whl"}]} + ]}""")) + env.expect.that_str(got).equals("""\ +cryptography==50.0.2 ; platform_machine != 's390x' and sys_platform == 'linux' \\ + --hash=sha256:cccc + # via secretstorage + +keyring==25.7.0 ; platform_machine != 's390x' \\ + --hash=sha256:aaaa + # via root-pkg + +secretstorage==3.5.0 ; platform_machine != 's390x' and sys_platform == 'linux' \\ + --hash=sha256:bbbb + # via keyring +""") + +_tests.append(_test_transitive_dependency_edge_markers) + +def _test_diamond_dependency_edge_markers(env): + """Test diamond paths with conditional vs unconditional edges.""" + got = uv_lock_to_requirements(json.decode("""{"package":[ + {"name":"both-conditional","version":"1.0.0","source":{"registry":"https://pypi.org/simple"},"wheels":[{"hash":"sha256:1111","url":"https://example.org/bc.whl"}]}, + {"name":"one-unconditional","version":"1.0.0","source":{"registry":"https://pypi.org/simple"},"wheels":[{"hash":"sha256:2222","url":"https://example.org/ou.whl"}]}, + {"name":"pkg-a","version":"0.1.0","source":{"registry":"https://pypi.org/simple"},"dependencies":[{"name":"both-conditional","marker":"sys_platform == 'linux'"},{"name":"one-unconditional","marker":"sys_platform == 'linux'"}],"wheels":[{"hash":"sha256:aaaa","url":"https://example.org/a.whl"}]}, + {"name":"pkg-b","version":"0.2.0","source":{"registry":"https://pypi.org/simple"},"dependencies":[{"name":"both-conditional","marker":"sys_platform == 'win32'"},{"name":"one-unconditional"}],"wheels":[{"hash":"sha256:bbbb","url":"https://example.org/b.whl"}]}, + {"name":"root-pkg","version":"0.0.0","source":{"virtual":"."},"dependencies":[{"name":"pkg-a"},{"name":"pkg-b"}]} + ]}""")) + env.expect.that_str(got).equals("""\ +both-conditional==1.0.0 ; sys_platform == 'linux' or sys_platform == 'win32' \\ + --hash=sha256:1111 + # via + # pkg-a + # pkg-b + +one-unconditional==1.0.0 \\ + --hash=sha256:2222 + # via + # pkg-a + # pkg-b + +pkg-a==0.1.0 \\ + --hash=sha256:aaaa + # via root-pkg + +pkg-b==0.2.0 \\ + --hash=sha256:bbbb + # via root-pkg +""") + +_tests.append(_test_diamond_dependency_edge_markers) + def uv_lock_to_requirements_test_suite(name): test_suite(name = name, basic_tests = _tests)