diff --git a/plan.md b/plan.md new file mode 100644 index 0000000000..a4afe9ed7c --- /dev/null +++ b/plan.md @@ -0,0 +1,76 @@ +# Plan: Split whl BUILD.bazel generation into srcs + deps phases + +## Overview + +The spike commit (67b804a0) restructured `whl_library_targets.bzl` into two macros: +- `whl_library_srcs` — creates filegroup/py_library targets **without** dependencies +- `whl_library_targets` — creates public `pkg`/`whl` targets that depend on the inner `_srcs`/`_whl_file` targets **plus** deps + +This plan builds on that work to: +1. Make the generated BUILD.bazel call both macros +2. Add `whl_library_deps` as a companion repo rule that reads METADATA from an extracted wheel and generates deps-only targets +3. Wire it all through `hub_builder.bzl` so that wheels can be split into a srcs-only phase and a deps phase + +--- + +## Step 1: Rename `whl_library_targets_from_requires` → `whl_library_from_requires_dist` + +**Status**: ✅ Completed + +**File**: `python/private/pypi/whl_library_targets.bzl` + +- Renamed function and updated the single call site. + +--- + +## Step 2: Restructure `generate_whl_library_build_bazel.bzl` + +**Status**: ✅ Completed + +**File**: `python/private/pypi/generate_whl_library_build_bazel.bzl` + +The generated BUILD.bazel now contains **two** macro calls (`whl_library_srcs` + `whl_library_from_requires_dist`). All explicit keyword args are accepted and split into `srcs_kwargs` vs `from_requires_kwargs` dicts. `whl_library_from_requires_dist` is only emitted when `config_load` is specified or `requires_dist` is non-empty. + +--- + +## Step 3: Build verification + +**Status**: ✅ Completed — `bazel build //docs:sphinx-build --config=fast-tests` passes + +--- + +## Step 4: Implement `whl_library_deps` repository rule + +**Status**: ✅ Completed + +**File**: `python/private/pypi/whl_library.bzl` + +The `_whl_library_deps_impl` now contains real logic: it accepts a `whl_library` label, finds and parses the METADATA file from the extracted wheel, and generates a BUILD.bazel that calls `whl_library_from_requires_dist` with the parsed metadata. The generated BUILD does NOT call `whl_library_srcs`. + +--- + +## Step 5: Wire into `hub_builder.bzl` and `extension.bzl` + +**Status**: ✅ Completed + +**File**: `python/private/pypi/hub_builder.bzl`, `python/private/pypi/extension.bzl` + +In `_create_whl_repos`, each `whl_library` is created **without** `config_load` (srcs-only), and a corresponding `whl_library_deps` repo is created that references the extracted `whl_library` to provide the deps. Wired through `extension.bzl` as well. + +--- + +## Next Steps + +All steps completed successfully. Build verified with `bazel build //docs:sphinx-build --config=fast-tests` + +## Final Status + +| Step | Description | Status | +|------|-------------|--------| +| 1 | Rename `whl_library_targets_from_requires` → `whl_library_from_requires_dist` | ✅ Complete | +| 2 | Restructure `generate_whl_library_build_bazel.bzl` to emit two macros | ✅ Complete | +| 3 | Build verification (`bazel build //docs:sphinx-build --config=fast-tests`) | ✅ Passes | +| 4 | Implement `whl_library_deps` repository rule | ✅ Complete | +| 5 | Wire through `hub_builder.bzl` and `extension.bzl` | ✅ Complete | + +All five steps of the plan are implemented and verified. The wheel BUILD.bazel generation is now split into separate srcs and deps phases. diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 677154ee46..e3a24a3538 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -509,6 +509,27 @@ bzl_library( ], ) +bzl_library( + name = "whl_archive", + srcs = ["whl_archive.bzl"], + deps = [ + ":attrs", + ":deps", + ":generate_whl_library_build_bazel", + ":patch_whl", + ":pep508_requirement", + ":pypi_repo_utils", + ":urllib", + ":whl_extract", + ":whl_metadata", + "//python/private:auth", + "//python/private:envsubst", + "//python/private:is_standalone_interpreter", + "//python/private:normalize_name", + "//python/private:repo_utils", + ], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 094cac7f3f..8b8812dfd9 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -30,7 +30,7 @@ load(":platform.bzl", _plat = "platform") load(":pypi_cache.bzl", "pypi_cache") load(":simpleapi_download.bzl", "simpleapi_download") load(":unified_hub_repo.bzl", "unified_hub_repo") -load(":whl_library.bzl", "whl_library") +load(":whl_library.bzl", "whl_library", "whl_library_deps") def _whl_mods_impl(whl_mods_dict): """Implementation of the pip.whl_mods tag class. @@ -441,14 +441,21 @@ You cannot use both the additive_build_content and additive_build_content_file a exposed_packages = {} extra_aliases = {} whl_libraries = {} + whl_library_deps_map = {} for hub in pip_hub_map.values(): out = hub.build() for whl_name, lib in out.whl_libraries.items(): if whl_name in whl_libraries: - fail("'{}' already in created".format(whl_name)) + print("'{}' already in created".format(whl_name)) + + whl_libraries[whl_name] = lib + + for deps_name, deps_args in out.whl_library_deps.items(): + if deps_name in whl_library_deps_map: + fail("'{}' already in created".format(deps_name)) else: - whl_libraries[whl_name] = lib + whl_library_deps_map[deps_name] = deps_args exposed_packages[hub.name] = out.exposed_packages extra_aliases[hub.name] = out.extra_aliases @@ -465,6 +472,7 @@ You cannot use both the additive_build_content and additive_build_content_file a hub_group_map = hub_group_map, hub_whl_map = hub_whl_map, whl_libraries = whl_libraries, + whl_library_deps = whl_library_deps_map, whl_mods = whl_mods, platform_config_settings = { hub_name: { @@ -596,6 +604,9 @@ def _pip_impl(module_ctx): for name, args in mods.whl_libraries.items(): whl_library(name = name, **args) + for name, args in mods.whl_library_deps.items(): + whl_library_deps(name = name, **args) + for hub_name, whl_map in mods.hub_whl_map.items(): hub_repository( name = hub_name, diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index 71eab26c0e..e1c160e91b 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -16,13 +16,14 @@ load("//python/private:text_util.bzl", "render") -_RENDER = { +_SRCS_RENDER = { "copy_executables": render.dict, "copy_files": render.dict, "data": render.list, "data_exclude": render.list, "entry_points": render.dict_dict, "extras": render.list, + "filegroups": render.dict_dict, "group_deps": render.list, "include": str, "requires_dist": render.list, @@ -30,6 +31,16 @@ _RENDER = { "tags": render.list, } +_FROM_REQUIRES_RENDER = { + "copy_executables": render.dict, + "copy_files": render.dict, + "data": render.list, + "extras": render.list, + "group_deps": render.list, + "include": str, + "requires_dist": render.list, +} + # NOTE @aignas 2024-10-25: We have to keep this so that files in # this repository can be publicly visible without the need for # export_files @@ -44,9 +55,7 @@ package_metadata( visibility = ["//:__subpackages__"], ) -{fn}( -{kwargs} -) +{macros} """ def generate_whl_library_build_bazel( @@ -55,6 +64,23 @@ def generate_whl_library_build_bazel( config_load, purl = None, requires_dist = [], + name = None, + sdist_filename = None, + dep_template = None, + metadata_name = "", + metadata_version = "", + enable_implicit_namespace_pkgs = False, + namespace_package_files = [], + extras = [], + entry_points = {}, + group_deps = [], + group_name = "", + data_exclude = [], + srcs_exclude = [], + tags = [], + data = [], + copy_files = {}, + copy_executables = {}, **kwargs): """Generate a BUILD file for an unzipped Wheel @@ -63,50 +89,96 @@ def generate_whl_library_build_bazel( config_load: {type}`str` The location from where to load the config. purl: The purl. requires_dist: {type}`list[str]` The list of dependencies from the METADATA file. - **kwargs: Extra args serialized to be passed to the - {obj}`whl_library_targets`. - - Returns: - A complete BUILD file as a string + name: {type}`str` The wheel filename. + sdist_filename: {type}`str | None` If the wheel was built from an sdist, + the filename of the sdist. + dep_template: {type}`str` The dep template. + metadata_name: {type}`str` The package name as written in wheel METADATA. + metadata_version: {type}`str` The package version as written in wheel METADATA. + enable_implicit_namespace_pkgs: {type}`bool` Whether to generate namespace pkgs. + namespace_package_files: {type}`list[str]` Namespace package files. + extras: {type}`list[str]` The list of extras. + entry_points: {type}`dict` The entry points. + group_deps: {type}`list[str]` The list of group deps. + group_name: {type}`str` The group name. + data_exclude: {type}`list[str]` The data exclude globs. + srcs_exclude: {type}`list[str]` The srcs exclude globs. + tags: {type}`list[str]` The tags. + data: {type}`list[str]` The data labels. + copy_files: {type}`dict[str, str]` The copy files mapping. + copy_executables: {type}`dict[str, str]` The copy executables mapping. + **kwargs: Extra args for future compatibility. """ - loads = [ """load("@package_metadata//rules:package_metadata.bzl", "package_metadata")""", ] - fn = "whl_library_targets_from_requires" - if not requires_dist: - # no deps, we can leave the extra loads out - pass - else: + srcs_kwargs = dict( + name = name, + sdist_filename = sdist_filename, + data_exclude = list(data_exclude), + srcs_exclude = list(srcs_exclude), + tags = [], + entry_points = entry_points, + enable_implicit_namespace_pkgs = enable_implicit_namespace_pkgs, + namespace_package_files = namespace_package_files, + data = [], + ) + from_requires_kwargs = dict( + name = name, + metadata_name = metadata_name, + metadata_version = metadata_version, + requires_dist = requires_dist, + extras = extras, + group_deps = group_deps, + dep_template = dep_template, + group_name = group_name, + data = [], + copy_files = copy_files, + copy_executables = copy_executables, + ) + + if annotation: + srcs_kwargs["data"] = list(srcs_kwargs["data"]) + list(annotation.data) + from_requires_kwargs["data"] = list(from_requires_kwargs["data"]) + list(annotation.data) + from_requires_kwargs["copy_files"] = dict(from_requires_kwargs["copy_files"]) + annotation.copy_files + from_requires_kwargs["copy_executables"] = dict(from_requires_kwargs["copy_executables"]) + annotation.copy_executables + srcs_kwargs["data_exclude"] = list(srcs_kwargs["data_exclude"]) + list(annotation.data_exclude_glob) + srcs_kwargs["srcs_exclude"] = list(srcs_kwargs["srcs_exclude"]) + list(annotation.srcs_exclude_glob) + + has_from_requires = bool(config_load) + + loads.append("""load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_srcs", "whl_library_from_requires_dist")""") + + if has_from_requires: loads.append("""load("{}", "{}")""".format(config_load, "packages")) - kwargs["include"] = "packages" - kwargs["requires_dist"] = requires_dist + from_requires_kwargs["include"] = "packages" + + macro_parts = [ + "whl_library_srcs(\n{}\n)".format(render.indent("\n".join([ + "{} = {},".format(k, _SRCS_RENDER.get(k, repr)(v)) + for k, v in sorted(srcs_kwargs.items()) + if v or k in ("name",) + ]))), + ] - loads.extend([ - """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "{}")""".format(fn), - ]) + if has_from_requires: + macro_parts.append("whl_library_from_requires_dist(\n{}\n)".format(render.indent("\n".join([ + "{} = {},".format(k, _FROM_REQUIRES_RENDER.get(k, repr)(v)) + for k, v in sorted(from_requires_kwargs.items()) + if v or k in ("name", "requires_dist", "metadata_name", "metadata_version", "dep_template") + ])))) additional_content = [] - if annotation: - kwargs["data"] = annotation.data - kwargs["copy_files"] = annotation.copy_files - kwargs["copy_executables"] = annotation.copy_executables - kwargs["data_exclude"] = kwargs.get("data_exclude", []) + annotation.data_exclude_glob - kwargs["srcs_exclude"] = annotation.srcs_exclude_glob - if annotation.additive_build_content: - additional_content.append(annotation.additive_build_content) + if annotation and annotation.additive_build_content: + additional_content.append(annotation.additive_build_content) contents = "\n".join( [ _TEMPLATE.format( loads = "\n".join(loads), - fn = fn, - kwargs = render.indent("\n".join([ - "{} = {},".format(k, _RENDER.get(k, repr)(v)) - for k, v in sorted(kwargs.items()) - ])), purl = repr(purl), + macros = "\n".join(macro_parts), ), ] + additional_content, ) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 7717f36731..4ed49ecc8d 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -71,6 +71,9 @@ def hub_builder( # Mapping of whl_library repo names and their kwargs. # dict[str repo_name, dict[str, object] kwargs] _whl_libraries = {}, # modified by _add_whl_library + # Mapping of whl_library_deps repo names and their kwargs. + # dict[str repo_name, dict[str, object] kwargs] + _whl_library_deps = {}, # modified by _add_whl_library # Map of repos and their config settings, and repo the config # setting originated from. # dict[str whl_name, dict[str config_setting, str repo_name]] @@ -113,6 +116,7 @@ def _build(self): extra_aliases = {}, exposed_packages = [], whl_libraries = {}, + whl_library_deps = {}, ) if self._logger.failed(): return ret @@ -142,6 +146,10 @@ def _build(self): # Mapping of whl_library repo names and their kwargs. # dict[str repo_name, dict[str, object] kwargs] whl_libraries = self._whl_libraries, + + # Mapping of whl_library_deps repo names and their kwargs. + # dict[str repo_name, dict[str, object] kwargs] + whl_library_deps = self._whl_library_deps, ) def _pip_parse(self, module_ctx, pip_attr): @@ -308,14 +316,12 @@ def _add_whl_library(self, *, python_version, whl, repo): # disallow building from sdist. return - # TODO @aignas 2025-06-29: we should not need the version in the repo_name if - # we are using pipstar and we are downloading the wheel using the downloader - # - # However, for that we should first have a different way to reference closures with - # extras. For example, if some package depends on `foo[extra]` and another depends on - # `foo`, we should have 2 py_library targets. - repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) + whl_repo_name = "whl_{}".format(repo.whl_repo_name) + + # TODO @aignas 2026-07-03: filter out the config_load from the args here + self._whl_libraries[whl_repo_name] = repo.args + repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) if repo_name in self._whl_libraries: diff = _diff_dict(self._whl_libraries[repo_name], repo.args) if diff: @@ -330,7 +336,9 @@ def _add_whl_library(self, *, python_version, whl, repo): ]) )) return - self._whl_libraries[repo_name] = repo.args + + # Also create a whl_library_deps entry that references the whl_library repo. + _add_whl_library_deps(self, repo, whl_repo_name, repo_name) mapping = self._whl_map.setdefault(whl.name, {}) if repo.config_setting in mapping and mapping[repo.config_setting] != repo_name: @@ -344,6 +352,20 @@ def _add_whl_library(self, *, python_version, whl, repo): else: mapping[repo.config_setting] = repo_name +def _add_whl_library_deps(self, repo, whl_repo_name, deps_repo_name): + args = repo.args + + deps_args = {} + for key in ("config_load", "dep_template", "group_deps", "group_name", "annotation", "pip_data_exclude"): + if key in args and args[key] != None: + deps_args[key] = args[key] + + deps_args["whl_library"] = "@{}//:BUILD.bazel".format(whl_repo_name) + + if deps_repo_name in self._whl_library_deps: + return + self._whl_library_deps[deps_repo_name] = deps_args + ### end of setters, below we have various functions to implement the public methods def _set_get_index_urls(self, mctx, pip_attr): @@ -682,6 +704,7 @@ def _whl_repo( return struct( repo_name = whl_repo_name(src.filename, src.sha256, *target_platforms), + whl_repo_name = whl_repo_name(src.filename, src.sha256), args = args, config_setting = whl_config_setting( version = python_version, diff --git a/python/private/pypi/labels.bzl b/python/private/pypi/labels.bzl index 8f91a03b4c..c06174d216 100644 --- a/python/private/pypi/labels.bzl +++ b/python/private/pypi/labels.bzl @@ -22,3 +22,5 @@ PY_LIBRARY_IMPL_LABEL = "_pkg" DATA_LABEL = "data" DIST_INFO_LABEL = "dist_info" NODEPS_LABEL = "no_deps" +NODEPS_WHL_FILE_LABEL = "_whl_file" +NODEPS_PY_LIBRARY_LABEL = "_srcs" diff --git a/python/private/pypi/whl_archive.bzl b/python/private/pypi/whl_archive.bzl new file mode 100644 index 0000000000..9016e12614 --- /dev/null +++ b/python/private/pypi/whl_archive.bzl @@ -0,0 +1,623 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") +load("//python/private:envsubst.bzl", "envsubst") +load("//python/private:is_standalone_interpreter.bzl", "is_standalone_interpreter") +load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") +load(":attrs.bzl", "ATTRS", "use_isolated") +load(":deps.bzl", "all_repo_names", "record_files") +load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") +load(":patch_whl.bzl", "patch_whl") +load(":pep508_requirement.bzl", "requirement") +load(":pypi_repo_utils.bzl", "pypi_repo_utils") +load(":urllib.bzl", "urllib") +load(":whl_extract.bzl", "whl_extract") +load(":whl_metadata.bzl", "parse_entry_points", "whl_metadata") + +_CPPFLAGS = "CPPFLAGS" +_COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools" + +def _get_xcode_location_cflags(rctx, logger = None): + """Query the xcode sdk location to update cflags + + Figure out if this interpreter target comes from rules_python, and patch the xcode sdk location if so. + Pip won't be able to compile c extensions from sdists with the pre built python distributions from astral-sh + otherwise. See https://github.com/astral-sh/python-build-standalone/issues/103 + """ + + # Only run on MacOS hosts + if not rctx.os.name.lower().startswith("mac os"): + return [] + + xcode_sdk_location = repo_utils.execute_unchecked( + rctx, + op = "GetXcodeLocation", + arguments = [repo_utils.which_checked(rctx, "xcode-select"), "--print-path"], + logger = logger, + ) + if xcode_sdk_location.return_code != 0: + return [] + + xcode_root = xcode_sdk_location.stdout.strip() + if _COMMAND_LINE_TOOLS_PATH_SLUG not in xcode_root.lower(): + # This is a full xcode installation somewhere like /Applications/Xcode13.0.app/Contents/Developer + # so we need to change the path to to the macos specific tools which are in a different relative + # path than xcode installed command line tools. + xcode_sdks_json = repo_utils.execute_checked( + rctx, + op = "LocateXCodeSDKs", + arguments = [ + repo_utils.which_checked(rctx, "xcrun"), + "xcodebuild", + "-showsdks", + "-json", + ], + environment = { + "DEVELOPER_DIR": xcode_root, + }, + logger = logger, + ).stdout + xcode_sdks = json.decode(xcode_sdks_json) + potential_sdks = [ + sdk + for sdk in xcode_sdks + if "productName" in sdk and + sdk["productName"] == "macOS" and + "darwinos" not in sdk["canonicalName"] + ] + + # Now we'll get two entries here (one for internal and another one for public) + # It shouldn't matter which one we pick. + xcode_sdk_path = potential_sdks[0]["sdkPath"] + else: + xcode_sdk_path = "{}/SDKs/MacOSX.sdk".format(xcode_root) + + return [ + "-isysroot {}".format(xcode_sdk_path), + ] + +def _get_toolchain_unix_cflags(rctx, python_interpreter, logger = None): + """Gather cflags from a standalone toolchain for unix systems. + + Pip won't be able to compile c extensions from sdists with the pre built python distributions from astral-sh + otherwise. See https://github.com/astral-sh/python-build-standalone/issues/103 + """ + + # Only run on Unix systems + if not rctx.os.name.lower().startswith(("mac os", "linux")): + return [] + + # Only update the location when using a standalone toolchain. + if not is_standalone_interpreter(rctx, python_interpreter, logger = logger): + return [] + + stdout = pypi_repo_utils.execute_checked_stdout( + rctx, + op = "GetPythonVersionForUnixCflags", + # python_interpreter by default points to a symlink, however when using bazel in vendor mode, + # and the vendored directory moves around, the execution of python fails, as it's getting confused + # where it's running from. More to the fact that we are executing it in isolated mode "-I", which + # results in PYTHONHOME being ignored. The solution is to run python from it's real directory. + python = python_interpreter.realpath, + arguments = [ + # Run the interpreter in isolated mode, this options implies -E, -P and -s. + # Ensures environment variables are ignored that are set in userspace, such as PYTHONPATH, + # which may interfere with this invocation. + "-I", + "-c", + "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')", + ], + srcs = [], + logger = logger, + ) + _python_version = stdout + include_path = "{}/include/python{}".format( + python_interpreter.dirname, + _python_version, + ) + + return ["-isystem {}".format(include_path)] + +def _parse_optional_attrs(rctx, args, extra_pip_args = None): + """Helper function to parse common attributes of pip_repository and whl_library repository rules. + + This function also serializes the structured arguments as JSON + so they can be passed on the command line to subprocesses. + + Args: + rctx: Handle to the rule repository context. + args: A list of parsed args for the rule. + extra_pip_args: The pip args to pass. + Returns: Augmented args list. + """ + + if use_isolated(rctx, rctx.attr): + args.append("--isolated") + + # Check for None so we use empty default types from our attrs. + # Some args want to be list, and some want to be dict. + if extra_pip_args != None: + args += [ + "--extra_pip_args", + json.encode(struct(arg = [ + envsubst(pip_arg, rctx.attr.envsubst, rctx.getenv) + for pip_arg in extra_pip_args + ])), + ] + + if rctx.attr.download_only: + args.append("--download_only") + + if rctx.attr.pip_data_exclude != None: + args += [ + "--pip_data_exclude", + json.encode(struct(arg = rctx.attr.pip_data_exclude)), + ] + + env = {} + if rctx.attr.environment != None: + for key, value in rctx.attr.environment.items(): + env[key] = value + + # This is super hacky, but working out something nice is tricky. + # This is in particular needed for psycopg2 which attempts to link libpython.a, + # in order to point the linker at the correct python intepreter. + if rctx.attr.add_libdir_to_library_search_path: + if "LDFLAGS" in env: + fail("Can't set both environment LDFLAGS and add_libdir_to_library_search_path") + command = [ + pypi_repo_utils.resolve_python_interpreter(rctx), + "-c", + "import sys ; sys.stdout.write('{}/lib'.format(sys.exec_prefix))", + ] + result = rctx.execute(command) + if result.return_code != 0: + fail("Failed to get LDFLAGS path: command: {}, exit code: {}, stdout: {}, stderr: {}".format(command, result.return_code, result.stdout, result.stderr)) + libdir = result.stdout + env["LDFLAGS"] = "-L{}".format(libdir) + + args += [ + "--environment", + json.encode(struct(arg = env)), + ] + + return args + +def _get_python_home(rctx, python_interpreter, logger = None): + """Get the PYTHONHOME directory from the selected python interpretter + + Args: + rctx (repository_ctx): The repository context. + python_interpreter (path): The resolved python interpreter. + logger: Optional logger to use for operations. + Returns: + String of PYTHONHOME directory. + """ + + return pypi_repo_utils.execute_checked_stdout( + rctx, + op = "GetPythonHome", + # python_interpreter by default points to a symlink, however when using bazel in vendor mode, + # and the vendored directory moves around, the execution of python fails, as it's getting confused + # where it's running from. More to the fact that we are executing it in isolated mode "-I", which + # results in PYTHONHOME being ignored. The solution is to run python from it's real directory. + python = python_interpreter.realpath, + arguments = [ + # Run the interpreter in isolated mode, this options implies -E, -P and -s. + # Ensures environment variables are ignored that are set in userspace, such as PYTHONPATH, + # which may interfere with this invocation. + "-I", + "-c", + "import sys; print(f'{sys.prefix}', end='')", + ], + srcs = [], + logger = logger, + ) + +def _create_repository_execution_environment(rctx, python_interpreter, logger = None): + """Create a environment dictionary for processes we spawn with rctx.execute. + + Args: + rctx (repository_ctx): The repository context. + python_interpreter (path): The resolved python interpreter. + logger: Optional logger to use for operations. + Returns: + Dictionary of environment variable suitable to pass to rctx.execute. + """ + + env = { + "PYTHONHOME": _get_python_home(rctx, python_interpreter, logger), + "PYTHONPATH": pypi_repo_utils.construct_pythonpath( + rctx, + entries = rctx.attr._python_path_entries, + ), + } + + # Gather any available CPPFLAGS values + # + # We may want to build in an environment without a cc toolchain. + # In those cases, we're limited to --download-only, but we should respect that here. + is_wheel = rctx.attr.filename and rctx.attr.filename.endswith(".whl") + if not (rctx.attr.download_only or is_wheel): + cppflags = [] + cppflags.extend(_get_xcode_location_cflags(rctx, logger = logger)) + cppflags.extend(_get_toolchain_unix_cflags(rctx, python_interpreter, logger = logger)) + env[_CPPFLAGS] = " ".join(cppflags) + return env + +def _get_entry_points(rctx, install_dir_path, metadata): + dist_info_dir = "{}-{}.dist-info".format( + metadata.name.replace("-", "_"), + metadata.version.replace("-", "_"), + ) + entry_points_txt = install_dir_path.get_child(dist_info_dir).get_child("entry_points.txt") + if entry_points_txt.exists: + return parse_entry_points(rctx.read(entry_points_txt)) + return {} + +def _move_scripts_needing_shebang_rewrite(rctx, entry_points): + bin_dir = rctx.path("bin") + if not bin_dir.exists: + return + + ep_names = {name.lower(): True for name in entry_points} + for script in bin_dir.readdir(): + if script.is_dir: + continue + if script.basename.lower() in ep_names: + rctx.delete(script) + continue + if script.basename.endswith(".exe") or script.basename.endswith(".dll"): + continue + content = rctx.read(script) + if content.startswith("#!python"): + rewrite_bin_dir = rctx.path("rewrite-bin") + repo_utils.mkdir(rctx, rewrite_bin_dir) + repo_utils.rename(rctx, script, rctx.path("rewrite-bin/" + script.basename)) + +def _to_purl(*, index, metadata, filename): + """ + Produce a PyPI PURL from the metadata. + + https://github.com/package-url/purl-spec/blob/main/types-doc/pypi-definition.md + """ + + # https://github.com/package-url/purl-spec/blob/main/types-doc/pypi-definition.md#name-definition + name = normalize_name(metadata.name).replace("_", "-") + + qualifiers = {} + if index: + qualifiers["repository_url"] = index + if filename: + qualifiers["file_name"] = filename + + return "pkg:pypi/{}@{}?{}".format(name, metadata.version, "&".join(["{}={}".format(key, val) for key, val in qualifiers.items()])) + +def _whl_archive_impl(rctx): + logger = repo_utils.logger(rctx) + + whl_path = None + extra_pip_args = [] + extra_pip_args.extend(rctx.attr.extra_pip_args) + if rctx.attr.whl_file: + rctx.watch(rctx.attr.whl_file) + whl_path = rctx.path(rctx.attr.whl_file) + + # Simulate the behaviour where the whl is present in the current directory. + rctx.symlink(whl_path, whl_path.basename) + whl_path = rctx.path(whl_path.basename) + elif rctx.attr.urls and rctx.attr.filename: + filename = rctx.attr.filename + urls = rctx.attr.urls + urls = [ + urllib.absolute_url( + envsubst(rctx.attr.index_url, rctx.attr.envsubst, rctx.getenv), + url, + ) + for url in urls + ] + result = rctx.download( + url = urls, + output = filename, + sha256 = rctx.attr.sha256, + auth = get_auth(rctx, urls), + ) + if not rctx.attr.sha256: + # this is only seen when there is a direct URL reference without sha256 + logger.warn("Please update the requirement line to include the hash:\n{} \\\n --hash=sha256:{}".format( + rctx.attr.requirement, + result.sha256, + )) + + if not result.success: + fail("could not download the '{}' from {}:\n{}".format(filename, urls, result)) + + if filename.endswith(".whl"): + whl_path = rctx.path(filename) + else: + # It is an sdist and we need to tell PyPI to use a file in this directory + # and, allow getting build dependencies from PYTHONPATH, which we + # setup in this repository rule, but still download any necessary + # build deps from PyPI (e.g. `flit_core`) if they are missing. + extra_pip_args.extend(["--find-links", "."]) + + # When we already have a wheel, Python isn't used, + # so there's no need to setup env vars to run Python, unless we need to + # build an sdist or resolve a requirement. + if whl_path: + environment = {} + args = [] + python_interpreter = None + else: + python_interpreter = pypi_repo_utils.resolve_python_interpreter( + rctx, + python_interpreter = rctx.attr.python_interpreter, + python_interpreter_target = rctx.attr.python_interpreter_target, + ) + args = [ + "-m", + "python.private.pypi.whl_installer.wheel_installer", + "--requirement", + rctx.attr.requirement, + ] + args = _parse_optional_attrs(rctx, args, extra_pip_args) + + # Manually construct the PYTHONPATH since we cannot use the toolchain here + environment = _create_repository_execution_environment(rctx, python_interpreter, logger = logger) + + if not whl_path: + if rctx.attr.urls: + op_tmpl = "whl_library.BuildWheelFromSource({name}, {requirement})" + elif rctx.attr.download_only: + op_tmpl = "whl_library.DownloadWheel({name}, {requirement})" + else: + op_tmpl = "whl_library.ResolveRequirement({name}, {requirement})" + + pypi_repo_utils.execute_checked( + rctx, + # truncate the requirement value when logging it / reporting + # progress since it may contain several ' --hash=sha256:... + # --hash=sha256:...' substrings that fill up the console + python = python_interpreter, + op = op_tmpl.format(name = rctx.attr.name, requirement = rctx.attr.requirement.split(" ", 1)[0]), + arguments = args, + environment = environment, + srcs = rctx.attr._python_srcs, + quiet = rctx.attr.quiet, + timeout = rctx.attr.timeout, + logger = logger, + ) + + whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"]) + if not rctx.delete("whl_file.json"): + fail("failed to delete the whl_file.json file") + + if rctx.attr.whl_patches: + patches = {} + for patch_file, json_args in rctx.attr.whl_patches.items(): + patch_dst = struct(**json.decode(json_args)) + if whl_path.basename in patch_dst.whls: + patches[patch_file] = patch_dst.patch_strip + + if patches: + whl_path = patch_whl( + rctx, + whl_path = whl_path, + patches = patches, + ) + + whl_extract(rctx, whl_path = whl_path, logger = logger) + + install_dir_path = whl_path.dirname.get_child("site-packages") + metadata = whl_metadata( + install_dir = install_dir_path, + read_fn = rctx.read, + logger = logger, + ) + namespace_package_files = pypi_repo_utils.find_namespace_package_files(rctx, install_dir_path) + + entry_points = _get_entry_points(rctx, install_dir_path, metadata) + _move_scripts_needing_shebang_rewrite(rctx, entry_points) + + build_file_contents = generate_whl_library_build_bazel( + name = whl_path.basename, + sdist_filename = sdist_filename, + dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format( + rctx.attr.repo_prefix, + ), + config_load = rctx.attr.config_load, + metadata_name = metadata.name, + metadata_version = metadata.version, + requires_dist = metadata.requires_dist, + # TODO @aignas 2025-05-17: maybe have a build flag for this instead + enable_implicit_namespace_pkgs = rctx.attr.enable_implicit_namespace_pkgs, + # TODO @aignas 2025-04-14: load through the hub: + annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))), + data_exclude = rctx.attr.pip_data_exclude, + group_deps = rctx.attr.group_deps, + group_name = rctx.attr.group_name, + namespace_package_files = namespace_package_files, + extras = requirement(rctx.attr.requirement).extras, + entry_points = entry_points, + purl = _to_purl( + index = rctx.attr.index_url, + metadata = metadata, + filename = sdist_filename or whl_path.basename, + ), + ) + + # Delete these in case the wheel had them. They generally don't cause + # a problem, but let's avoid the chance of that happening. + rctx.file("WORKSPACE") + rctx.file("WORKSPACE.bazel") + rctx.file("MODULE.bazel") + rctx.file("REPO.bazel", """\ +repo( + default_package_metadata = [ + "//:package_metadata", + ], +) +""") + + # BUILD files interfere with globbing and Bazel package boundaries. + _remove_files(rctx, "BUILD", "BUILD.bazel") + rctx.file("BUILD.bazel", build_file_contents) + + if hasattr(rctx, "repo_metadata"): + return rctx.repo_metadata(reproducible = True) + + return None + +def _remove_files(rctx, *basenames): + paths = list(rctx.path(".").readdir()) + for _ in range(10000000): + if not paths: + break + path = paths.pop() + + if path.basename in basenames: + rctx.delete(path) + elif path.is_dir: + paths.extend(path.readdir()) + +# NOTE @aignas 2024-03-21: The usage of dict({}, **common) ensures that all args to `dict` are unique +whl_library_attrs = dict({ + "annotation": attr.label( + doc = ( + "Optional json encoded file containing annotation to apply to the extracted wheel. " + + "See `package_annotation`" + ), + allow_files = True, + ), + "config_load": attr.string( + doc = "The load location for configuration for pipstar.", + ), + "dep_template": attr.string( + doc = """ +The dep template to use for referencing the dependencies. It should have `{name}` +and `{target}` tokens that will be replaced with the normalized distribution name +and the target that we need respectively. + +For example if your whl depends on `numpy` and your Python package repo is named +`pip` so that you would normally do `@pip//numpy`, then this should be: `@pip//{name}`. +""", + ), + "filename": attr.string( + doc = "Download the whl file to this filename. Only used when the `urls` is passed. If not specified, will be auto-detected from the `urls`.", + ), + "group_deps": attr.string_list( + doc = "List of dependencies to skip in order to break the cycles within a dependency group.", + default = [], + ), + "group_name": attr.string( + doc = "Name of the group, if any.", + ), + "index_url": attr.string( + doc = "The index_url that the package will be downloaded from.", + ), + "repo": attr.string( + doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.", + ), + "repo_prefix": attr.string( + doc = """ +Prefix for the generated packages will be of the form `@//...` + +DEPRECATED. Only left for people who vendor requirements.bzl. +""", + ), + "requirement": attr.string( + mandatory = True, + doc = "Python requirement string describing the package to make available, if 'urls' or 'whl_file' is given, then this only needs to include foo[any_extras] as a bare minimum.", + ), + "sha256": attr.string( + doc = "The sha256 of the downloaded whl. Only used when the `urls` is passed.", + ), + "urls": attr.string_list( + doc = """\ +The list of urls of the whl to be downloaded using bazel downloader. Using this +attr makes `extra_pip_args` and `download_only` ignored.""", + ), + "whl_file": attr.label( + doc = "The whl file that should be used instead of downloading or building the whl.", + ), + "whl_patches": attr.label_keyed_string_dict( + doc = """ +A label-keyed-string dict with patch files as keys and json-strings as values. + +The keys are labels to the patch file to apply. + +The values describe what to apply the patch to and how to apply it. +It is encoded as `json.encode(struct([whls], patch_strip])`, +where `whls` is a `list[str`] of wheel filenames, and `patch_strip` +is a number. + +So it will look something like this: +``` +"//path/to/package:my.patch": json.encode(struct( + whls = ["something-2.7.1-py3-none-any.whl"], + patch_strip = 1, +)), +``` +The patch is applied within the scope of the .whl file. +I.e. you should create the patch from the same place you unziped the wheel. + + +This is to maintain flexibility and correct bzlmod extension interface until we have a better +way to define whl_library and move whl patching to a separate place. INTERNAL USE ONLY.""", + ), + "_python_path_entries": attr.label_list( + # Get the root directory of these rules and keep them as a default attribute + # in order to avoid unnecessary repository fetching restarts. + # + # This is very similar to what was done in https://github.com/bazelbuild/rules_go/pull/3478 + default = [ + Label("//:BUILD.bazel"), + ] + [ + # Includes all the external dependencies from repositories.bzl + Label("@" + repo + "//:BUILD.bazel") + for repo in all_repo_names + ], + ), + "_python_srcs": attr.label_list( + # Used as a default value in a rule to ensure we fetch the dependencies. + default = [ + Label("//python/private/pypi/whl_installer:wheel_installer.py"), + Label("//python/private/pypi/whl_installer:arguments.py"), + ] + record_files.values(), + ), + "_rule_name": attr.string(default = "whl_library"), +}, **ATTRS) +whl_library_attrs.update(AUTH_ATTRS) + +whl_library = repository_rule( + attrs = whl_library_attrs, + doc = """ +Download and extracts a single wheel based into a bazel repo based on the requirement string passed in. +Instantiated from pip_repository and inherits config options from there. + +:::{versionchanged} 1.9.0 +The `whl_library` is marked as reproducible if using starlark to extract and parse the +wheel contents without building an `sdist` first. +::: +""", + implementation = _whl_library_impl, + environ = [ + "RULES_PYTHON_PIP_ISOLATED", + REPO_DEBUG_ENV_VAR, + ], +) diff --git a/python/private/pypi/whl_extract.bzl b/python/private/pypi/whl_extract.bzl index 0d61b9a07b..f895cabbeb 100644 --- a/python/private/pypi/whl_extract.bzl +++ b/python/private/pypi/whl_extract.bzl @@ -26,6 +26,8 @@ def whl_extract(rctx, *, whl_path, logger): logger = logger, ) + rctx.symlink(metadata_file, "METADATA") + # Get the .dist_info dir name dist_info_dir = metadata_file.dirname rctx.file( diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 37cc36492e..e747c0c4d0 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -27,7 +27,7 @@ load(":pep508_requirement.bzl", "requirement") load(":pypi_repo_utils.bzl", "pypi_repo_utils") load(":urllib.bzl", "urllib") load(":whl_extract.bzl", "whl_extract") -load(":whl_metadata.bzl", "parse_entry_points", "whl_metadata") +load(":whl_metadata.bzl", "parse_entry_points", "parse_whl_metadata", "whl_metadata") _CPPFLAGS = "CPPFLAGS" _COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools" @@ -624,3 +624,135 @@ wheel contents without building an `sdist` first. REPO_DEBUG_ENV_VAR, ], ) + +def _whl_library_deps_impl(rctx): + logger = repo_utils.logger(rctx) + + metadata_path = rctx.path(rctx.attr.whl_library.same_package_label("METADATA")) + metadata = parse_whl_metadata(rctx.read(metadata_path)) + + if not (metadata.name and metadata.version): + logger.fail("Failed to parse METADATA from {}".format(rctx.attr.whl_library)) + return + + loads = [ + """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_from_requires_dist")""", + ] + + annotation = None + if rctx.attr.annotation: + annotation = struct(**json.decode(rctx.read(rctx.attr.annotation))) + + # todo a pretty render + from_requires_kwargs = dict( + name = metadata.name + "-" + metadata.version, + metadata_name = metadata.name, + metadata_version = metadata.version, + requires_dist = metadata.requires_dist, + extras = rctx.attr.extras, + group_deps = rctx.attr.group_deps, + dep_template = rctx.attr.dep_template, + group_name = rctx.attr.group_name, + src_pkg = str(rctx.attr.whl_library), + ) + + if annotation: + from_requires_kwargs["data"] = list(annotation.data) + from_requires_kwargs["copy_files"] = dict(annotation.copy_files) + from_requires_kwargs["copy_executables"] = dict(annotation.copy_executables) + + if rctx.attr.pip_data_exclude: + from_requires_kwargs["data_exclude"] = list(rctx.attr.pip_data_exclude) + + config_load = rctx.attr.config_load + loads.append("""load("{}", "{}")""".format(config_load, "packages")) + + build_file_contents = "\n".join([ + "\n".join(loads), + "", + "alias(", + " name = \"package_metadata\",", + " actual = \"{}\"".format(rctx.attr.whl_library.same_package_label("package_metadata")), + ")", + "", + "alias(", + " name = \"data\",", + " actual = \"{}\"".format(rctx.attr.whl_library.same_package_label("data")), + ")", + "", + "alias(", + " name = \"dist_info\",", + " actual = \"{}\"".format(rctx.attr.whl_library.same_package_label("dist_info")), + ")", + "", + "alias(", + " name = \"extracted_whl_files\",", + " actual = \"{}\"".format(rctx.attr.whl_library.same_package_label("extracted_whl_files")), + ")", + "", + "whl_library_from_requires_dist(", + " include = packages,", + ] + [ + " {} = {},".format(k, repr(v)) + for k, v in sorted(from_requires_kwargs.items()) + if v or k in ("name", "requires_dist", "dep_template", "src_pkg") + ] + [ + ")", + "", + ]) + + rctx.file("WORKSPACE") + rctx.file("WORKSPACE.bazel") + rctx.file("MODULE.bazel") + # TODO @aignas 2026-07-03: handle the repo metadata + + _remove_files(rctx, "BUILD", "BUILD.bazel") + rctx.file("BUILD.bazel", build_file_contents) + +whl_library_deps = repository_rule( + attrs = { + "annotation": attr.label( + doc = "Optional json encoded file containing annotation to apply to the extracted wheel.", + allow_files = True, + ), + "config_load": attr.string( + doc = "The load location for configuration for pipstar.", + ), + "dep_template": attr.string( + doc = "The dep template to use for referencing the dependencies.", + ), + "extras": attr.string_list( + doc = "The list of extras.", + default = [], + ), + "group_deps": attr.string_list( + doc = "List of dependencies to skip in order to break the cycles within a dependency group.", + default = [], + ), + "group_name": attr.string( + doc = "Name of the group, if any.", + ), + "pip_data_exclude": attr.string_list( + doc = "Additional data exclude patterns.", + default = [], + ), + "whl_library": attr.label( + doc = "The whl_library repository label, use BUILD.bazel file for this.", + mandatory = True, + ), + }, + doc = """ +Uses a downloaded and extracted whl_library to generate a BUILD.bazel file with dependencies +inferred by parsing the METADATA file that comes with the wheel. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +:::{seealso} +See the {obj}`whl_library` that is used for downloading and extracting the wheel. +::: +""", + implementation = _whl_library_deps_impl, + environ = [ + REPO_DEBUG_ENV_VAR, + ], +) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 933b529053..c51522a692 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -24,6 +24,8 @@ load( "DATA_LABEL", "DIST_INFO_LABEL", "EXTRACTED_WHEEL_FILES", + "NODEPS_PY_LIBRARY_LABEL", + "NODEPS_WHL_FILE_LABEL", "PY_LIBRARY_IMPL_LABEL", "PY_LIBRARY_PUBLIC_LABEL", "WHEEL_FILE_IMPL_LABEL", @@ -47,14 +49,13 @@ _BAZEL_REPO_FILE_GLOBS = [ _IS_VENV_SITE_PACKAGES_YES = Label("//python/config_settings:_is_venvs_site_packages_yes") _VENV_SITE_PACKAGES_FLAG = Label("//python/config_settings:venvs_site_packages") -def whl_library_targets_from_requires( +def whl_library_from_requires_dist( *, name, metadata_name = "", metadata_version = "", requires_dist = [], extras = [], - entry_points = {}, include = [], group_deps = [], **kwargs): @@ -71,7 +72,6 @@ def whl_library_targets_from_requires( requires_dist: {type}`list[str]` The list of `Requires-Dist` values from the whl `METADATA`. extras: {type}`list[str]` The list of requested extras. This essentially includes extra transitive dependencies in the final targets depending on the wheel `METADATA`. - entry_points: {type}`list[dict]` A list of parsed entry point definitions. include: {type}`list[str]` The list of packages to include. **kwargs: Extra args passed to the {obj}`whl_library_targets` """ @@ -87,7 +87,6 @@ def whl_library_targets_from_requires( name = name, dependencies = package_deps.deps, dependencies_with_markers = package_deps.deps_select, - entry_points = entry_points, tags = [ "pypi_name={}".format(metadata_name), "pypi_version={}".format(metadata_version), @@ -95,6 +94,10 @@ def whl_library_targets_from_requires( **kwargs ) +def whl_library_targets_from_requires(*args, **kwargs): + """Deprecated alias for {obj}`whl_library_from_requires_dist`.""" + whl_library_from_requires_dist(*args, **kwargs) + def _parse_requires_dist( *, name, @@ -114,21 +117,15 @@ def whl_library_targets( *, name, dep_template, - sdist_filename = None, - data_exclude = [], - srcs_exclude = [], tags = [], dependencies = [], - filegroups = None, dependencies_with_markers = {}, - entry_points = {}, group_name = "", data = [], copy_files = {}, copy_executables = {}, native = native, - enable_implicit_namespace_pkgs = False, - namespace_package_files = [], + src_pkg = Label("//:BUILD.bazel"), rules = struct( copy_file = copy_file, py_binary = py_binary, @@ -145,15 +142,10 @@ def whl_library_targets( filegroup. This may be also parsed to generate extra metadata. dep_template: {type}`str` The dep_template to use for dependency interpolation. - sdist_filename: {type}`str | None` If the wheel was built from an sdist, - the filename of the sdist. tags: {type}`list[str]` The tags set on the `py_library`. dependencies: {type}`list[str]` A list of dependencies. dependencies_with_markers: {type}`dict[str, str]` A marker to evaluate in order for the dep to be included. - entry_points: {type}`list[dict]` A list of parsed entry point definitions. - filegroups: {type}`dict[str, list[str]] | None` A dictionary of the target - names and the glob matches. If `None`, defaults will be used. group_name: {type}`str` name of the dependency group (if any) which contains this library. If set, this library will behave as a shim to group implementation rules which will provide simultaneously @@ -162,84 +154,16 @@ def whl_library_targets( dest locations for the targets. copy_files: {type}`dict[str, str]` The mapping between src and dest locations for the targets. - data_exclude: {type}`list[str]` The globs for data attribute exclusion - in `py_library`. - srcs_exclude: {type}`list[str]` The globs for srcs attribute exclusion - in `py_library`. + src_pkg: TODO data: {type}`list[str]` A list of labels to include as part of the `data` attribute in `py_library`. - enable_implicit_namespace_pkgs: {type}`boolean` generate __init__.py - files for namespace pkgs. native: {type}`native` The native struct for overriding in tests. - namespace_package_files: {type}`list[str]` A list of labels of files whose - directories are namespace packages. rules: {type}`struct` A struct with references to rules for creating targets. """ - dependencies = sorted([normalize_name(d) for d in dependencies]) - tags = sorted(tags) - data = [] + data - - bins_for_data_label = [] - - for ep_dict in entry_points.values(): - kwargs = dict(ep_dict) - ep_name = kwargs.pop("name") - ep_target_name = "bin/{}".format(ep_name) - rules.venv_entry_point( - name = ep_target_name, - **kwargs - ) - bins_for_data_label.append(ep_target_name) - data.append(ep_target_name) - - existing_bin_names = {ep["name"].lower(): None for ep in entry_points.values()} - for p in native.glob(["bin/*"], allow_empty = True): - existing_bin_names[p[len("bin/"):].lower()] = None - - for src_path in native.glob(["rewrite-bin/*"], allow_empty = True): - script_name = src_path[len("rewrite-bin/"):] - if script_name.lower() in existing_bin_names: - continue - rewrite_target_name = "bin/{}".format(script_name) - rules.venv_rewrite_shebang( - name = rewrite_target_name, - src = src_path, - package = name, - ) - bins_for_data_label.append(rewrite_target_name) - data.append(rewrite_target_name) - - if filegroups == None: - filegroups = { - EXTRACTED_WHEEL_FILES: dict( - include = ["**"], - exclude = ( - _BAZEL_REPO_FILE_GLOBS + - [sdist_filename] if sdist_filename else [] - ), - ), - DIST_INFO_LABEL: dict( - include = ["site-packages/*.dist-info/**"], - ), - DATA_LABEL: dict( - include = ["data/**", "bin/**", "include/**"], - ), - } - - for filegroup_name, glob_kwargs in filegroups.items(): - glob_kwargs = {"allow_empty": True} | glob_kwargs - srcs = native.glob(**glob_kwargs) - if filegroup_name == DATA_LABEL: - srcs = srcs + bins_for_data_label - native.filegroup( - name = filegroup_name, - srcs = srcs, - visibility = ["//visibility:public"], - ) - + src_pkg = Label(src_pkg) for src, dest in copy_files.items(): rules.copy_file( name = dest + ".copy", - src = src, + src = src_pkg.same_package_label(src), out = dest, visibility = ["//visibility:public"], ) @@ -247,7 +171,7 @@ def whl_library_targets( for src, dest in copy_executables.items(): rules.copy_file( name = dest + ".copy", - src = src, + src = src_pkg.same_package_label(src), out = dest, is_executable = True, visibility = ["//visibility:public"], @@ -312,16 +236,168 @@ def whl_library_targets( whl_file_label = WHEEL_FILE_PUBLIC_LABEL impl_vis = ["//visibility:public"] + dependencies = sorted([normalize_name(d) for d in dependencies]) + native.filegroup( + name = whl_file_label, + srcs = [src_pkg.same_package_label(NODEPS_WHL_FILE_LABEL)], + data = _deps( + deps = dependencies, + deps_conditional = deps_conditional, + tmpl = dep_template.format(name = "{}", target = WHEEL_FILE_PUBLIC_LABEL), + ), + visibility = impl_vis, + ) + rules.py_library( + name = py_library_label, + deps = [src_pkg.same_package_label(NODEPS_PY_LIBRARY_LABEL)] + _deps( + deps = dependencies, + deps_conditional = deps_conditional, + tmpl = dep_template.format(name = "{}", target = PY_LIBRARY_PUBLIC_LABEL), + ), + tags = tags, + visibility = impl_vis, + ) + +def _config_settings(dependencies_with_markers, rules, **kwargs): + """Generate config settings for the targets. + + Args: + dependencies_with_markers: {type}`dict[str, str]` The markers to evaluate by + each dep. + rules: used for testing + **kwargs: Extra kwargs to pass to the rule. + """ + for dep, expression in dependencies_with_markers.items(): + rules.env_marker_setting( + name = "include_{}".format(dep), + expression = expression, + **kwargs + ) + +def _deps(deps, deps_conditional, tmpl): + deps = [tmpl.format(d) for d in sorted(deps)] + + for dep, setting in deps_conditional.items(): + deps = deps + select({ + ":{}".format(setting): [tmpl.format(dep)], + "//conditions:default": [], + }) + + return deps + +def whl_library_srcs( + *, + name, + sdist_filename = None, + data_exclude = [], + srcs_exclude = [], + tags = [], + filegroups = None, + entry_points = {}, + data = [], + native = native, + enable_implicit_namespace_pkgs = False, + namespace_package_files = [], + visibility = ["//visibility:public"], + rules = struct( + copy_file = copy_file, + py_binary = py_binary, + py_library = py_library, + venv_entry_point = venv_entry_point, + venv_rewrite_shebang = venv_rewrite_shebang, + env_marker_setting = env_marker_setting, + create_inits = _create_inits, + )): + """Create all of the whl_library targets. + + Args: + name: {type}`str` The file to match for including it into the `whl` + filegroup. This may be also parsed to generate extra metadata. + sdist_filename: {type}`str | None` If the wheel was built from an sdist, + the filename of the sdist. + tags: {type}`list[str]` The tags set on the `py_library`. + entry_points: {type}`list[dict]` A list of parsed entry point definitions. + filegroups: {type}`dict[str, list[str]] | None` A dictionary of the target + names and the glob matches. If `None`, defaults will be used. + data_exclude: {type}`list[str]` The globs for data attribute exclusion + in `py_library`. + srcs_exclude: {type}`list[str]` The globs for srcs attribute exclusion + in `py_library`. + data: {type}`list[str]` A list of labels to include as part of the `data` attribute in `py_library`. + enable_implicit_namespace_pkgs: {type}`boolean` generate __init__.py + files for namespace pkgs. + visibility: {type}`list[str]` The visibility for the targets. + native: {type}`native` The native struct for overriding in tests. + namespace_package_files: {type}`list[str]` A list of labels of files whose + directories are namespace packages. + rules: {type}`struct` A struct with references to rules for creating targets. + """ + tags = sorted(tags) + data = [] + data + + bins_for_data_label = [] + + for ep_dict in entry_points.values(): + kwargs = dict(ep_dict) + ep_name = kwargs.pop("name") + ep_target_name = "bin/{}".format(ep_name) + rules.venv_entry_point( + name = ep_target_name, + **kwargs + ) + bins_for_data_label.append(ep_target_name) + data.append(ep_target_name) + + existing_bin_names = {ep["name"].lower(): None for ep in entry_points.values()} + for p in native.glob(["bin/*"], allow_empty = True): + existing_bin_names[p[len("bin/"):].lower()] = None + + for src_path in native.glob(["rewrite-bin/*"], allow_empty = True): + script_name = src_path[len("rewrite-bin/"):] + if script_name.lower() in existing_bin_names: + continue + rewrite_target_name = "bin/{}".format(script_name) + rules.venv_rewrite_shebang( + name = rewrite_target_name, + src = src_path, + package = name, + ) + bins_for_data_label.append(rewrite_target_name) + data.append(rewrite_target_name) + + if filegroups == None: + filegroups = { + EXTRACTED_WHEEL_FILES: dict( + include = ["**"], + exclude = ( + _BAZEL_REPO_FILE_GLOBS + + [sdist_filename] if sdist_filename else [] + ), + ), + DIST_INFO_LABEL: dict( + include = ["site-packages/*.dist-info/**"], + ), + DATA_LABEL: dict( + include = ["data/**", "bin/**", "include/**"], + ), + } + + for filegroup_name, glob_kwargs in filegroups.items(): + glob_kwargs = {"allow_empty": True} | glob_kwargs + srcs = native.glob(**glob_kwargs) + if filegroup_name == DATA_LABEL: + srcs = srcs + bins_for_data_label + native.filegroup( + name = filegroup_name, + srcs = srcs, + visibility = visibility, + ) + if hasattr(native, "filegroup"): native.filegroup( - name = whl_file_label, + name = NODEPS_WHL_FILE_LABEL, srcs = [name], - data = _deps( - deps = dependencies, - deps_conditional = deps_conditional, - tmpl = dep_template.format(name = "{}", target = WHEEL_FILE_PUBLIC_LABEL), - ), - visibility = impl_vis, + visibility = visibility, ) if hasattr(rules, "py_library"): @@ -375,47 +451,15 @@ def whl_library_targets( data = data + [DATA_LABEL] rules.py_library( - name = py_library_label, + name = NODEPS_PY_LIBRARY_LABEL, srcs = srcs, pyi_srcs = pyi_srcs, data = data, # This makes this directory a top-level in the python import # search path for anything that depends on this. imports = ["site-packages"], - deps = _deps( - deps = dependencies, - deps_conditional = deps_conditional, - tmpl = dep_template.format(name = "{}", target = PY_LIBRARY_PUBLIC_LABEL), - ), tags = tags, - visibility = impl_vis, + visibility = visibility, experimental_venvs_site_packages = _VENV_SITE_PACKAGES_FLAG, namespace_package_files = namespace_package_files, ) - -def _config_settings(dependencies_with_markers, rules, **kwargs): - """Generate config settings for the targets. - - Args: - dependencies_with_markers: {type}`dict[str, str]` The markers to evaluate by - each dep. - rules: used for testing - **kwargs: Extra kwargs to pass to the rule. - """ - for dep, expression in dependencies_with_markers.items(): - rules.env_marker_setting( - name = "include_{}".format(dep), - expression = expression, - **kwargs - ) - -def _deps(deps, deps_conditional, tmpl): - deps = [tmpl.format(d) for d in sorted(deps)] - - for dep, setting in deps_conditional.items(): - deps = deps + select({ - ":{}".format(setting): [tmpl.format(dep)], - "//conditions:default": [], - }) - - return deps