From 27a764d3cee51958400f266ab733b5d85b30d082 Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Tue, 20 Oct 2026 11:38:27 -0400 Subject: [PATCH 1/5] ci_scripts: support dynamic Python version selection for builds python-wheels currently has a narrow, manually-set list of Python versions to build for, but this is often at odds with upstream projects. Instead, rework check_versions.py to check and define a list of upstream's targeted Python versions in each package's docs/packages/ file, then adjust pending_versions.py to pull directly from the data populated by check_versions.py. We may still want to exclude certain versions for a given package (for example, to omit an older wheel that won't build on riscv64), so allow passing flags to the script to indicate these versions. Signed-off-by: Trevor Gamblin --- ci_scripts/check_riscv64_deps.py | 3 +- ci_scripts/check_versions.py | 117 +++++++++++++++++++++++++++++-- ci_scripts/pending_versions.py | 95 +++++++++++++++++++++++++ ci_scripts/riscv64_resolve.py | 5 +- 4 files changed, 210 insertions(+), 10 deletions(-) diff --git a/ci_scripts/check_riscv64_deps.py b/ci_scripts/check_riscv64_deps.py index 20713f65bbec..cb1d9e5fbff6 100755 --- a/ci_scripts/check_riscv64_deps.py +++ b/ci_scripts/check_riscv64_deps.py @@ -85,7 +85,8 @@ def main(): nargs="+", default=DEFAULT_PYTHONS, metavar="TAG", - help="interpreter tags without the 'cp' prefix (default: %(default)s)", + help="interpreter tags without the 'cp' prefix; pass the workflow's " + "python-matrix tags (default: %(default)s)", ) parser.add_argument( "--no-registry", diff --git a/ci_scripts/check_versions.py b/ci_scripts/check_versions.py index b24123b8c7fb..fc94e0d593bc 100755 --- a/ci_scripts/check_versions.py +++ b/ci_scripts/check_versions.py @@ -21,10 +21,12 @@ from datetime import datetime, timedelta, timezone from typing import Dict, List, Optional, Set from packaging import version +from packaging.specifiers import SpecifierSet from pathlib import Path import requests import yaml +from pending_versions import RISCV64_PYTHONS from update_doc import normalize_name, version_blocks @@ -34,10 +36,10 @@ DEPRECATED_FILE = "ci_scripts/deprecated.txt" REPO = "riseproject-dev/python-wheels" -# Python versions (interpreter tags) a package must have upstream riscv64 -# wheels for before we deprecate our own build. "3.14t" is the free-threaded +# Python versions riscv64 can build (cp314t -> "3.14t"); an upstream riscv64 +# wheel for any of them deprecates our own build. "3.14t" is the free-threaded # build — a distinct ABI from "3.14", so it needs its own wheel. -TARGET_PYTHON_VERSIONS = ["3.12", "3.13", "3.14", "3.14t"] +TARGET_PYTHON_VERSIONS = [f"3.{tag[3:]}" for tag in RISCV64_PYTHONS] def read_packages() -> List[str]: @@ -173,6 +175,44 @@ def is_pure_python_wheel(package_info: Dict, target_version: str) -> bool: return has_wheels +def upstream_python_versions(package_info: Dict, target_version: str) -> Optional[List[str]]: + """ + The Python versions ("3.10", "3.14t") upstream builds target_version for, + from its wheels on PyPI, or None if PyPI doesn't have that version. + + A cpXY-cpXY or cpXY-cpXYt wheel means that interpreter, and a cpXY-abi3 + wheel a single cpXY build (it loads on every later CPython, gotcha 11). + Without any (sdist-only or pure wheels), requires_python decides over the + RISCV64_PYTHONS, leaving out free-threaded builds since nothing says + upstream supports them. + """ + files = package_info.get("releases", {}).get(target_version) + if not files: + return None + built = set() + for f in files: + tags = _parse_wheel_tags(f.get("filename", "")) + if not tags: + continue + for py in tags[0].split("."): + if not re.fullmatch(r"cp3\d+", py): + continue + for abi in tags[1].split("."): + if abi in (py, py + "t"): + built.add(abi) + elif abi.startswith("abi3"): + built.add(py) + if not built: + spec = SpecifierSet(next((f["requires_python"] for f in files if f.get("requires_python")), "")) + built = {t for t in RISCV64_PYTHONS if not t.endswith("t") and spec.contains(f"3.{t[3:]}")} + return sorted((f"3.{t[3:]}" for t in built), key=lambda v: (int(v[2:].rstrip("t")), v.endswith("t"))) + + +def python_versions_line(pythons: List[str]) -> str: + """Quoted, so YAML can't read '3.10' as the float 3.1.""" + return " python-versions: [" + ", ".join(f"'{v}'" for v in pythons) + "]" + + def find_upstream_issue(package: str) -> Optional[str]: """ Find issue for a package in the wheel_builder's Upstream milestone. @@ -301,10 +341,11 @@ def get_versions_since(package_info: Dict, declared: Set[str], since: datetime) return sorted(result, key=version.parse) -def declare_versions(yaml_file: Path, new_versions: List[str]) -> List[str]: +def declare_versions(yaml_file: Path, new_versions: List[str], package_info: Dict) -> List[str]: """ - Append a bare `- version:` entry for each version not yet listed in the - package YAML, and return the versions that were added. + Append a `- version:` entry, with the `python-versions` upstream builds it + for, for each version not yet listed in the package YAML, and return the + versions that were added. An entry without `tag`/`files` is what the build workflow builds, so this is all an upgrade PR has to change. @@ -317,10 +358,36 @@ def declare_versions(yaml_file: Path, new_versions: List[str]) -> List[str]: lines = [content.rstrip("\n")] for v in added: lines.append(yaml.safe_dump([{"version": v}], default_flow_style=False).rstrip("\n")) + pythons = upstream_python_versions(package_info, v) + if pythons is not None: + lines.append(python_versions_line(pythons)) yaml_file.write_text("\n".join(lines) + "\n") return added +def fill_python_versions(yaml_file: Path, package_info: Dict) -> List[str]: + """ + Add `python-versions` to every entry of the package YAML that lacks it and + whose version PyPI has, preserving the rest of the file byte-for-byte, and + return the versions that were filled in. + """ + lines = yaml_file.read_text().splitlines() + filled = [] + # Bottom-up so earlier block ranges stay valid after inserts. + for start, end in reversed(list(version_blocks(lines))): + raw = str(yaml.safe_load("\n".join(lines[start:end]))[0].get("version")) + if any(line.startswith(" python-versions:") for line in lines[start:end]): + continue + pythons = upstream_python_versions(package_info, raw) + if pythons is None: + continue + lines.insert(start + 1, python_versions_line(pythons)) + filled.append(raw) + if filled: + yaml_file.write_text("\n".join(lines) + "\n") + return filled[::-1] + + def get_yanked_versions(package_info: Dict, package_data: Dict) -> List[str]: """ Released versions of the YAML whose PyPI release is fully yanked and that @@ -461,7 +528,7 @@ def check_upgrade( if new_versions: print(f"[^] {package} can be upgraded: v{registry_version} -> {', '.join(f'v{v}' for v in new_versions)}") if declare: - added = declare_versions(package_yaml(package), new_versions) + added = declare_versions(package_yaml(package), new_versions, pypi_info) print(f" [+] declared {', '.join(f'v{v}' for v in added)} in {package_yaml(package).relative_to(ROOT_DIR)}") return { "status": "need_upgrade", @@ -705,6 +772,24 @@ def _self_test(): assert has_any_riscv64_wheel(abi3_info, "1.0", ["3.12", "3.13", "3.14"]) is True assert has_any_riscv64_wheel(abi3_info, "1.0", ["3.14t"]) is False + def files(requires_python, *wheels): + return [{"filename": f"pkg-1.0-{w}-manylinux_2_28_x86_64.whl", "requires_python": requires_python} + for w in wheels] + [{"filename": "pkg-1.0.tar.gz", "requires_python": requires_python}] + + def pythons(*args): + return upstream_python_versions({"releases": {"1.0": files(*args)}}, "1.0") + + assert upstream_python_versions({"releases": {}}, "1.0") is None + # Upstream's own list, cp313t included; ancient cp37m and pp wheels say nothing. + assert pythons(">=3.7", "cp37-cp37m", "pp310-pypy310_pp73", "cp310-cp310", "cp313-cp313", + "cp313-cp313t", "cp315-cp315t") == ["3.10", "3.13", "3.13t", "3.15t"] + # Compressed tag sets count; abi3 is one build, pure wheels say nothing. + assert pythons(None, "cp313.cp314-cp313") == ["3.13"] + assert pythons(None, "cp310-abi3", "cp314-cp314t", "py3-none") == ["3.10", "3.14t"] + assert pythons(">=3.12,<3.14", "py3-none") == ["3.12", "3.13"] + assert python_versions_line(["3.10", "3.14t"]) == " python-versions: ['3.10', '3.14t']" + assert yaml.safe_load(python_versions_line(["3.10"]).strip()) == {"python-versions": ["3.10"]} + # Per-package PR titles, one PR per package. upgrade = {"status": "need_upgrade", "package": "PyYAML", "registry_version": "6.0", "new_versions": ["6.0.1", "6.0.2"], "yanked": []} @@ -730,6 +815,9 @@ def main(): parser.add_argument("--declare", action="store_true", help="Apply the outcome to the working tree: declare new versions and mark yanked ones in " "docs/packages/.yaml, deprecate packages in the YAML, packages.txt and deprecated.txt") + parser.add_argument("--python-versions", action="store_true", + help="Only add the python-versions upstream builds for to every entry of " + "docs/packages/.yaml lacking it (e.g. a new port's pending version)") parser.add_argument("--report", type=Path, metavar="FILE", help="Write a Markdown summary of the whole run") parser.add_argument("--report-dir", type=Path, metavar="DIR", help="Write one Markdown PR body per changed package, plus a packages.json manifest, " @@ -743,6 +831,21 @@ def main(): _self_test() return + if args.python_versions: + if not args.packages: + parser.error("--python-versions needs the packages to fill in") + for package in args.packages: + package_data = load_package_yaml(package) + if package_data is None: + sys.exit(f"[X] No {package_yaml(package).relative_to(ROOT_DIR)}") + pypi_info = get_pypi_package_info(package_data.get("package-name", package)) + if pypi_info is None: + sys.exit(f"[X] Could not get PyPI info for {package}") + filled = fill_python_versions(package_yaml(package), pypi_info) + print(f"[+] {package}: python-versions added for " + f"{', '.join(f'v{v}' for v in filled) or 'nothing'}") + return + if args.packages: packages = args.packages print(f"Checking {len(packages)} specified package(s)...") diff --git a/ci_scripts/pending_versions.py b/ci_scripts/pending_versions.py index 89aef8515de7..ac5b5eba8c07 100755 --- a/ci_scripts/pending_versions.py +++ b/ci_scripts/pending_versions.py @@ -13,11 +13,17 @@ pending; with no --glob, those are the versions returned. With --glob, every version matching the glob is returned, released or not, so a workflow_dispatch can force a rebuild. + +With --python-matrix, print instead a matrix `include:` list of +{"version", "python"} pairs, where each version gets the interpreters of its +`python-versions` (the ones upstream builds for, recorded by +check_versions.py) that riscv64 can build, minus --python-exclude. """ import argparse import fnmatch import json +import re import sys from pathlib import Path @@ -25,6 +31,15 @@ DOCS_DIR = Path(__file__).resolve().parent.parent / "docs" / "packages" +# Interpreters cibuildwheel can build for riscv64 (the *-manylinux_riscv64 +# identifiers of its build-platforms.toml; there is no cp313t), oldest first. +# Not a policy list: only a technical limit on what upstream's tags can map to. +# Update it when bumping cibuildwheel. +RISCV64_PYTHONS = [ + "cp39", "cp310", "cp311", "cp312", "cp313", + "cp314", "cp314t", "cp315", "cp315t", +] + def load_versions(package): path = DOCS_DIR / f"{package}.yaml" @@ -48,6 +63,14 @@ def load_versions(package): raise SystemExit( f"{path}: version {version} has one of tag/files but not both" ) + pythons = entry.get("python-versions", []) + if not isinstance(pythons, list) or not all( + isinstance(v, str) and re.fullmatch(r"3\.\d+t?", v) for v in pythons + ): + raise SystemExit( + f"{path}: version {version} python-versions must be a list of " + f"quoted strings like '3.10' or '3.14t', got {pythons!r}" + ) return versions @@ -65,6 +88,60 @@ def select_versions(versions, glob=None): return [str(v["version"]) for v in versions if is_pending(v)] +def cp_tag(python_version): + """'3.14t' -> 'cp314t', the cibuildwheel identifier prefix.""" + return "cp" + python_version.replace(".", "") + + +def adjust(tags, exclude=()): + """Drop the tags matching an exclude glob.""" + return [t for t in tags if not any(fnmatch.fnmatchcase(t, g) for g in exclude)] + + +def python_matrix(package, entries, selected, exclude=()): + matrix = [] + for entry in entries: + version = str(entry["version"]) + if version not in selected: + continue + if "python-versions" not in entry: + raise SystemExit( + f"{package} {version} has no python-versions in docs/packages/{package}.yaml; " + f"run `check_versions.py --python-versions {package}` or write it by hand" + ) + tags = {cp_tag(v) for v in entry["python-versions"]} + tags = adjust([t for t in RISCV64_PYTHONS if t in tags], exclude) + if not tags: + raise SystemExit( + f"{package} {version}: no Python version to build (none of " + f"{entry['python-versions']} can be built for riscv64, or all excluded)" + ) + matrix += [{"version": version, "python": tag} for tag in tags] + return matrix + + +def _self_test(): + entries = [ + {"version": "1.0", "python-versions": ["3.8", "3.12", "3.13t", "3.14", "3.14t"]}, + {"version": "2.0", "python-versions": ["3.13"]}, + {"version": "3.0"}, + ] + # Upstream's list in RISCV64_PYTHONS order, minus what riscv64 can't build. + assert python_matrix("pkg", entries, ["1.0"], ["cp314t"]) == [ + {"version": "1.0", "python": "cp312"}, + {"version": "1.0", "python": "cp314"}, + ] + assert python_matrix("pkg", entries, ["2.0"]) == [{"version": "2.0", "python": "cp313"}] + for selected, exclude in ((["3.0"], []), (["2.0"], ["cp31?"])): + try: + python_matrix("pkg", entries, selected, exclude) + except SystemExit: + pass + else: + raise AssertionError(f"{selected} {exclude} should fail") + print("[+] self-test passed") + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("package", help="docs/packages/.yaml to read") @@ -78,6 +155,20 @@ def main(): action="store_true", help="only validate the YAML; print nothing", ) + parser.add_argument( + "--python-matrix", + action="store_true", + help="print {version, python} matrix entries instead of versions", + ) + parser.add_argument( + "--python-exclude", + default="", + help="space-separated interpreter tag globs (e.g. 'cp39 cp315*') to drop", + ) + parser.add_argument("--self-test", action="store_true", help=argparse.SUPPRESS) + if "--self-test" in sys.argv: + _self_test() + return args = parser.parse_args() versions = load_versions(args.package) @@ -90,6 +181,10 @@ def main(): f"no version of {args.package} matches {args.glob!r}; declared: " + ", ".join(str(v["version"]) for v in versions) ) + if args.python_matrix: + selected = python_matrix( + args.package, versions, selected, args.python_exclude.split() + ) json.dump(selected, sys.stdout, separators=(",", ":")) print() diff --git a/ci_scripts/riscv64_resolve.py b/ci_scripts/riscv64_resolve.py index 30839d724f71..700dbb7d49af 100644 --- a/ci_scripts/riscv64_resolve.py +++ b/ci_scripts/riscv64_resolve.py @@ -102,8 +102,9 @@ def main(): help="a requirements file, repeatable", ) parser.add_argument( - "--abi", action="append", choices=ABIS, - help=f"interpreter to resolve for, repeatable (default: {' '.join(ABIS)})", + "--abi", action="append", + help=f"interpreter to resolve for (e.g. cp310, cp315t), repeatable; pass the " + f"workflow's python-matrix tags (default: {' '.join(ABIS)})", ) parser.add_argument( "--extra-index-url", default=REGISTRY, From 752d67f2e03a6e0583cfca05d8e21c9a735df4a1 Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Tue, 20 Oct 2026 11:57:08 -0400 Subject: [PATCH 2/5] workflows: support dynamic Python build matrices Add support for taking a dynamic list of Python versions as an input when a new workflow parameter, 'python-matrix', is set true, based on the output of scripts in the ci_scripts/ directory. Default to an empty list if nothing is provided. Also support 'python-exclude' for cases when we want to omit versions for various reasons. Modify build-falcon.yml as a sample case. Signed-off-by: Trevor Gamblin --- .github/workflows/_setup.yml | 25 +++++++++++++++++++++++++ .github/workflows/build-falcon.yml | 4 ++-- .github/workflows/nightly.yml | 1 + 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_setup.yml b/.github/workflows/_setup.yml index 51ed19359172..63d5b641121a 100644 --- a/.github/workflows/_setup.yml +++ b/.github/workflows/_setup.yml @@ -15,10 +15,25 @@ on: required: false default: '' type: string + python-matrix: + description: Also compute the matrix output from the python-versions of each version in docs/packages/.yaml. + required: false + default: false + type: boolean + python-exclude: + description: Space-separated interpreter tag globs (e.g. "cp39 cp315*") never to build. + required: false + default: '' + type: string outputs: versions: description: JSON list of versions to build; "[]" when there is nothing to build. value: ${{ jobs.setup.outputs.versions }} + matrix: + description: >- + JSON matrix include list of {version, python} pairs (python being a + cibuildwheel tag like cp314t); "[]" unless python-matrix is set. + value: ${{ jobs.setup.outputs.matrix }} jobs: setup: @@ -26,6 +41,7 @@ jobs: runs-on: ubuntu-latest outputs: versions: ${{ steps.versions.outputs.versions }} + matrix: ${{ steps.versions.outputs.matrix || '[]' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -43,7 +59,16 @@ jobs: env: PACKAGE: ${{ inputs.package }} VERSION_GLOB: ${{ inputs.version }} + PYTHON_MATRIX: ${{ inputs.python-matrix }} + PYTHON_EXCLUDE: ${{ inputs.python-exclude }} run: | versions=$(uv run --quiet ci_scripts/pending_versions.py ${VERSION_GLOB:+--glob "$VERSION_GLOB"} "$PACKAGE") echo "versions=$versions" >> "$GITHUB_OUTPUT" echo "Versions to build: $versions" + if [ "$PYTHON_MATRIX" = "true" ] && [ "$versions" != "[]" ]; then + matrix=$(uv run --quiet ci_scripts/pending_versions.py --python-matrix \ + --python-exclude "$PYTHON_EXCLUDE" \ + ${VERSION_GLOB:+--glob "$VERSION_GLOB"} "$PACKAGE") + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + echo "Matrix: $matrix" + fi diff --git a/.github/workflows/build-falcon.yml b/.github/workflows/build-falcon.yml index 3826e738b2ab..405781d654c1 100644 --- a/.github/workflows/build-falcon.yml +++ b/.github/workflows/build-falcon.yml @@ -40,6 +40,7 @@ jobs: with: package: falcon version: ${{ inputs.version }} + python-matrix: true build_wheels: needs: [setup] @@ -50,8 +51,7 @@ jobs: strategy: fail-fast: false matrix: - version: ${{ fromJSON(needs.setup.outputs.versions) }} - python: ["cp312", "cp313", "cp314", "cp314t"] + include: ${{ fromJSON(needs.setup.outputs.matrix) }} env: FALCON_VERSION: ${{ matrix.version }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 16b499683fc3..1926a7b59679 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -11,6 +11,7 @@ on: paths: - '.github/workflows/nightly.yml' - 'ci_scripts/check_versions.py' + - 'ci_scripts/pending_versions.py' permissions: contents: write From c58a458b082cf81f349d5212b7706f73a28a4d38 Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Tue, 20 Oct 2026 12:02:32 -0400 Subject: [PATCH 3/5] docs/development.md: add notes about dynamic Python matrix Signed-off-by: Trevor Gamblin --- docs/development.md | 71 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 17 deletions(-) diff --git a/docs/development.md b/docs/development.md index 42f08fead8fc..5a24485fdf94 100644 --- a/docs/development.md +++ b/docs/development.md @@ -105,23 +105,54 @@ jobs: ### Target Python Versions -Previously RISE has used a Python version matrix covering the four latest -releases (`major.minor`, e.g. `3.14`), plus any freethreaded variants available -(e.g. `3.14t`). As of July 14th, 2026, this includes Pythons 3.11, 3.12, 3.13, -and 3.14 (along with 3.14t, the freethreaded equivalent). However, the NumPy -project (as of version 2.5.0) supports Python 3.12 as the minimum. Since this -package is fundamental to many others which we are supporting, we will follow -its precedent when defining our version matrix. Some wheels have previously been -built for 3.13t, but since this was an experimental version with limited support -we avoid it now. +RISE builds each release of a package for the Python versions upstream built +that release for, rather than for a fixed list. Each version entry of +`docs/packages/.yaml` records them: -With these factors, our default version matrix becomes: +```yaml +versions: +- version: 1.2.3 + python-versions: ['3.10', '3.11', '3.14', '3.14t'] +``` + +`ci_scripts/check_versions.py` fills this in from the wheels upstream +published on PyPI for that version: -`['3.12', '3.13', '3.14', '3.14t']` +- a `cpXY-cpXY` or `cpXY-cpXYt` wheel adds `3.X` or `3.Xt` (free-threaded); +- a `cpXY-abi3` wheel adds a single `3.X` build, which serves every later + CPython; +- with no version-specific wheels at all (sdist-only or pure-Python + releases), `requires_python` decides, and no free-threaded build is made. + +The nightly job writes it for every version it declares. For a version added +by hand (a new port, or a manual upgrade), run: + +``` +uv run --with-requirements ci_scripts/requirements.txt python3 ci_scripts/check_versions.py --python-versions +``` -Exceptions may be necessary for some packages, which should be carefully -considered to balance achieving similarity to upstream with feasibility of -maintenance. +which adds `python-versions` to every entry of the YAML that lacks it. The +values must be quoted: unquoted, YAML reads `3.10` as the number 3.1. + +A build workflow that sets `python-matrix: true` on its `setup` job gets one +matrix entry per listed version that cibuildwheel can build for riscv64 +(`RISCV64_PYTHONS` in `ci_scripts/pending_versions.py`: currently 3.9 to 3.15 +and 3.14t/3.15t, but not 3.13t). That list is a technical limit, updated +when cibuildwheel is bumped, not a policy choice. A version without +`python-versions` fails the `setup` job. + +To build more or fewer interpreters than upstream for one version, edit its +`python-versions`. To drop interpreters for every version of a package, use +the `python-exclude` input of `setup`: `python-exclude: "cp39 cp31[01]"` +drops interpreters (shell-style globs, on cibuildwheel tags), for example +when a dependency has no riscv64 wheel for them. NumPy 2.5.0 and later only +support Python 3.12+, so dependents of NumPy usually need to exclude older +interpreters. + +Explain any override in a comment next to it. Most workflows still carry a +hard-coded `python: [...]` list from before this scheme (the former default +was `['3.12', '3.13', '3.14', '3.14t']`); each is converted when it is next +updated to build a new release. ### uv @@ -200,6 +231,7 @@ jobs: with: package: numpy version: ${{ inputs.version }} + python-matrix: true build_wheels: needs: [setup] @@ -208,15 +240,20 @@ jobs: strategy: fail-fast: false matrix: - version: ${{ fromJSON(needs.setup.outputs.versions) }} - python: ["cp312", "cp313", "cp314", "cp314t"] + include: ${{ fromJSON(needs.setup.outputs.matrix) }} env: NUMPY_VERSION: ${{ matrix.version }} ... ``` The `if:` guard is required: GitHub Actions rejects a job whose matrix vector is -empty, and `versions` is `[]` whenever nothing is pending. Job outputs are not +empty, and `versions` is `[]` whenever nothing is pending. With +`python-matrix: true`, `_setup.yml` also outputs `matrix`: one +`{version, python}` entry per interpreter in the version's `python-versions` +(see [Target Python Versions](#target-python-versions)) that cibuildwheel +can build for riscv64. The `python-exclude` input drops interpreters for a +package. Workflows that still hard-code `python: [...]` switch to it when +next updated for a new release. Job outputs are not per matrix leg, so a job must not hand its version to another through `outputs:`; use `matrix.version` (or an artifact named after it) instead. From 6fad0cbea1f0c6cd34609c9856d7b8fde6d9333b Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Tue, 20 Oct 2026 12:05:25 -0400 Subject: [PATCH 4/5] skills/python-project-porting: update files to reflect new dynamic matrix logic Provide better instructions based on the changes in ci_scripts/ and .github/workflows, to support dynamic Python matrix generation and situational excludes. Make sure that any automated rework to existing workflows is gradual, i.e. it occurs when a workflow is modified during meaningful work, instead of bulk editing all workflows now. Signed-off-by: Trevor Gamblin --- skills/python-project-porting/SKILL.md | 12 +++- .../references/gotchas-index.md | 2 +- .../gotchas/cibuildwheel-matrix-and-abi3.md | 19 ++++--- .../gotchas/dependencies-and-registry.md | 10 ++-- .../gotchas/feasibility-and-triage.md | 4 +- .../pytest-config-servers-and-selection.md | 5 +- .../gotchas/test-failures-and-flakes.md | 7 ++- .../gotchas/testing-and-shadowing.md | 2 +- .../references/pr-and-publishing.md | 2 +- .../references/workflow-anatomy.md | 57 +++++++++++++++---- 10 files changed, 84 insertions(+), 36 deletions(-) diff --git a/skills/python-project-porting/SKILL.md b/skills/python-project-porting/SKILL.md index 757a1ed008b6..ab5494bef6cf 100644 --- a/skills/python-project-porting/SKILL.md +++ b/skills/python-project-porting/SKILL.md @@ -68,6 +68,12 @@ or a repeated ask. Follow them exactly; the reference files explain the why. `- version:` entry without `tag:`/`files:` is pending; `_setup.yml` turns the pending set into the `matrix.version` every job iterates over, and the publish fills the entry in. The workflow derives the git tag from `matrix.version`, not the other way round. +- **The interpreters come from upstream, never from a fixed list.** Set `python-matrix: true` + on the `setup` job and build from `include: ${{ fromJSON(needs.setup.outputs.matrix) }}`: + each version gets the interpreters in its `python-versions` in the package YAML, which + `check_versions.py` fills from upstream's PyPI wheels. Adjust by editing `python-versions` + or with `python-exclude`, with a "why" comment. Hard-code `python:` only when extra matrix + axes don't fit (see workflow-anatomy.md). - **Default to NO comments in workflows** — they are read as reference. One "why" line only for a genuine non-obvious deviation; never narrate standard steps. - **Never set `CIBW_BUILD_VERBOSITY`** — drop it if you inherit it from a template. @@ -88,8 +94,10 @@ name, repo, version, upstream build docs — come from the invoking prompt): 2. Add `.github/workflows/build-.yml` following the playbook below and [references/workflow-anatomy.md](references/workflow-anatomy.md), **and** `docs/packages/.yaml` with `package-name`, `source-code`, `license` and one - pending `- version: ` entry (no `tag:`/`files:` — the publish adds them). - For an upgrade of an existing package, only the `- version:` line is new. + pending `- version: ` entry (no `tag:`/`files:` — the publish adds them), + then fill its `python-versions` with `uv run --with-requirements ci_scripts/requirements.txt python3 ci_scripts/check_versions.py --python-versions `. + For an upgrade of an existing package, only that version entry is new — plus, if the + workflow still hard-codes `python: [...]`, its conversion to `python-matrix: true`. 3. Validate locally (gotcha 9), then push to `origin` and open a PR. The `pull_request: paths` trigger is what produces the **first** run of a new workflow, and that run is what registers it with GitHub; it builds every pending version of the YAML. diff --git a/skills/python-project-porting/references/gotchas-index.md b/skills/python-project-porting/references/gotchas-index.md index 12cc82f5f639..71eb866dfc1f 100644 --- a/skills/python-project-porting/references/gotchas-index.md +++ b/skills/python-project-porting/references/gotchas-index.md @@ -291,7 +291,7 @@ The porting gotchas (559 of them) live in [`references/gotchas/`](gotchas/), spl - **478** — Gotcha 343's "revisit once `` has a riscv64 build" is wrong for an *archived* dependent: a read-only upstream can never widen its pin, so the port is blocked on a historical version of the dep (TF 2.15, not this repo's 2.21.0), the pin window's own - interpreter coverage can miss the default matrix entirely, CUDA is not the blocker in a TF + interpreter coverage can miss the port's matrix entirely, CUDA is not the blocker in a TF custom-ops package, and a negated-x86 arch allowlist mis-classifies riscv64 into `-mavx` (the tensorflow-addons case). - **480** — A non-NVIDIA accelerator vendor (Huawei Ascend/CANN) can keep its whole toolkit diff --git a/skills/python-project-porting/references/gotchas/cibuildwheel-matrix-and-abi3.md b/skills/python-project-porting/references/gotchas/cibuildwheel-matrix-and-abi3.md index f05e086e670a..6428294396bb 100644 --- a/skills/python-project-porting/references/gotchas/cibuildwheel-matrix-and-abi3.md +++ b/skills/python-project-porting/references/gotchas/cibuildwheel-matrix-and-abi3.md @@ -100,8 +100,10 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/cibuildwheel-matrix-an the cp312 abi3 wheel. Some packages ship **only** the abi3 wheel with no cp314t variant (litellm: upstream publishes `cp310-abi3` only, no free-threaded wheel). In that case the matrix - collapses to a single build; run it on cp312 (our minimum) and test-reuse on - cp313/cp314 via cibuildwheel's `find_compatible_wheel` logic. + collapses to a single build and test-reuse on newer interpreters via cibuildwheel's + `find_compatible_wheel` logic. `check_versions.py` derives this shape from the PyPI + wheel names on its own (`cp310-abi3` becomes `python-versions: ['3.10']`); if the abi3 + floor can't build on riscv64, edit it to a floor that can (e.g. `['3.12']`). - **maturin abi3 is a pyproject/Cargo feature; setuptools-rust abi3 is a build-time *flag* you must inject.** For maturin the abi3 tag comes from `wheel.py-api`/the `pyo3/abi3-pyNN` crate feature — set once, inherited. But @@ -1062,7 +1064,7 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/cibuildwheel-matrix-an stop at 3.12. Upstream's cp310-cp312-only wheel set is the tell, and the matrix has to follow it (`python: ["cp312"]` here) even though cp313/cp314 wheels would build and pass an import check. Exercise the tool's front door — the CLI on a two-line file, not - `import ` — on the newest interpreter of the default matrix before committing to it. + `import ` — on the newest interpreter of the matrix before committing to it. Applies to anything that consumes the bytecode or AST details of the interpreter it runs on: type checkers, bytecode rewriters, coverage/debug tooling. @@ -1115,15 +1117,16 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/cibuildwheel-matrix-an comment is explicit that this is unsupported and untested ("no guarantee this will work and no support will be provided") — setting it to force a `cp314` build substitutes our own untested guess for upstream's own tested ceiling, exactly the divergence goal 2 - (mirror upstream's own recipe) exists to prevent. Trim the matrix instead: - `python: ["cp312", "cp313"]`, deviating from the repo-default four with a one-line - comment citing the guard. + (mirror upstream's own recipe) exists to prevent. Trim the matrix instead + (drop `'3.14'` from `python-versions`, `python-exclude: "cp314*"`, or `python: + ["cp312", "cp313"]` in an unconverted workflow) with a one-line comment citing the + guard. - **Generalizes past cocotb**: any pure-setuptools project that gates its own supported range with a version check in `setup.py` (rather than only via `python_requires`, which pip enforces as a resolver constraint and never touches the build) needs the same treatment — grep the checkout's `setup.py` for `sys.version_info` comparisons against a - literal tuple before assuming the repo-default `cp312/cp313/cp314/cp314t` matrix - applies. + literal tuple before assuming every interpreter `check_versions.py` derives from + `requires_python` applies. - **Verify site-independent**: `python3.14 -c "exec(open('setup.py').read())"` from the checkout root (or just `pip wheel . --no-deps` under a 3.14 interpreter) reproduces the `RuntimeError` on any host, x86 included — no riscv64/QEMU needed to confirm the ceiling diff --git a/skills/python-project-porting/references/gotchas/dependencies-and-registry.md b/skills/python-project-porting/references/gotchas/dependencies-and-registry.md index 7ac354583ed3..99fda10a556e 100644 --- a/skills/python-project-porting/references/gotchas/dependencies-and-registry.md +++ b/skills/python-project-porting/references/gotchas/dependencies-and-registry.md @@ -299,13 +299,13 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/dependencies-and-regis GIL-ful entries keep their full suite: ```yaml matrix: - python: ["cp312", "cp313", "cp314", "cp314t"] - include: - - python: "cp314t" - test_skip: "*" + include: ${{ fromJSON(needs.setup.outputs.matrix) }} ... - CIBW_TEST_SKIP: ${{ matrix.test_skip }} + CIBW_TEST_SKIP: ${{ endsWith(matrix.python, 't') && '*' || '' }} ``` + (A `python-matrix` include list can't take extra per-entry keys, so derive the + value from `matrix.python` in an expression. Unconverted workflows can still use a + `python:` list plus `include: - python: "cp314t"` / `test_skip: "*"`.) An unset matrix key interpolates to the empty string, and cibuildwheel's `test-skip` selector treats `""` as "skip nothing" (same `_resolve_cascade` behaviour gotcha 51 relies on for `before-build`), so the other entries need no second command shape. diff --git a/skills/python-project-porting/references/gotchas/feasibility-and-triage.md b/skills/python-project-porting/references/gotchas/feasibility-and-triage.md index aed830ee46ed..a3e62d8d7cde 100644 --- a/skills/python-project-porting/references/gotchas/feasibility-and-triage.md +++ b/skills/python-project-porting/references/gotchas/feasibility-and-triage.md @@ -3678,8 +3678,8 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/feasibility-and-triage unblock this. **For an archived dependent, resolve the pin to a concrete version before writing "revisit once the dep lands"** — otherwise the note promises an unblocking that the named work cannot deliver. - - **Intersect the pinned dependency window's own *interpreter* coverage with this - repo's default matrix; an empty intersection settles the port on its own.** TF + - **Intersect the pinned dependency window's own *interpreter* coverage with the + matrix the port would build; an empty intersection settles the port on its own.** TF 2.13.1/2.14.0/2.15.0 ship `cp38`-`cp311` wheels and nothing later, while this repo builds cp312/cp313/cp314/cp314t — zero overlap, so no matrix row could be valid even with a riscv64 TF 2.15 in hand. This is a cheap PyPI-JSON check (`{f['filename']}` diff --git a/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md b/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md index 3b4065bf3cdf..3c6d67087135 100644 --- a/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md +++ b/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md @@ -252,7 +252,8 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/pytest-config-servers- - **Mirror upstream's own interpreter cap while you are reading their wheel job.** pyiceberg's sets `CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10,<3.14"` and `CIBW_SKIP: "cp3*t-*"`, and the classifiers stop at 3.13 — so the matrix is - `[cp312, cp313]`, not the repo default. Building `cp314`/`cp314t` there would ship + `[cp312, cp313]` (`check_versions.py` records this in `python-versions` from the PyPI wheel set on + its own). Building `cp314`/`cp314t` there would ship riscv64 an interpreter upstream ships on no platform, which is the goal-2 divergence, not extra coverage. @@ -922,7 +923,7 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/pytest-config-servers- 631. **An old `unittest` suite that calls the deprecated `TestCase` aliases (`failUnless`, `failUnlessEqual`, `failUnlessRaises`, `failIfEqual`, `assertEquals`, …) fails on every - interpreter of the default matrix, because Python 3.12 removed them (the + interpreter from 3.12 on, because Python 3.12 removed them (the ed25519-blake2b-fork case; see `build-ed25519-blake2b-fork.yml`).** The tests are correct and the extension is fine — upstream's own wheel CI simply never runs them (its last test runner was Travis on 3.7), so nobody noticed. The error is diff --git a/skills/python-project-porting/references/gotchas/test-failures-and-flakes.md b/skills/python-project-porting/references/gotchas/test-failures-and-flakes.md index c2870bd847a7..a0921257bd48 100644 --- a/skills/python-project-porting/references/gotchas/test-failures-and-flakes.md +++ b/skills/python-project-porting/references/gotchas/test-failures-and-flakes.md @@ -173,9 +173,10 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/test-failures-and-flak parametrisation — 100 of 100 is a gate, a scattered mix is not. That one command separates "upstream doesn't support this on riscv64" from "our wheel is broken", and it costs nothing next to re-reading tracebacks. - - **Deselect per matrix entry, not globally.** Turn `python: [cp312, ...]` into - `include:` with a per-entry filter and interpolate it into the test command - (`-k "${{ matrix.pytest_k }}"`), so the interpreter that *can* exercise the + - **Deselect per matrix entry, not globally.** Derive a per-entry filter from + `matrix.python` (`-k "${{ matrix.python != 'cp314' && 'not feature' || '' }}"`; in + an unconverted workflow, turn `python: [cp312, ...]` into `include:` entries with a + `pytest_k` key) and interpolate it into the test command, so the interpreter that *can* exercise the feature keeps testing it — dropping it everywhere would have thrown away 105 real tests on cp314. `-k ""` is a valid no-op filter, so the unrestricted entry needs no second command shape. diff --git a/skills/python-project-porting/references/gotchas/testing-and-shadowing.md b/skills/python-project-porting/references/gotchas/testing-and-shadowing.md index 2c962c2818a3..ad1104d56d64 100644 --- a/skills/python-project-porting/references/gotchas/testing-and-shadowing.md +++ b/skills/python-project-porting/references/gotchas/testing-and-shadowing.md @@ -691,7 +691,7 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/testing-and-shadowing. into `{project}` for cibuildwheel to carry into the container (gotcha 7) therefore finds `{project}/.egg-info`, the in-tree build's leftover, even though cibuildwheel's test cwd is a fresh temp dir (the sqlcipher3 case). `cd` doesn't help there. Run it as - `python -P {project}/