From c72849fad9150f8a070fe455eda99752cc29a7df Mon Sep 17 00:00:00 2001 From: chen21019 <19357113+chen21019@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:18:45 +0800 Subject: [PATCH 1/2] docs: record verified cache 5.7.6 publication --- README.md | 18 +++++++++--------- RELEASE-NOTES-5.7.6.md | 19 ++++++++++++++----- SECURITY-MAINTENANCE.md | 4 ++-- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index de6b98766ca4..0c7692c451b2 100755 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ > [!IMPORTANT] > This repository is a narrowly scoped PastureStack fork of Hazelcast 5.7.0. It preserves the upstream Git history, copyright notices, license files, and authorship. PastureStack does not claim ownership of upstream work and is not affiliated with or endorsed by Hazelcast, Inc. -Current source prepares `5.7.6` using the official Temurin `25.0.5+8` +Release `5.7.6` uses the official Temurin `25.0.5+8` security update. CI uses Python `3.14.9`, pip `26.2.2`, and certifi `2026.10.19`. These are build and verification tools; the delivered JAR does not contain a JRE or Python runtime. The optional OCI build retains Ubuntu @@ -12,18 +12,18 @@ The generated Hazelcast cluster version remains `5.7.3`. Earlier public Maven artifacts remain immutable historical evidence. The current public GitHub Release is -[`v5.7.5`](https://github.com/PastureStack/distributed-cache-runtime/releases/tag/v5.7.5). -It publishes the exact reviewed `hazelcast-5.7.5.jar` from source -`a9aea563870201462dc24f778a06279a08ed5841`, with SHA-256 -`0f536a9c7bcd00f2369586fb6ca1606f7e45f3225e24795d10d38397051c8715`. -See [5.7.5 release notes](RELEASE-NOTES-5.7.5.md) for its focused tests, +[`v5.7.6`](https://github.com/PastureStack/distributed-cache-runtime/releases/tag/v5.7.6). +It publishes the exact reviewed `hazelcast-5.7.6.jar` from signed source +`bd071c12bc11808a5f91e8b5d0308decd79e5af9`, with SHA-256 +`825b5fc9526a10549bcb580f2d6c03bf1f5667625661722c0406ea6f796e83f2`. +See [5.7.6 release notes](RELEASE-NOTES-5.7.6.md) for its focused tests, embedded Jackson versions and verification boundaries. The Engine consumes this GitHub release asset; no new OCI/GHCR carrier is published by this release. Source dependency-management floors are LZ4 `1.11.4` for optional Kafka connectors and RabbitMQ Java client `5.37.0` for JMS integration tests. -Both are absent from the published `5.7.5` core JAR, its resolved runtime tree, -and its release SBOM; this source-only maintenance does not require a new core -artifact or downstream Engine release. See the dependency assessments in +Both are absent from the core JAR, its resolved runtime tree, +and release SBOM; they do not drive the JDK rebuild in this release. +See the dependency assessments in [SECURITY-MAINTENANCE.md](SECURITY-MAINTENANCE.md#lz4-source-floor-and-release-impact). Every current and future PastureStack publication uses a pure numeric version, while diff --git a/RELEASE-NOTES-5.7.6.md b/RELEASE-NOTES-5.7.6.md index eb7ef2aef507..a10be68cd4cd 100644 --- a/RELEASE-NOTES-5.7.6.md +++ b/RELEASE-NOTES-5.7.6.md @@ -1,6 +1,6 @@ -# Distributed Cache Runtime 5.7.6 candidate +# Distributed Cache Runtime 5.7.6 -This candidate rebuilds the maintained core JAR with the official Temurin +This release rebuilds the maintained core JAR with the official Temurin 25.0.5+8 security release. Maven artifact coordinates and filenames move to 5.7.6; the cluster protocol and generated runtime version remain 5.7.3. Managed application dependencies and public Java APIs are unchanged. @@ -11,6 +11,15 @@ artifact-carrier build keeps its pinned Ubuntu 24.04 base, installs the matching CA update, and verifies official JDK archives before replacing the old build JDK. The final carrier contains the JAR and license files only. -Publication requires the existing focused regression, artifact identity, -SBOM, and vulnerability gates. Candidate source is not proof of a published -or deployed artifact. Previous release tags and assets remain immutable. +Signed producer `bd071c12bc11808a5f91e8b5d0308decd79e5af9` produced the +23,852,245-byte JAR with SHA-256 +`825b5fc9526a10549bcb580f2d6c03bf1f5667625661722c0406ea6f796e83f2`. +The anonymous release download matches those bytes and embeds the same source. +[CI](https://github.com/PastureStack/distributed-cache-runtime/actions/runs/39949973022) +passed 510 tests in 40 suites with zero failures, errors or skips, artifact +identity checks, and source/JAR/dependency scans with zero Critical/High findings. +The CycloneDX 1.6 SBOM's 15 components and 16 dependency nodes match Maven. +All required CodeQL checks passed before protected PR #18 merged the same tree. +These are the original CI artifacts; no release-time rebuild occurred. +No new OCI carrier, downstream deployment or full historical matrix is implied. +Previous release tags and assets remain immutable. diff --git a/SECURITY-MAINTENANCE.md b/SECURITY-MAINTENANCE.md index 403f04c1462d..97596fd5079c 100644 --- a/SECURITY-MAINTENANCE.md +++ b/SECURITY-MAINTENANCE.md @@ -4,7 +4,7 @@ The PastureStack maintenance delta intentionally preserves the Hazelcast 5.7.0 API while updating reviewed runtime and build-time dependencies with published security fixes: -The current published Maven artifact is `5.7.5`, from signed tag `v5.7.5` and source `a9aea563870201462dc24f778a06279a08ed5841`; it is based on upstream `5.7.0`, not an upstream Hazelcast release. Public `v5.7.4` remains immutable historical evidence. Hazelcast's cluster protocol parser accepts numeric product versions only, so generated runtime metadata deliberately reports `5.7.3`. The artifact version must be used by Maven coordinates, filenames, any artifact carrier, SBOM, and release evidence; the numeric runtime version must be used by cluster/member compatibility code. See [5.7.5 release notes](RELEASE-NOTES-5.7.5.md) for the narrow Jackson update, exact artifact and validation boundaries. +The current published Maven artifact is `5.7.6`, from signed tag `v5.7.6` and source `bd071c12bc11808a5f91e8b5d0308decd79e5af9`; it is based on upstream `5.7.0`, not an upstream Hazelcast release. Public `v5.7.4` and `v5.7.5` remain immutable historical evidence. Hazelcast's cluster protocol parser accepts numeric product versions only, so generated runtime metadata deliberately reports `5.7.3`. The artifact version must be used by Maven coordinates, filenames, any artifact carrier, SBOM, and release evidence; the numeric runtime version must be used by cluster/member compatibility code. See [5.7.6 release notes](RELEASE-NOTES-5.7.6.md) for the patched build toolchains, exact artifact and validation boundaries. | Component | Upstream 5.7.0 | PastureStack maintained version | Scope | | --- | --- | --- | --- | @@ -24,7 +24,7 @@ The current published Maven artifact is `5.7.5`, from signed tag `v5.7.5` and so | Checkstyle | 12.3.1 | 14.0.0 | Build-time source policy | | gRPC Java / Python | 1.73.0 | 1.83.1 / 1.83.0 | gRPC and cloud extension dependencies | | Aircompressor | 2.0.2 | 2.0.3 | Current Parquet 1.18 byte-array codec dependency; Aircompressor v3 uses a different artifact/package and MemorySegment API | -| Hazelcast test-job dependency | 5.0.5 | 5.7.5 | Command-line integration test fixtures | +| Hazelcast test-job dependency | 5.0.5 | 5.7.6 | Command-line integration test fixtures | | Maven Compiler Plugin | implicit | 3.15.0 | Reproducible test-fixture builds | | Maven JAR Plugin | 3.1.2 | 3.5.1 | Reproducible test-fixture packaging | | Maven Install Plugin | implicit | 3.1.4 | Latest stable Maven 3-compatible install lifecycle | From 72b8a13213ec56eb26bd09629666630289e10273 Mon Sep 17 00:00:00 2001 From: chen21019 <19357113+chen21019@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:59:03 +0800 Subject: [PATCH 2/2] fix(ci): isolate patched pip verifier toolchain --- .github/workflows/codeql-verification.yml | 3 +- .github/workflows/security-release-gate.yml | 3 +- README.md | 7 +- scripts/ci-python-requirements.lock | 4 +- scripts/setup-ci-python-tools | 15 ++ .../pip-security-overlay/build-pip-overlay.py | 197 ++++++++++++++++++ tools/pip-security-overlay/inputs.json | 188 +++++++++++++++++ .../pip-security-overlay/test-pip-overlay.py | 143 +++++++++++++ 8 files changed, 552 insertions(+), 8 deletions(-) create mode 100644 scripts/setup-ci-python-tools create mode 100644 tools/pip-security-overlay/build-pip-overlay.py create mode 100644 tools/pip-security-overlay/inputs.json create mode 100644 tools/pip-security-overlay/test-pip-overlay.py diff --git a/.github/workflows/codeql-verification.yml b/.github/workflows/codeql-verification.yml index 62a663341c74..0afedadda5e8 100644 --- a/.github/workflows/codeql-verification.yml +++ b/.github/workflows/codeql-verification.yml @@ -50,8 +50,7 @@ jobs: - name: Verify Python packaging and CA tools run: | set -euo pipefail - python -m pip install --disable-pip-version-check --require-hashes --only-binary=:all: -r scripts/ci-python-requirements.lock - python -c "import sys, pip, certifi, pip._vendor.certifi as bundled; assert sys.version_info[:3] == (3,14,9); assert pip.__version__ == '26.2.2'; assert certifi.__version__ == '2026.10.19'; assert bundled.__version__ == '2026.10.19'; print(sys.version, pip.__version__, certifi.__version__, bundled.__version__)" + bash scripts/setup-ci-python-tools - name: Verify source and wrapper integrity shell: bash diff --git a/.github/workflows/security-release-gate.yml b/.github/workflows/security-release-gate.yml index a441fcbcb54b..4ab974a7536d 100644 --- a/.github/workflows/security-release-gate.yml +++ b/.github/workflows/security-release-gate.yml @@ -64,8 +64,7 @@ jobs: - name: Verify Python packaging and CA tools run: | set -euo pipefail - python -m pip install --disable-pip-version-check --require-hashes --only-binary=:all: -r scripts/ci-python-requirements.lock - python -c "import sys, pip, certifi, pip._vendor.certifi as bundled; assert sys.version_info[:3] == (3,14,9); assert pip.__version__ == '26.2.2'; assert certifi.__version__ == '2026.10.19'; assert bundled.__version__ == '2026.10.19'; print(sys.version, pip.__version__, certifi.__version__, bundled.__version__)" + bash scripts/setup-ci-python-tools - name: Verify candidate identity and supply-chain pins id: candidate diff --git a/README.md b/README.md index 0c7692c451b2..10aa34cbaff8 100755 --- a/README.md +++ b/README.md @@ -4,8 +4,11 @@ > This repository is a narrowly scoped PastureStack fork of Hazelcast 5.7.0. It preserves the upstream Git history, copyright notices, license files, and authorship. PastureStack does not claim ownership of upstream work and is not affiliated with or endorsed by Hazelcast, Inc. Release `5.7.6` uses the official Temurin `25.0.5+8` -security update. CI uses Python `3.14.9`, pip `26.2.2`, and certifi -`2026.10.19`. These are build and verification tools; the delivered JAR does +security update. CI uses Python `3.14.9`, the internal pip +`26.2.2+pasturestack.1` rebuild with patched urllib3 `2.8.0`, and certifi +`2026.10.19`. A frozen upstream-vendoring helper verifies input/wheel hashes +and streaming/TLS behavior in an isolated CI environment. This CI-only update +does not replace the existing release artifact. These are verification tools; the delivered JAR does not contain a JRE or Python runtime. The optional OCI build retains Ubuntu 24.04 and installs its matching CA security update before the verified JDK. The generated Hazelcast cluster version remains `5.7.3`. diff --git a/scripts/ci-python-requirements.lock b/scripts/ci-python-requirements.lock index 92ab2225a4ed..93d55061546a 100644 --- a/scripts/ci-python-requirements.lock +++ b/scripts/ci-python-requirements.lock @@ -1,5 +1,5 @@ # CI verification tools; these packages are not included in product artifacts. -pip==26.2.2 \ - --hash=sha256:fd7f4dfa8190e014013b9f08f3e973f4e4c26ffd179a204895533a7254813a70 +pip==26.2.2+pasturestack.1 \ + --hash=sha256:be5f63d537bbe611dc422fabb6740a44b5997888b592888df5091727f023802a certifi==2026.10.19 \ --hash=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b diff --git a/scripts/setup-ci-python-tools b/scripts/setup-ci-python-tools new file mode 100644 index 000000000000..af6400f0aec6 --- /dev/null +++ b/scripts/setup-ci-python-tools @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Install only CI verification tools; shipped browser/JAR artifacts are unchanged. +set -Eeuo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work=$(mktemp -d "${RUNNER_TEMP:-/tmp}/pasturestack-ci-python.XXXXXX") +python "$root/tools/pip-security-overlay/build-pip-overlay.py" --work "$work/build" --output "$work/dist" +python -m venv --without-pip "$work/env" +export PATH="$work/env/bin:$PATH" +PYTHONPATH="$work/dist/pip-26.2.2+pasturestack.1-py3-none-any.whl" \ + python -m pip install --no-index --no-deps "$work/dist/pip-26.2.2+pasturestack.1-py3-none-any.whl" +python -m pip install --disable-pip-version-check --require-hashes --only-binary=:all: \ + --find-links "$work/dist" -r "$root/scripts/ci-python-requirements.lock" +python -c "import sys,pip,certifi,pip._vendor.certifi as bundled; assert sys.version_info[:3]==(3,14,9); assert pip.__version__=='26.2.2+pasturestack.1'; assert certifi.__version__==bundled.__version__=='2026.10.19'; print(sys.version,pip.__version__,certifi.__version__,bundled.__version__)" +python "$root/tools/pip-security-overlay/test-pip-overlay.py" +printf '%s\n' "$work/env/bin" >>"${GITHUB_PATH:?GitHub Actions path file is required}" diff --git a/tools/pip-security-overlay/build-pip-overlay.py b/tools/pip-security-overlay/build-pip-overlay.py new file mode 100644 index 000000000000..88b53806c998 --- /dev/null +++ b/tools/pip-security-overlay/build-pip-overlay.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Rebuild pip using released urllib3 and pip's unmodified vendoring machinery. + +Build dependencies never enter the product wheel. All network inputs are HTTPS +and SHA-256 pinned. The resulting local version is not an official pip release. +""" +import argparse +import base64 +import csv +import hashlib +import io +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tarfile +import tempfile +from urllib.request import HTTPRedirectHandler, HTTPSHandler, build_opener +from urllib.parse import urlsplit +import venv +import zipfile + + +class HTTPSOnlyRedirect(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + if urlsplit(newurl).scheme != "https": + raise ValueError("Refusing a non-HTTPS build-input redirect") + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def run(command, cwd, env): + print("+", " ".join(map(str, command)), flush=True) + subprocess.run(list(map(str, command)), cwd=cwd, env=env, check=True) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--work", type=Path, required=True) + parser.add_argument("--input-cache", type=Path) + parser.add_argument("--offline", action="store_true") + args = parser.parse_args() + output, work = args.output.resolve(), args.work.resolve() + output.mkdir(parents=True, exist_ok=True) + work.mkdir(parents=True, exist_ok=True) + config = json.loads(Path(__file__).with_name("inputs.json").read_text()) + assert sys.version_info[:2] in ((3, 12), (3, 14)) and os.name == "posix" + scratch = Path(tempfile.mkdtemp(prefix="pip-revendor-", dir=work)) + downloads = scratch / "downloads" + downloads.mkdir() + cache = args.input_cache.resolve() if args.input_cache else downloads + cache.mkdir(parents=True, exist_ok=True) + paths = {} + for item in config["artifacts"]: + assert urlsplit(item["url"]).scheme == "https" + target = downloads / Path(urlsplit(item["url"]).path).name + cached = cache / target.name + if not cached.is_file(): + assert not args.offline, ("Offline input is missing", target.name) + opener = build_opener(HTTPSHandler(), HTTPSOnlyRedirect()) + with opener.open(item["url"], timeout=90) as response, cached.open("wb") as dest: + shutil.copyfileobj(response, dest) + assert sha(cached) == item["sha256"], ("cached input hash mismatch", target.name) + if cached != target: + shutil.copy2(cached, target) + assert sha(target) == item["sha256"], ("input hash mismatch", target.name) + paths.setdefault(item["name"], []).append(target) + + environment = os.environ.copy() + environment.update(PIP_NO_INDEX="1", PIP_FIND_LINKS=str(downloads), + PIP_DISABLE_PIP_VERSION_CHECK="1", PIP_ONLY_BINARY=":all:", + PYTHONHASHSEED="0", SOURCE_DATE_EPOCH=str(config["sourceDateEpoch"])) + for key in ("PYTHONPATH", "PYTHONHOME", "PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", + "PIP_TRUSTED_HOST", "PIP_CERT", "PIP_CLIENT_CERT"): + environment.pop(key, None) + environment["PIP_CONFIG_FILE"] = os.devnull + tool_env = scratch / "tool-env" + venv.EnvBuilder(with_pip=False).create(tool_env) + python = tool_env / "bin/python" + requirements = scratch / "tools.lock" + groups = {} + for item in config["artifacts"]: + if item["kind"] == "tool": + groups.setdefault((item["name"], item["version"]), []).append(item["sha256"]) + requirements.write_text("\n".join( + f"{name}=={version} " + " ".join(f"--hash=sha256:{value}" for value in hashes) + for (name, version), hashes in sorted(groups.items())) + "\n") + bootstrap_env = dict(environment, PYTHONPATH=str(paths["pip"][0])) + run([python, "-m", "pip", "install", "--require-hashes", "--no-compile", "-r", requirements], scratch, bootstrap_env) + environment["PATH"] = str(tool_env / "bin") + os.pathsep + environment["PATH"] + + source_parent = scratch / "source" + source_parent.mkdir() + with tarfile.open(paths["pip-source"][0]) as archive: + # Reject links and traversal before extracting this fixed official archive. + for member in archive.getmembers(): + assert not member.issym() and not member.islnk() + resolved = (source_parent / member.name).resolve() + assert resolved.is_relative_to(source_parent) + archive.extractall(source_parent, filter="data") + source = next(source_parent.iterdir()) + vendor = source / "src/pip/_vendor" + original = {str(p.relative_to(vendor)): sha(p) for p in vendor.rglob("*") if p.is_file()} + assert (source / "src/pip/__init__.py").read_text().count('__version__ = "26.2.2"') == 1 + + # A scoped project invokes the same upstream tool, namespace and urllib3 + # patches. Unrelated vendored libraries are never regenerated or replaced. + scoped = scratch / "urllib3-vendoring" + scoped_vendor = scoped / "src/pip/_vendor" + scoped_vendor.mkdir(parents=True) + (scoped_vendor / "vendor.txt").write_text("urllib3==2.8.0\n") + patches = scoped / "tools/vendoring/patches" + patches.mkdir(parents=True) + for patch in (source / "tools/vendoring/patches").glob("urllib3*.patch"): + shutil.copy2(patch, patches / patch.name) + assert len(list(patches.glob("*.patch"))) == 3 + (scoped / "pyproject.toml").write_text('''[tool.vendoring] +destination = "src/pip/_vendor/" +requirements = "src/pip/_vendor/vendor.txt" +namespace = "pip._vendor" +protected-files = ["vendor.txt"] +patches-dir = "tools/vendoring/patches" +[tool.vendoring.transformations] +drop = ["*.dist-info/", "bin/"] +''') + run([python, "-m", "vendoring", "sync", "-v"], scoped, environment) + generated = scoped_vendor / "urllib3" + assert (generated / "__init__.py").is_file() + shutil.rmtree(vendor / "urllib3") # exact disposable source subtree only + shutil.copytree(generated, vendor / "urllib3") + vendor_txt = vendor / "vendor.txt" + text = vendor_txt.read_text() + assert text.count("urllib3==2.7.0") == 1 + vendor_txt.write_text(text.replace("urllib3==2.7.0", "urllib3==2.8.0")) + version_file = source / "src/pip/__init__.py" + version_file.write_text(version_file.read_text().replace('__version__ = "26.2.2"', + f'__version__ = "{config["version"]}"')) + readme = vendor / "README.rst" + with readme.open("a") as handle: + handle.write("\n\nPastureStack security rebuild\n============================\n\n" + "This downstream local-version wheel uses released urllib3 2.8.0\n" + "through upstream vendoring 1.4.0 and pip's unchanged urllib3\n" + "patches. Other vendored library sources and licenses are unchanged.\n" + "It is not an official PyPA pip release.\n") + run([python, "-c", "from pathlib import Path; from vendoring.sbom import create_sbom_file; create_sbom_file('pip._vendor', Path('src/pip/_vendor/vendor.txt'), Path('src/pip/_vendor/bom.cdx.json'))"], source, environment) + current = {str(p.relative_to(vendor)): sha(p) for p in vendor.rglob("*") if p.is_file()} + allowed = {"vendor.txt", "README.rst", "bom.cdx.json"} + assert {p: value for p, value in original.items() if not p.startswith("urllib3/") and p not in allowed} == { + p: value for p, value in current.items() if not p.startswith("urllib3/") and p not in allowed} + wheels = [] + for suffix in ("first", "second"): + directory = scratch / suffix + directory.mkdir() + run([python, "-c", f"from flit_core.buildapi import build_wheel; build_wheel({str(directory)!r})"], source, environment) + wheels.append(next(directory.glob("*.whl"))) + assert sha(wheels[0]) == sha(wheels[1]), "wheel rebuild is not byte reproducible" + assert sha(wheels[0]) == config["expectedWheelSha256"], "unexpected rebuilt product bytes" + wheel = output / wheels[0].name + shutil.copy2(wheels[0], wheel) + licenses = [] + with zipfile.ZipFile(wheel) as archive: + names = archive.namelist() + record = next(name for name in names if name.endswith(".dist-info/RECORD")) + for name, digest, size in csv.reader(io.StringIO(archive.read(record).decode())): + if not digest: + assert name == record + continue + data = archive.read(name) + expected = "sha256=" + base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() + assert digest == expected and int(size) == len(data), name + assert not any(name.endswith((".so", ".pyd")) for name in names) + licenses = [name for name in names if "LICENSE" in name or "COPYING" in name] + assert any("urllib3" in name for name in licenses) + assert "pip/_vendor/pkg_resources/__init__.py" in names + assert not any("setuptools/package_index" in name for name in names) + shutil.copy2(vendor / "bom.cdx.json", output / "pip-vendor.cdx.json") + report = {"version": config["version"], "upstreamVersion": config["upstreamVersion"], + "upstreamCommit": config["sourceCommit"], "wheel": wheel.name, + "sha256": sha(wheel), "twoBuildsByteIdentical": True, + "allRecordHashesVerified": True, "nativeExtensionsPresent": False, + "licenses": licenses, "onlyVendoredLibraryChanged": "urllib3", "urllib3": "2.8.0"} + (output / "build-evidence.json").write_text(json.dumps(report, indent=2) + "\n") + shutil.copy2(requirements, output / "tools.lock") + shutil.copy2(Path(__file__).with_name("inputs.json"), output / "inputs.json") + print(json.dumps(report), flush=True) + + +if __name__ == "__main__": + if sys.flags.optimize: + raise RuntimeError("Security build verification must not run with Python optimization") + main() diff --git a/tools/pip-security-overlay/inputs.json b/tools/pip-security-overlay/inputs.json new file mode 100644 index 000000000000..fc6b7de90ee3 --- /dev/null +++ b/tools/pip-security-overlay/inputs.json @@ -0,0 +1,188 @@ +{ + "schemaVersion": 1, + "upstreamVersion": "26.2.2", + "version": "26.2.2+pasturestack.1", + "expectedWheelSha256": "be5f63d537bbe611dc422fabb6740a44b5997888b592888df5091727f023802a", + "sourceCommit": "b3b696fe2122f2339d782517559bfade2cb6b879", + "sourceDateEpoch": 1792509744, + "updatedVendoredLibraries": { + "urllib3": "2.8.0" + }, + "artifacts": [ + { + "kind": "tool", + "name": "vendoring", + "version": "1.4.0", + "url": "https://files.pythonhosted.org/packages/e3/ba/2fe43514bcf426888101c2968f3c821c6338afc122ccbbde6e29ee2a5036/vendoring-1.4.0-py3-none-any.whl", + "sha256": "3a5b5c44bdec2683a94cd4c7c25bad556b38539cf316defca65948d76107eb30" + }, + { + "kind": "tool", + "name": "flit_core", + "version": "3.12.0", + "url": "https://files.pythonhosted.org/packages/f2/65/b6ba90634c984a4fcc02c7e3afe523fef500c4980fec67cc27536ee50acf/flit_core-3.12.0-py3-none-any.whl", + "sha256": "e7a0304069ea895172e3c7bb703292e992c5d1555dd1233ab7b5621b5b69e62c" + }, + { + "kind": "tool", + "name": "click", + "version": "8.5.0", + "url": "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", + "sha256": "255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360" + }, + { + "kind": "tool", + "name": "jsonschema", + "version": "4.26.0", + "url": "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", + "sha256": "d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce" + }, + { + "kind": "tool", + "name": "packaging", + "version": "26.3", + "url": "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", + "sha256": "d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c" + }, + { + "kind": "tool", + "name": "requests", + "version": "2.34.2", + "url": "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", + "sha256": "2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0" + }, + { + "kind": "tool", + "name": "rich", + "version": "15.0.0", + "url": "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", + "sha256": "33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb" + }, + { + "kind": "tool", + "name": "attrs", + "version": "26.1.0", + "url": "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", + "sha256": "c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309" + }, + { + "kind": "tool", + "name": "idna", + "version": "3.20", + "url": "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", + "sha256": "ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c" + }, + { + "kind": "tool", + "name": "jsonschema-specifications", + "version": "2025.9.1", + "url": "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", + "sha256": "98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe" + }, + { + "kind": "tool", + "name": "markdown-it-py", + "version": "4.2.0", + "url": "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", + "sha256": "9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a" + }, + { + "kind": "tool", + "name": "Pygments", + "version": "2.21.0", + "url": "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", + "sha256": "2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9" + }, + { + "kind": "tool", + "name": "referencing", + "version": "0.37.0", + "url": "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", + "sha256": "381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231" + }, + { + "kind": "tool", + "name": "urllib3", + "version": "2.8.0", + "url": "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", + "sha256": "0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3" + }, + { + "kind": "tool", + "name": "mdurl", + "version": "0.1.2", + "url": "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", + "sha256": "84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8" + }, + { + "kind": "tool", + "name": "typing_extensions", + "version": "4.16.0", + "url": "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", + "sha256": "481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8" + }, + { + "kind": "tool", + "name": "tomli", + "version": "2.5.0", + "url": "https://files.pythonhosted.org/packages/60/3f/3e3f8fd0919249b0200c80fbc4f9a1e70be19f9883da71dfb7f8b9ab8aca/tomli-2.5.0-py3-none-any.whl", + "sha256": "32a7b79ac57a2e83670ce329ccf675798bc5a2094783a63676866b70503f2e2b" + }, + { + "kind": "tool", + "name": "charset-normalizer", + "version": "3.5.2", + "url": "https://files.pythonhosted.org/packages/fc/ad/d07d7862a62ffa6d79d68074d14823243dd235a77c45262acbf6adeb28bf/charset_normalizer-3.5.2-py3-none-any.whl", + "sha256": "b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685" + }, + { + "kind": "tool", + "name": "certifi", + "version": "2026.10.19", + "url": "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", + "sha256": "42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b" + }, + { + "kind": "tool", + "name": "rpds-py", + "version": "2026.9.1", + "url": "https://files.pythonhosted.org/packages/42/ff/bf7d54f362748fd6a49277b9d6531c394074110fedf50ad791ec59133fbb/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "sha256": "2693b2728bbcc48d09a981a356954b0c47c53ff25b545856f28a889ea619f69a" + }, + { + "kind": "tool", + "name": "rpds-py", + "version": "2026.9.1", + "url": "https://files.pythonhosted.org/packages/1d/6d/b26eb1e75395925b3a142ed351cbed2ec8212f5c9d937aee3df32c701baa/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "sha256": "addeda51556dac7c1a2f14cda62db8b621cd12afba3091d03a96c72932387eab" + }, + { + "kind": "tool", + "name": "rpds-py", + "version": "2026.9.1", + "url": "https://files.pythonhosted.org/packages/72/e2/5873bc4643c250db9e05d48dc0c93763d4aa68bc3b81164cb1af3b45b284/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "sha256": "07deecbfce94c78473018bc7d10b337cc651d12df87a1eb2cb3e4024bc9c33d0" + }, + { + "kind": "tool", + "name": "rpds-py", + "version": "2026.9.1", + "url": "https://files.pythonhosted.org/packages/22/9d/d08a1128ab199b2f0cf25bfeb0639bd05119fff4b7c47bec24ef9a8ec23f/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "sha256": "eac2f5dbafd585dfe31f86a23ebf0d3ba480a9d49ebc87947267b5608d4ea0cd" + }, + { + "kind": "source", + "name": "pip-source", + "version": "26.2.2", + "url": "https://github.com/pypa/pip/archive/b3b696fe2122f2339d782517559bfade2cb6b879.tar.gz", + "sha256": "5202e6f34650ed74b8cfc74a4808af45a46ebd994b187e160a92d426e1bb3e0f" + }, + { + "kind": "tool", + "name": "pip", + "version": "26.2.2", + "url": "https://files.pythonhosted.org/packages/3f/06/3e4420302363fbb71be3c7f35ce042f54bfca330e3e66b3a2e3c4cca579f/pip-26.2.2-py3-none-any.whl", + "sha256": "fd7f4dfa8190e014013b9f08f3e973f4e4c26ffd179a204895533a7254813a70" + } + ] +} diff --git a/tools/pip-security-overlay/test-pip-overlay.py b/tools/pip-security-overlay/test-pip-overlay.py new file mode 100644 index 000000000000..a2c626511361 --- /dev/null +++ b/tools/pip-security-overlay/test-pip-overlay.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Focused streaming regression, CLI and TLS controls for the installed wheel.""" +import argparse +import contextlib +import http.client +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import io +import json +from pathlib import Path +import ssl +import subprocess +import sys +import tempfile +import threading + +import pip +from pip._internal.network.utils import response_chunks +from pip._vendor import msgpack, requests, urllib3 +from pip._vendor.urllib3.exceptions import ProtocolError + + +class TrackedBody(io.BytesIO): + largest_readline = 0 + def readline(self, size=-1): + data = super().readline(size) + self.largest_readline = max(self.largest_readline, len(data)) + return data + + +class Socket: + def __init__(self, body): + self.body = body + def makefile(self, *_): + return self.body + + +def streamed(body): + wire = TrackedBody(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n" + body) + origin = http.client.HTTPResponse(Socket(wire), method="GET") + origin.begin() + response = requests.Response() + response.status_code = 200 + response.raw = urllib3.response.HTTPResponse( + body=origin, original_response=origin, headers=dict(origin.headers.items()), + preload_content=False) + return response, wire + + +def boundary_tests(expect_fixed): + outcome = [] + for label, body in ( + ("chunk-size", b"f" * 131072), + ("chunk-trailer", b"3\r\nfoo\r\n0\r\n" + b"X" * 131072), + ): + response, wire = streamed(body) + error_description = "no exception" + try: + list(response_chunks(response)) + except Exception as error: + error_description = repr(error) + if expect_fixed: + assert isinstance(error, ProtocolError), repr(error) + assert "exceeded maximum allowed length" in str(error), str(error) + if expect_fixed: + assert error_description != "no exception", "malicious chunk line was accepted" + assert wire.largest_readline <= 65537, wire.largest_readline + else: + assert wire.largest_readline == 131072, (wire.largest_readline, error_description) + outcome.append({"path": label, "max_readline": wire.largest_readline}) + for body in (b"3\r\nfoo\r\n0\r\n\r\n", b"3;" + b"x" * 65532 + b"\r\nfoo\r\n0\r\n\r\n"): + response, _ = streamed(body) + assert b"".join(response_chunks(response)) == b"foo" + return outcome + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--expect-vulnerable", action="store_true") + args = parser.parse_args() + result = {"pip": pip.__version__, "urllib3": urllib3.__version__, + "streaming": boundary_tests(not args.expect_vulnerable)} + if args.expect_vulnerable: + print(json.dumps(result)); return + assert pip.__version__ == "26.2.2+pasturestack.1" + assert urllib3.__version__ == "2.8.0" + assert msgpack.Unpacker.__module__ == "pip._vendor.msgpack.fallback" + import importlib.util + assert importlib.util.find_spec("pip._vendor.msgpack._cmsgpack") is None + unpacker = msgpack.Unpacker() + unpacker.feed(b"\x83\x73\xc4\x00") + try: + unpacker.unpack() + except (ValueError, msgpack.OutOfData): + pass + try: + unpacker.skip() + except (ValueError, msgpack.OutOfData): + pass + with tempfile.TemporaryDirectory(prefix="pip-tls-") as name: + root = Path(name) + cert, key = root / "cert.pem", root / "key.pem" + subprocess.run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", + "-days", "1", "-subj", "/CN=localhost", "-addext", + "subjectAltName=DNS:localhost", "-keyout", str(key), "-out", str(cert)], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = b"trusted fixture" + self.send_response(200); self.send_header("Content-Length", str(len(body))) + self.end_headers(); self.wfile.write(body) + def log_message(self, *_): + pass + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.minimum_version = ssl.TLSVersion.TLSv1_2 + context.load_cert_chain(cert, key) + server.socket = context.wrap_socket(server.socket, server_side=True) + thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start() + try: + port = server.server_address[1] + session = requests.Session(); session.trust_env = False + assert session.get(f"https://localhost:{port}/", verify=str(cert), timeout=5).status_code == 200 + for url, verify in ((f"https://localhost:{port}/", True), + (f"https://127.0.0.1:{port}/", str(cert))): + try: + session.get(url, verify=verify, timeout=5) + except requests.exceptions.SSLError: + pass + else: + raise AssertionError("untrusted or wrong-host TLS was accepted") + finally: + server.shutdown(); server.server_close(); thread.join() + subprocess.run([sys.executable, "-m", "pip", "--version"], check=True) + subprocess.run([sys.executable, "-m", "pip", "check"], check=True) + result.update(tls_trusted="accepted", tls_untrusted="rejected", tls_wrong_host="rejected", + msgpack_implementation=msgpack.Unpacker.__module__, cli="passed") + print(json.dumps(result)) + + +if __name__ == "__main__": + if sys.flags.optimize: + raise RuntimeError("Security regression tests must not run with Python optimization") + main()