diff --git a/.github/workflows/codeql-verification.yml b/.github/workflows/codeql-verification.yml index 8fa27463..0baf1c9c 100644 --- a/.github/workflows/codeql-verification.yml +++ b/.github/workflows/codeql-verification.yml @@ -69,7 +69,7 @@ jobs: shell: bash run: | set -euo pipefail - test "$(python3 --version)" = 'Python 3.14.7' + test "$(python3 --version)" = 'Python 3.14.9' PYTHONDONTWRITEBYTECODE=1 python3 -m compileall -q tests/integration - name: Initialize CodeQL diff --git a/.github/workflows/security-release-gate.yml b/.github/workflows/security-release-gate.yml index 4d43a7a7..357d96a8 100644 --- a/.github/workflows/security-release-gate.yml +++ b/.github/workflows/security-release-gate.yml @@ -21,7 +21,7 @@ jobs: env: DAPPER_IMAGE: pasturestack/compose-cli-dapper:${{ github.sha }} TRIVY_IMAGE: aquasec/trivy:0.74.0@sha256:62b1e65e8869bc4b4c6aa4fa2b21595256c7c2f6018a9d9ad61caf87187c1969 - VERSION_OVERRIDE: 0.14.38 + VERSION_OVERRIDE: 0.14.39 PLATFORM_COMPAT_JAR_URL: https://github.com/PastureStack/orchestration-engine/releases/download/v0.183.281/orchestration-engine-0.183.281.jar PLATFORM_COMPAT_JAR_SHA256: da2a8a51562ed16e296f7e29e99482bb44042ff0834cca679bbe01d951ba1682 @@ -69,7 +69,7 @@ jobs: } run_ci - artifact="dist/artifacts/compose-executor-0.14.38-linux-amd64.gz" + artifact="dist/artifacts/compose-executor-0.14.39-linux-amd64.gz" test -s "$artifact" cp "$artifact" /tmp/compose-executor-first.gz @@ -81,7 +81,7 @@ jobs: mkdir -p evidence/product gzip -cd "$artifact" > evidence/product/compose-executor chmod +x evidence/product/compose-executor - evidence/product/compose-executor --version | grep -F '0.14.38' >/dev/null + evidence/product/compose-executor --version | grep -F '0.14.39' >/dev/null cp "$artifact" evidence/ ( cd evidence @@ -149,15 +149,23 @@ jobs: -v "$PWD:/work" -w /work \ -v "$trivy_cache:/root/.cache/trivy" \ "$TRIVY_IMAGE" image \ - --scanners vuln,secret --format json \ + --scanners vuln,secret --list-all-pkgs --format json \ --output /work/evidence/dapper-image-raw.json "$DAPPER_IMAGE" + docker run --rm --entrypoint python3 "$DAPPER_IMAGE" \ + /opt/python-security/scripts/check-pip-vendor-slice.py \ + > evidence/pip-vendor-proof.json + python3 scripts/pip-security/test-pip-vex.py + python3 scripts/pip-security/filter-pip-vendor-findings.py \ + --raw evidence/dapper-image-raw.json --proof evidence/pip-vendor-proof.json \ + --output evidence/dapper-image-applicable.json --vex evidence/pip-vendor.openvex.json \ + --subject "$(docker image inspect --format '{{.Id}}' "$DAPPER_IMAGE")" jq -r ' .Results[]?.Vulnerabilities[]? | select(.Severity == "CRITICAL" or .Severity == "HIGH") | [.VulnerabilityID, .PkgIdentifier.PURL] | @tsv - ' evidence/dapper-image-raw.json | LC_ALL=C sort -u \ + ' evidence/dapper-image-applicable.json | LC_ALL=C sort -u \ > /tmp/dapper-critical-high.tsv grep -F $'build\tCGO_ENABLED=0' evidence/product-go-version.txt >/dev/null if grep -Eq '^linux-(image|modules)([-:]|[[:space:]])' evidence/dapper-dpkg.tsv; then @@ -168,10 +176,10 @@ jobs: .Results[]?.Vulnerabilities[]? | select(.Severity == "CRITICAL" or .Severity == "HIGH") | select(.PkgName != "linux-libc-dev" or ((.FixedVersion // "") != "")) - ] | length' evidence/dapper-image-raw.json)" + ] | length' evidence/dapper-image-applicable.json)" test "$dapper_unreviewed" -eq 0 jq -n \ - --slurpfile report evidence/dapper-image-raw.json \ + --slurpfile report evidence/dapper-image-applicable.json \ --arg author 'PastureStack Security' \ --arg timestamp "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ --arg impact 'linux-libc-dev supplies user-space kernel API headers only in the disposable build and race-test image. The package is not copied into the CGO-disabled product binary, so the vulnerable kernel implementation is not shipped.' \ @@ -238,11 +246,11 @@ jobs: product_critical=$(count evidence/product-security.json '[.Results[]?.Vulnerabilities[]? | select(.Severity == "CRITICAL")] | length') product_high=$(count evidence/product-security.json '[.Results[]?.Vulnerabilities[]? | select(.Severity == "HIGH")] | length') product_applicable=$(count evidence/product-security.json '[.Results[]?.Vulnerabilities[]? | select(.Status != "not_affected")] | length') - dapper_raw_critical=$(count evidence/dapper-image-raw.json '[.Results[]?.Vulnerabilities[]? | select(.Severity == "CRITICAL")] | length') - dapper_raw_high=$(count evidence/dapper-image-raw.json '[.Results[]?.Vulnerabilities[]? | select(.Severity == "HIGH")] | length') + dapper_raw_critical=$(count evidence/dapper-image-applicable.json '[.Results[]?.Vulnerabilities[]? | select(.Severity == "CRITICAL")] | length') + dapper_raw_high=$(count evidence/dapper-image-applicable.json '[.Results[]?.Vulnerabilities[]? | select(.Severity == "HIGH")] | length') dapper_reviewed=$(count evidence/dapper.openvex.json '.statements | length') - dapper_unreviewed=$(count evidence/dapper-image-raw.json '[.Results[]?.Vulnerabilities[]? | select(.Severity == "CRITICAL" or .Severity == "HIGH") | select(.PkgName != "linux-libc-dev" or ((.FixedVersion // "") != ""))] | length') - dapper_secrets=$(count evidence/dapper-image-raw.json '[.Results[]?.Secrets[]?] | length') + dapper_unreviewed=$(count evidence/dapper-image-applicable.json '[.Results[]?.Vulnerabilities[]? | select(.Severity == "CRITICAL" or .Severity == "HIGH") | select(.PkgName != "linux-libc-dev" or ((.FixedVersion // "") != ""))] | length') + dapper_secrets=$(count evidence/dapper-image-applicable.json '[.Results[]?.Secrets[]?] | length') { printf 'source_secrets=%s\n' "$source_secrets" printf 'source_applicable=%s\n' "$source_applicable" diff --git a/.python-version b/.python-version index a128d5c0..70c445ec 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.14.7 +3.14.9 diff --git a/Dockerfile.dapper b/Dockerfile.dapper index e7b243f7..978ffe1f 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -1,7 +1,7 @@ FROM ubuntu:26.04@sha256:2260313b31c8c011cd2eebe728008efac1b3982be73eb71348ea2648d2c0e09b COPY ubuntu-apt.lock /licenses/ubuntu-apt.lock -ADD --checksum=sha256:6077d27c6b6f8b23590cb01ff877ed8c804a67a5442cc32b5a33da10d2bd0e90 https://archive.ubuntu.com/ubuntu/pool/main/c/ca-certificates/ca-certificates_20260601~26.04.1_all.deb /tmp/ca-certificates.deb +ADD --checksum=sha256:5b6326f1932cfd7be57130561dc78192f09ed9f58bfbe6606ed0c50510c3b6cd https://archive.ubuntu.com/ubuntu/pool/main/c/ca-certificates/ca-certificates_20260816ubuntu0.26.04.1_all.deb /tmp/ca-certificates.deb ARG GO_VERSION=1.27.3 ARG GO_SHA256_amd64=65efbc80fdc1fa2cd2631cad5b481777a68e8725f34e89f50fc6ac714e379b40 @@ -17,6 +17,7 @@ ENV DEBIAN_FRONTEND=noninteractive \ XDG_CONFIG_HOME=/tmp/go-config \ PATH=/go/bin:/usr/local/go/bin:${PATH} +ADD --checksum=sha256:5b6326f1932cfd7be57130561dc78192f09ed9f58bfbe6606ed0c50510c3b6cd https://archive.ubuntu.com/ubuntu/pool/main/c/ca-certificates/ca-certificates_20260816ubuntu0.26.04.1_all.deb /opt/pasturestack-ca-certificates.deb RUN set -eux; \ . /licenses/ubuntu-apt.lock; \ rm -f /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; \ @@ -55,6 +56,11 @@ RUN set -eux; \ tox="${UBUNTU_APT_TOX_VERSION}" \ xz-utils="${UBUNTU_APT_XZ_UTILS_VERSION}" \ zip="${UBUNTU_APT_ZIP_VERSION}"; \ + dpkg -i /opt/pasturestack-ca-certificates.deb && \ + update-ca-certificates --fresh && \ + test "$(dpkg-query -W -f='${Version}' ca-certificates)" = '20260816ubuntu0.26.04.1' && \ + test ! -e /usr/share/ca-certificates/mozilla/emSign_ECC_Root_CA_-_G3.crt && \ + rm -f /opt/pasturestack-ca-certificates.deb && \ { \ printf 'archive\tubuntu-26.04-supported-pockets\n'; \ dpkg-query -W -f='${binary:Package}\t${Version}\n' | LC_ALL=C sort; \ @@ -64,6 +70,14 @@ RUN set -eux; \ RUN git config --system --add safe.directory '*' +COPY scripts/pip-security /opt/python-security/scripts +RUN python3 /opt/python-security/scripts/build-pip-overlay.py --work /tmp/pip-overlay-build --output /opt/python-security && \ + rm -rf /tmp/pip-overlay-build +ADD --checksum=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl /opt/python-security/certifi-2026.10.19-py3-none-any.whl +COPY scripts/install-python-security-tools /usr/local/bin/install-python-security-tools +ENV VIRTUALENV_PIP=26.2.2+pasturestack.1 VIRTUALENV_EXTRA_SEARCH_DIR=/usr/share/python-wheels VIRTUALENV_NO_PERIODIC_UPDATE=1 +RUN PYTHON_SECURITY_VIRTUALENV_PYTHON=python3 bash /usr/local/bin/install-python-security-tools + RUN case "${DAPPER_HOST_ARCH}" in \ amd64) go_arch=amd64; go_sha="${GO_SHA256_amd64}" ;; \ arm64) go_arch=arm64; go_sha="${GO_SHA256_arm64}" ;; \ diff --git a/README.md b/README.md index 8bacae2d..8da9eb7a 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ PastureStack is an independent community effort to preserve, audit, and moderniz ## Project status -The maintained source uses Ubuntu 26.04, Go 1.27.3, and Python 3.14.7 for integration-source verification. Ubuntu build packages, the base image, and Go archives are fixed by snapshot, exact version, and SHA-256. Go dependencies are managed by `go.mod` and a reproducible module vendor tree. AWS access uses AWS SDK for Go v2; container structures use the maintained Moby API module; the command parser uses urfave/cli v3. The abandoned AWS SDK v1, Rancher generated-client modules, event-subscriber module, `docker/libcompose`, and other GOPATH-era dependencies have been removed. The small Rancher 1.6 protocol boundary retained by this compatibility CLI is maintained and tested locally. The Dapper builder does not embed a Docker CLI or mount a Docker socket; Docker is required only on the host to create and run the isolated builder. The maintained binaries are `pasturestack-compose` and `compose-executor`; `platform-compose.yml` is the preferred companion file. Releases are built and verified from the preserved source history; this repository does not enable automated deployment. +The maintained source uses Ubuntu 26.04, Go 1.27.3, and Python 3.14.9 for integration-source verification. Ubuntu build packages, the base image, and Go archives are fixed by snapshot, exact version, and SHA-256. Go dependencies are managed by `go.mod` and a reproducible module vendor tree. AWS access uses AWS SDK for Go v2; container structures use the maintained Moby API module; the command parser uses urfave/cli v3. The abandoned AWS SDK v1, Rancher generated-client modules, event-subscriber module, `docker/libcompose`, and other GOPATH-era dependencies have been removed. The small Rancher 1.6 protocol boundary retained by this compatibility CLI is maintained and tested locally. The Dapper builder does not embed a Docker CLI or mount a Docker socket; Docker is required only on the host to create and run the isolated builder. The maintained binaries are `pasturestack-compose` and `compose-executor`; `platform-compose.yml` is the preferred companion file. Releases are built and verified from the preserved source history; this repository does not enable automated deployment. ## Configuration @@ -26,12 +26,12 @@ make test make package ``` -Release [v0.14.38](https://github.com/PastureStack/compose-cli/releases/tag/v0.14.38) -publishes `compose-executor-0.14.38-linux-amd64.gz`. Published tags and assets +Release [v0.14.39](https://github.com/PastureStack/compose-cli/releases/tag/v0.14.39) +publishes `compose-executor-0.14.39-linux-amd64.gz`. Published tags and assets remain immutable; a later recipe or documentation commit does not replace their bytes or change their producer identity. -Set `VERSION_OVERRIDE=0.14.38` for this recipe. Source changes alone are not +Set `VERSION_OVERRIDE=0.14.39` for this recipe. Source changes alone are not publication proof: verify the official release assets and their SHA-256. Component publication and matching Server acceptance are separate evidence boundaries. The Python integration suite must run against an @@ -47,3 +47,10 @@ The compatibility-server test requires an explicitly reviewed artifact URL throu ## License and attribution The inherited project remains licensed under [Apache License 2.0](LICENSE). Copyright and attribution for inherited work and vendored dependencies remain with their respective authors and contributors. PastureStack contributors claim authorship only for their own changes. + +## October 2026 security maintenance + +Maintained Go builds select the patched Go 1.27.3 compiler without raising the existing Go language requirement. Ubuntu build/runtime stages apply the checksum-pinned `ca-certificates 20260816ubuntu0.26.04.1` security package after their frozen APT baseline; the final trust store and installed version are verified. Python tooling uses the patched pip/CA path; CI interpreter pins remain on the Python 3.14 series. Release artifacts must pass the repository's existing checks before publication. +# Python installer security rebuild + +The disposable Dapper image rebuilds pip `26.2.2+pasturestack.1` from checksum-pinned official inputs using upstream vendoring and the original urllib3 patches. Only urllib3 changes to released `2.8.0`; this is a local security rebuild, not a new official PyPA release. The expected wheel SHA-256 is `be5f63d537bbe611dc422fabb6740a44b5997888b592888df5091727f023802a`. Ubuntu's Python ABI is retained. The image tests live pip, available ensurepip/new-venv seeds, virtualenv, streaming limits, and trusted/untrusted TLS. See `scripts/pip-security/README.md` for source and license evidence. diff --git a/go.mod b/go.mod index ade2cd6a..eeea6f6e 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/PastureStack/compose-cli go 1.26.0 -toolchain go1.27.0 +toolchain go1.27.3 require ( github.com/Masterminds/sprig/v3 v3.3.0 diff --git a/scripts/check-pasturestack-source b/scripts/check-pasturestack-source index b0fba59e..4a7a9129 100755 --- a/scripts/check-pasturestack-source +++ b/scripts/check-pasturestack-source @@ -50,7 +50,7 @@ require_line Dockerfile.dapper ' GOFLAGS=-mod=vendor \' require_line Dockerfile.dapper " printf 'Types: deb\\nURIs: https://archive.ubuntu.com/ubuntu\\nSuites: resolute resolute-updates resolute-backports\\nComponents: main universe restricted multiverse\\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg\\nSnapshot: no\\n\\nTypes: deb\\nURIs: https://security.ubuntu.com/ubuntu\\nSuites: resolute-security\\nComponents: main universe restricted multiverse\\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg\\nSnapshot: no\\n' \\" # Preserve module declarations independently of the checksum-pinned build compiler. require_line go.mod 'go 1.26.0' -require_line go.mod 'toolchain go1.27.0' +require_line go.mod 'toolchain go1.27.3' require_line go.mod $'\tgithub.com/aws/aws-sdk-go-v2 v1.43.8' require_line go.mod $'\tgithub.com/aws/aws-sdk-go-v2/service/s3 v1.107.4' require_line go.mod $'\tgithub.com/moby/moby/api v1.55.0' @@ -69,7 +69,7 @@ require_line Dockerfile.dapper ' libssl3t64="${UBUNTU_APT_OPENSSL_VERSION require_line Dockerfile.dapper ' linux-libc-dev="${UBUNTU_APT_LINUX_LIBC_DEV_VERSION}" \' require_line Dockerfile.dapper ' openssl="${UBUNTU_APT_OPENSSL_VERSION}" \' require_line Dockerfile.dapper ' openssl-provider-legacy="${UBUNTU_APT_OPENSSL_VERSION}" \' -require_line .python-version '3.14.7' +require_line .python-version '3.14.9' require_line tests/integration/requirements.txt 'pytest==9.1.1' require_line tests/integration/requirements.txt 'setuptools==84.0.0' require_line .github/workflows/security-release-gate.yml ' PLATFORM_COMPAT_JAR_URL: https://github.com/PastureStack/orchestration-engine/releases/download/v0.183.281/orchestration-engine-0.183.281.jar' @@ -122,4 +122,4 @@ fi require_line README.md 'PastureStack is an independent community effort to preserve, audit, and modernize the Rancher 1.6 ecosystem. It is not affiliated with or endorsed by Rancher Labs or SUSE.' require_line ORIGIN.md '- Migration model: unchanged upstream history followed by one PastureStack maintenance commit' -printf 'PASTURESTACK_COMPOSE_SOURCE_GATE_OK modules=vendor aws_sdk=v2 moby_api=1.55.0 archived_go_dependencies=absent system_images=ghcr local_resource_boundary=root_scoped unused_docker_cli=absent ubuntu=26.04 apt_direct_versions=locked language_floor=1.26.0 toolchain=1.27.3 module_toolchain=1.27.0 python=3.14.7\n' +printf 'PASTURESTACK_COMPOSE_SOURCE_GATE_OK modules=vendor aws_sdk=v2 moby_api=1.55.0 archived_go_dependencies=absent system_images=ghcr local_resource_boundary=root_scoped unused_docker_cli=absent ubuntu=26.04 apt_direct_versions=locked language_floor=1.26.0 toolchain=1.27.3 module_toolchain=1.27.3 python=3.14.9\n' diff --git a/scripts/install-python-security-tools b/scripts/install-python-security-tools new file mode 100644 index 00000000..2076a0db --- /dev/null +++ b/scripts/install-python-security-tools @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Keep Ubuntu's ABI; install the reproducible local pip security rebuild and seeds. +set -euo pipefail +expected=26.2.2+pasturestack.1 +pip_wheel=/opt/python-security/pip-$expected-py3-none-any.whl +certifi_wheel=/opt/python-security/certifi-2026.10.19-py3-none-any.whl +test -s "$pip_wheel" +printf '%s %s\n' be5f63d537bbe611dc422fabb6740a44b5997888b592888df5091727f023802a "$pip_wheel" | sha256sum -c - +test -s "$certifi_wheel" +if dpkg-query -W -f='${db:Status-Status}' python3-pip 2>/dev/null | grep -qx installed; then + apt-get purge -y python3-pip +fi +PYTHONPATH="$pip_wheel" python3 -m pip install --break-system-packages --no-index --no-deps "$pip_wheel" "$certifi_wheel" +install -d /usr/share/python-wheels +find /usr/share/python-wheels -maxdepth 1 -type f -name 'pip-*.whl' -delete +install -m 0644 "$pip_wheel" /usr/share/python-wheels/ +python3 -c 'import pip, certifi, pip._vendor.certifi as c; assert pip.__version__ == "26.2.2+pasturestack.1"; assert certifi.__version__ == c.__version__ == "2026.10.19"' +python3 /opt/python-security/scripts/test-pip-overlay.py +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +if python3 -c 'import ensurepip' 2>/dev/null; then + python3 -c 'import ensurepip; assert ensurepip.version() == "26.2.2+pasturestack.1"' + python3 -m venv "$scratch/venv" + ensurepip_status=$expected +else + # Do not add an unused distro ensurepip package just to exercise a test. + python3 -m venv --without-pip "$scratch/venv" + PYTHONPATH="$pip_wheel" "$scratch/venv/bin/python" -m pip install --no-index --no-deps "$pip_wheel" + ensurepip_status=not-installed +fi +"$scratch/venv/bin/python" /opt/python-security/scripts/test-pip-overlay.py +if [ -n "${PYTHON_SECURITY_VIRTUALENV_PYTHON:-}" ]; then + "$PYTHON_SECURITY_VIRTUALENV_PYTHON" -m virtualenv --no-download --no-periodic-update --pip="$expected" --extra-search-dir=/usr/share/python-wheels "$scratch/virtualenv" + "$scratch/virtualenv/bin/python" /opt/python-security/scripts/test-pip-overlay.py +fi +printf 'PYTHON_SECURITY_TOOLS_OK pip=%s certifi=2026.10.19 ensurepip=%s\n' "$expected" "$ensurepip_status" diff --git a/scripts/pip-security/README.md b/scripts/pip-security/README.md new file mode 100644 index 00000000..e354536a --- /dev/null +++ b/scripts/pip-security/README.md @@ -0,0 +1,75 @@ +# 共用 pip 安全重包 + +這是內部衍生版本 `26.2.2+pasturestack.1`,不是 PyPA 官方 pip 發布。 +所有專案及映像的對外 release tag 仍使用原本的純數字規則。 + +來源是官方 pip `26.2.2` 的 commit +`b3b696fe2122f2339d782517559bfade2cb6b879`。使用上游 `vendoring==1.4.0` +及 pip 原有的三份 urllib3 patches,把官方 PyPI urllib3 改成 `2.8.0`。 +不手改 urllib3 原始碼、不 debundle。其餘內嵌 library 的每一個檔案均須 +保持原始 SHA-256;licenses、內嵌 CycloneDX SBOM 及 wheel RECORD 全部保留。 + +## 建置 + +Linux Python 3.12/3.14,amd64/arm64;所有工具與原始碼的 URL/SHA 在 +`inputs.json`。工具的相依套件不會進入產物。 + +```sh +python3 build-pip-overlay.py --work /tmp/pip-build --output /tmp/pip-dist \ + --input-cache /tmp/pip-inputs +# 從完全相同的已驗證上游檔案重新離線建置: +python3 build-pip-overlay.py --work /tmp/pip-rebuild --output /tmp/pip-dist2 \ + --input-cache /tmp/pip-inputs --offline +``` + +下載後的 vendoring、安裝工具與 wheel 建置使用 `PIP_NO_INDEX=1`。 +每次建置都做兩次 wheel 並比較位元組,驗證全部 RECORD hashes、無 native +extension、保留 license,以及只有 urllib3 library 有變更。 +輸出為 wheel、`build-evidence.json`、`pip-vendor.cdx.json`、工具 hash lock +與來源 manifest。已在 Python 3.12 建出 wheel SHA-256: +`be5f63d537bbe611dc422fabb6740a44b5997888b592888df5091727f023802a`。 + +## 安裝與三種 seed 路徑 + +編譯 CPython 前,以 `install-ensurepip-wheel.py --stdlib /Lib +--wheel --sha256 ` 同步 `_PIP_VERSION` 與 `_bundled` wheel +檔名,再正常編譯/安裝;不能只改 site-packages。 + +Ubuntu 的 distro Python 使用 `/usr/share/python-wheels` 時,將該固定 wheel +放入這個 seed directory 並移除其中舊的 pip wheel(不要動其他 wheel); +以此 wheel `--no-index --no-deps` 安裝 live pip。既有 Python ABI 不變。 + +完成映像內安裝後執行: + +```sh +bash verify-pip-runtime.sh +# Engine/compose 已有 virtualenv 時,再驗第三條 seed 路徑: +bash verify-pip-runtime.sh --virtualenv /usr/share/python-wheels +``` + +此 gate 驗證 live pip、ensurepip.version、新 venv、選用的新 virtualenv +全部載入衍生 pip 與 urllib3 2.8.0,並重跑攻防、可信/不可信 TLS、CLI。 +本機 Python 3.12 的 wheel/攻防/TLS/CLI 已通過;真正 3.14 的 seed 三路驗證 +須由消費映像 CI 執行,不能把本機結果冒充為 3.14 PASS。 + +## 已驗證的漏洞界線 + +- CVE-2026-97689:pip `response_chunks` 使用 `raw.stream`,確實可達。 + 測試在舊版對惡意 chunk-size/trailer 讀入 131072 bytes;新版在 65537 bytes + 即拒絕,正常資料與邊界內的 chunk extension 保持可用。 +- CVE-2026-57585:上游 fix commit + `2c56ddb5d0025ed481d962c0f5d62d19dec7476d` 只修 Cython/C 實作;本 wheel + 無 `.so/.pyd`,實際 `Unpacker` 為 `pip._vendor.msgpack.fallback`。 + 不為消除套件名稱掃描結果而更動未受影響的 msgpack。 +- setuptools 70.3.0 僅保留 `pkg_resources`;CVE-2025-47273 所需的 + `setuptools/package_index.py` 與 CVE-2026-59890 所需的 + `setuptools/command/egg_info.py` 均不存在。 + +`check-pip-vendor-slice.py` 會比對實際載入的全部 299 個 vendor 檔案與 +固定 wheel,拒絕額外檔案、其他同版本完整 distribution。只有取得此證據後, +`filter-pip-vendor-findings.py` 才可對上述三個 CVE(含 msgpack GHSA 別名) +產生綁定該映像的 OpenVEX 與有效掃描檔。原始掃描必須另行保存;未知 CVE、 +其他版本、非 SBOM 套件來源與 urllib3 finding 全部保留,不做泛用版本排除。 +執行 `python3 test-pip-vex.py` 可驗證以上六組正反控制。 + +官方政策:https://pip.pypa.io/en/stable/development/vendoring-policy/ diff --git a/scripts/pip-security/build-pip-overlay.py b/scripts/pip-security/build-pip-overlay.py new file mode 100644 index 00000000..88b53806 --- /dev/null +++ b/scripts/pip-security/build-pip-overlay.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Rebuild pip using released urllib3 and pip's unmodified vendoring machinery. + +Build dependencies never enter the product wheel. All network inputs are HTTPS +and SHA-256 pinned. The resulting local version is not an official pip release. +""" +import argparse +import base64 +import csv +import hashlib +import io +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tarfile +import tempfile +from urllib.request import HTTPRedirectHandler, HTTPSHandler, build_opener +from urllib.parse import urlsplit +import venv +import zipfile + + +class HTTPSOnlyRedirect(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + if urlsplit(newurl).scheme != "https": + raise ValueError("Refusing a non-HTTPS build-input redirect") + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def run(command, cwd, env): + print("+", " ".join(map(str, command)), flush=True) + subprocess.run(list(map(str, command)), cwd=cwd, env=env, check=True) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--work", type=Path, required=True) + parser.add_argument("--input-cache", type=Path) + parser.add_argument("--offline", action="store_true") + args = parser.parse_args() + output, work = args.output.resolve(), args.work.resolve() + output.mkdir(parents=True, exist_ok=True) + work.mkdir(parents=True, exist_ok=True) + config = json.loads(Path(__file__).with_name("inputs.json").read_text()) + assert sys.version_info[:2] in ((3, 12), (3, 14)) and os.name == "posix" + scratch = Path(tempfile.mkdtemp(prefix="pip-revendor-", dir=work)) + downloads = scratch / "downloads" + downloads.mkdir() + cache = args.input_cache.resolve() if args.input_cache else downloads + cache.mkdir(parents=True, exist_ok=True) + paths = {} + for item in config["artifacts"]: + assert urlsplit(item["url"]).scheme == "https" + target = downloads / Path(urlsplit(item["url"]).path).name + cached = cache / target.name + if not cached.is_file(): + assert not args.offline, ("Offline input is missing", target.name) + opener = build_opener(HTTPSHandler(), HTTPSOnlyRedirect()) + with opener.open(item["url"], timeout=90) as response, cached.open("wb") as dest: + shutil.copyfileobj(response, dest) + assert sha(cached) == item["sha256"], ("cached input hash mismatch", target.name) + if cached != target: + shutil.copy2(cached, target) + assert sha(target) == item["sha256"], ("input hash mismatch", target.name) + paths.setdefault(item["name"], []).append(target) + + environment = os.environ.copy() + environment.update(PIP_NO_INDEX="1", PIP_FIND_LINKS=str(downloads), + PIP_DISABLE_PIP_VERSION_CHECK="1", PIP_ONLY_BINARY=":all:", + PYTHONHASHSEED="0", SOURCE_DATE_EPOCH=str(config["sourceDateEpoch"])) + for key in ("PYTHONPATH", "PYTHONHOME", "PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", + "PIP_TRUSTED_HOST", "PIP_CERT", "PIP_CLIENT_CERT"): + environment.pop(key, None) + environment["PIP_CONFIG_FILE"] = os.devnull + tool_env = scratch / "tool-env" + venv.EnvBuilder(with_pip=False).create(tool_env) + python = tool_env / "bin/python" + requirements = scratch / "tools.lock" + groups = {} + for item in config["artifacts"]: + if item["kind"] == "tool": + groups.setdefault((item["name"], item["version"]), []).append(item["sha256"]) + requirements.write_text("\n".join( + f"{name}=={version} " + " ".join(f"--hash=sha256:{value}" for value in hashes) + for (name, version), hashes in sorted(groups.items())) + "\n") + bootstrap_env = dict(environment, PYTHONPATH=str(paths["pip"][0])) + run([python, "-m", "pip", "install", "--require-hashes", "--no-compile", "-r", requirements], scratch, bootstrap_env) + environment["PATH"] = str(tool_env / "bin") + os.pathsep + environment["PATH"] + + source_parent = scratch / "source" + source_parent.mkdir() + with tarfile.open(paths["pip-source"][0]) as archive: + # Reject links and traversal before extracting this fixed official archive. + for member in archive.getmembers(): + assert not member.issym() and not member.islnk() + resolved = (source_parent / member.name).resolve() + assert resolved.is_relative_to(source_parent) + archive.extractall(source_parent, filter="data") + source = next(source_parent.iterdir()) + vendor = source / "src/pip/_vendor" + original = {str(p.relative_to(vendor)): sha(p) for p in vendor.rglob("*") if p.is_file()} + assert (source / "src/pip/__init__.py").read_text().count('__version__ = "26.2.2"') == 1 + + # A scoped project invokes the same upstream tool, namespace and urllib3 + # patches. Unrelated vendored libraries are never regenerated or replaced. + scoped = scratch / "urllib3-vendoring" + scoped_vendor = scoped / "src/pip/_vendor" + scoped_vendor.mkdir(parents=True) + (scoped_vendor / "vendor.txt").write_text("urllib3==2.8.0\n") + patches = scoped / "tools/vendoring/patches" + patches.mkdir(parents=True) + for patch in (source / "tools/vendoring/patches").glob("urllib3*.patch"): + shutil.copy2(patch, patches / patch.name) + assert len(list(patches.glob("*.patch"))) == 3 + (scoped / "pyproject.toml").write_text('''[tool.vendoring] +destination = "src/pip/_vendor/" +requirements = "src/pip/_vendor/vendor.txt" +namespace = "pip._vendor" +protected-files = ["vendor.txt"] +patches-dir = "tools/vendoring/patches" +[tool.vendoring.transformations] +drop = ["*.dist-info/", "bin/"] +''') + run([python, "-m", "vendoring", "sync", "-v"], scoped, environment) + generated = scoped_vendor / "urllib3" + assert (generated / "__init__.py").is_file() + shutil.rmtree(vendor / "urllib3") # exact disposable source subtree only + shutil.copytree(generated, vendor / "urllib3") + vendor_txt = vendor / "vendor.txt" + text = vendor_txt.read_text() + assert text.count("urllib3==2.7.0") == 1 + vendor_txt.write_text(text.replace("urllib3==2.7.0", "urllib3==2.8.0")) + version_file = source / "src/pip/__init__.py" + version_file.write_text(version_file.read_text().replace('__version__ = "26.2.2"', + f'__version__ = "{config["version"]}"')) + readme = vendor / "README.rst" + with readme.open("a") as handle: + handle.write("\n\nPastureStack security rebuild\n============================\n\n" + "This downstream local-version wheel uses released urllib3 2.8.0\n" + "through upstream vendoring 1.4.0 and pip's unchanged urllib3\n" + "patches. Other vendored library sources and licenses are unchanged.\n" + "It is not an official PyPA pip release.\n") + run([python, "-c", "from pathlib import Path; from vendoring.sbom import create_sbom_file; create_sbom_file('pip._vendor', Path('src/pip/_vendor/vendor.txt'), Path('src/pip/_vendor/bom.cdx.json'))"], source, environment) + current = {str(p.relative_to(vendor)): sha(p) for p in vendor.rglob("*") if p.is_file()} + allowed = {"vendor.txt", "README.rst", "bom.cdx.json"} + assert {p: value for p, value in original.items() if not p.startswith("urllib3/") and p not in allowed} == { + p: value for p, value in current.items() if not p.startswith("urllib3/") and p not in allowed} + wheels = [] + for suffix in ("first", "second"): + directory = scratch / suffix + directory.mkdir() + run([python, "-c", f"from flit_core.buildapi import build_wheel; build_wheel({str(directory)!r})"], source, environment) + wheels.append(next(directory.glob("*.whl"))) + assert sha(wheels[0]) == sha(wheels[1]), "wheel rebuild is not byte reproducible" + assert sha(wheels[0]) == config["expectedWheelSha256"], "unexpected rebuilt product bytes" + wheel = output / wheels[0].name + shutil.copy2(wheels[0], wheel) + licenses = [] + with zipfile.ZipFile(wheel) as archive: + names = archive.namelist() + record = next(name for name in names if name.endswith(".dist-info/RECORD")) + for name, digest, size in csv.reader(io.StringIO(archive.read(record).decode())): + if not digest: + assert name == record + continue + data = archive.read(name) + expected = "sha256=" + base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() + assert digest == expected and int(size) == len(data), name + assert not any(name.endswith((".so", ".pyd")) for name in names) + licenses = [name for name in names if "LICENSE" in name or "COPYING" in name] + assert any("urllib3" in name for name in licenses) + assert "pip/_vendor/pkg_resources/__init__.py" in names + assert not any("setuptools/package_index" in name for name in names) + shutil.copy2(vendor / "bom.cdx.json", output / "pip-vendor.cdx.json") + report = {"version": config["version"], "upstreamVersion": config["upstreamVersion"], + "upstreamCommit": config["sourceCommit"], "wheel": wheel.name, + "sha256": sha(wheel), "twoBuildsByteIdentical": True, + "allRecordHashesVerified": True, "nativeExtensionsPresent": False, + "licenses": licenses, "onlyVendoredLibraryChanged": "urllib3", "urllib3": "2.8.0"} + (output / "build-evidence.json").write_text(json.dumps(report, indent=2) + "\n") + shutil.copy2(requirements, output / "tools.lock") + shutil.copy2(Path(__file__).with_name("inputs.json"), output / "inputs.json") + print(json.dumps(report), flush=True) + + +if __name__ == "__main__": + if sys.flags.optimize: + raise RuntimeError("Security build verification must not run with Python optimization") + main() diff --git a/scripts/pip-security/check-pip-vendor-slice.py b/scripts/pip-security/check-pip-vendor-slice.py new file mode 100644 index 00000000..789bc280 --- /dev/null +++ b/scripts/pip-security/check-pip-vendor-slice.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Prove the exact installed pip vendor slice before producing narrow VEX data.""" +import argparse +import hashlib +import importlib.metadata +import importlib.resources +import json +from pathlib import Path +import zipfile +import pip +from pip._vendor import msgpack, urllib3 + +EXPECTED_SHA = "be5f63d537bbe611dc422fabb6740a44b5997888b592888df5091727f023802a" +FINDINGS = { + "GHSA-6v7p-g79w-8964": ("msgpack", "1.1.2", "C/Cython unpacker is absent; runtime selects pure Python fallback"), + "CVE-2026-57585": ("msgpack", "1.1.2", "C/Cython unpacker is absent; runtime selects pure Python fallback"), + "CVE-2025-47273": ("setuptools", "70.3.0", "setuptools/package_index.py is absent; only pkg_resources is vendored"), + "CVE-2026-59890": ("setuptools", "70.3.0", "setuptools/command/egg_info.py and FileList are absent"), +} + + +def verify(wheel): + assert hashlib.sha256(wheel.read_bytes()).hexdigest() == EXPECTED_SHA + assert pip.__version__ == "26.2.2+pasturestack.1" + assert urllib3.__version__ == "2.8.0" + assert msgpack.Unpacker.__module__ == "pip._vendor.msgpack.fallback" + live = importlib.resources.files("pip") + checked = 0 + with zipfile.ZipFile(wheel) as archive: + names = archive.namelist() + expected_files = set() + for name in names: + if not name.startswith("pip/_vendor/") or name.endswith("/"): + continue + assert live.joinpath(name.removeprefix("pip/")).read_bytes() == archive.read(name), name + expected_files.add(name.removeprefix("pip/_vendor/")) + checked += 1 + assert not any(name.startswith("pip/_vendor/") and name.endswith((".so", ".pyd")) for name in names) + assert not any(name.startswith("pip/_vendor/setuptools/") for name in names) + assert not any(name.startswith("pip/_vendor/msgpack/_cmsgpack") for name in names) + def live_files(directory, prefix=""): + for entry in directory.iterdir(): + if entry.name == "__pycache__": + continue + relative = prefix + entry.name + if entry.is_dir(): + yield from live_files(entry, relative + "/") + else: + yield relative + assert set(live_files(live.joinpath("_vendor"))) == expected_files, "Unexpected live vendor files" + # A separate full setuptools installation cannot inherit the vendor-slice VEX. + for distribution in importlib.metadata.distributions(): + if distribution.metadata.get("Name", "").lower() == "setuptools": + assert distribution.version != "70.3.0", "A full setuptools 70.3.0 distribution is installed" + if distribution.metadata.get("Name", "").lower() == "msgpack": + assert distribution.version != "1.1.2", "A separate msgpack 1.1.2 distribution is installed" + return {"verifiedWheelSha256": EXPECTED_SHA, "pip": pip.__version__, + "urllib3": urllib3.__version__, "vendorFilesByteCompared": checked, + "msgpackImplementation": msgpack.Unpacker.__module__, + "setuptoolsPackageAbsent": True, "scope": "this exact pip vendor slice only", + "notAffected": [{"id": key, "package": value[0], "version": value[1], + "justification": "vulnerable_code_not_present", "detail": value[2]} + for key, value in FINDINGS.items()]} + + +if __name__ == "__main__": + import sys + if sys.flags.optimize: + raise RuntimeError("Security verification must not run with Python optimization") + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path) + args = parser.parse_args() + # The security proof is about the interpreter's actual selected seed, not + # an arbitrary file supplied through a CLI path. + try: + import ensurepip + except ImportError: + # compose's distro runtime intentionally has no ensurepip package. + # Its build retains the verified input at this fixed, non-user path. + result = verify(Path("/opt/python-security/pip-26.2.2+pasturestack.1-py3-none-any.whl")) + else: + with ensurepip._get_pip_whl_path_ctx() as wheel: + result = verify(Path(wheel)) + data = json.dumps(result, indent=2) + "\n" + if args.output: + args.output.write_text(data) + print(data, end="") diff --git a/scripts/pip-security/filter-pip-vendor-findings.py b/scripts/pip-security/filter-pip-vendor-findings.py new file mode 100644 index 00000000..5a7c1907 --- /dev/null +++ b/scripts/pip-security/filter-pip-vendor-findings.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Apply narrowly proven pip-vendor VEX, retaining the original Trivy report.""" +import argparse +from copy import deepcopy +from datetime import datetime, timezone +import json +from pathlib import Path + +WHEEL_SHA = "be5f63d537bbe611dc422fabb6740a44b5997888b592888df5091727f023802a" +RULES = { + "GHSA-6v7p-g79w-8964": ("msgpack", "1.1.2"), + "CVE-2026-57585": ("msgpack", "1.1.2"), + "CVE-2025-47273": ("setuptools", "70.3.0"), + "CVE-2026-59890": ("setuptools", "70.3.0"), +} + + +def apply(raw, proof, subject): + valid = (proof["verifiedWheelSha256"] == WHEEL_SHA and + proof["pip"] == "26.2.2+pasturestack.1" and proof["urllib3"] == "2.8.0" and + proof["msgpackImplementation"] == "pip._vendor.msgpack.fallback" and + proof["setuptoolsPackageAbsent"] is True and proof["vendorFilesByteCompared"] == 299) + if not valid: + raise ValueError("Exact installed pip vendor proof is required") + reasons = {item["id"]: item for item in proof["notAffected"]} + result = deepcopy(raw) + statements = [] + for section in result.get("Results", []): + retained = [] + for item in section.get("Vulnerabilities", []): + key = item.get("VulnerabilityID") + expected = RULES.get(key) + actual = (item.get("PkgName"), item.get("InstalledVersion")) + packages = [pkg for pkg in section.get("Packages", []) if + (pkg.get("Name"), pkg.get("Version")) == actual] + reason = reasons.get(key, {}) + eligible = (expected == actual and key in reasons and + (reason.get("package"), reason.get("version")) == actual and + section.get("Type") == "python-pkg" and packages and + all(pkg.get("AnalyzedBy") == "sbom" for pkg in packages)) + if not eligible: + retained.append(item) + continue + purl = "pkg:pypi/" + actual[0] + "@" + actual[1] + statements.append({ + "vulnerability": {"name": key}, + "products": [{"@id": subject, "subcomponents": [{"@id": purl}]}], + "status": "not_affected", "justification": "vulnerable_code_not_present", + "impact_statement": reasons[key]["detail"] + "; exact pip wheel SHA-256 " + WHEEL_SHA, + }) + section["Vulnerabilities"] = retained + vex = {"@context": "https://openvex.dev/ns/v0.2.0", "@id": subject + "/pip-vendor-vex", + "author": "PastureStack", "timestamp": datetime.now(timezone.utc).isoformat(), + "version": 1, "statements": statements} + return result, vex + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--raw", type=Path, required=True) + parser.add_argument("--proof", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--vex", type=Path, required=True) + parser.add_argument("--subject", required=True) + args = parser.parse_args() + report, vex = apply(json.loads(args.raw.read_text()), json.loads(args.proof.read_text()), args.subject) + args.output.write_text(json.dumps(report, indent=2) + "\n") + args.vex.write_text(json.dumps(vex, indent=2) + "\n") + print(f"PROVEN_PIP_VENDOR_VEX statements={len(vex['statements'])}") diff --git a/scripts/pip-security/inputs.json b/scripts/pip-security/inputs.json new file mode 100644 index 00000000..fc6b7de9 --- /dev/null +++ b/scripts/pip-security/inputs.json @@ -0,0 +1,188 @@ +{ + "schemaVersion": 1, + "upstreamVersion": "26.2.2", + "version": "26.2.2+pasturestack.1", + "expectedWheelSha256": "be5f63d537bbe611dc422fabb6740a44b5997888b592888df5091727f023802a", + "sourceCommit": "b3b696fe2122f2339d782517559bfade2cb6b879", + "sourceDateEpoch": 1792509744, + "updatedVendoredLibraries": { + "urllib3": "2.8.0" + }, + "artifacts": [ + { + "kind": "tool", + "name": "vendoring", + "version": "1.4.0", + "url": "https://files.pythonhosted.org/packages/e3/ba/2fe43514bcf426888101c2968f3c821c6338afc122ccbbde6e29ee2a5036/vendoring-1.4.0-py3-none-any.whl", + "sha256": "3a5b5c44bdec2683a94cd4c7c25bad556b38539cf316defca65948d76107eb30" + }, + { + "kind": "tool", + "name": "flit_core", + "version": "3.12.0", + "url": "https://files.pythonhosted.org/packages/f2/65/b6ba90634c984a4fcc02c7e3afe523fef500c4980fec67cc27536ee50acf/flit_core-3.12.0-py3-none-any.whl", + "sha256": "e7a0304069ea895172e3c7bb703292e992c5d1555dd1233ab7b5621b5b69e62c" + }, + { + "kind": "tool", + "name": "click", + "version": "8.5.0", + "url": "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", + "sha256": "255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360" + }, + { + "kind": "tool", + "name": "jsonschema", + "version": "4.26.0", + "url": "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", + "sha256": "d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce" + }, + { + "kind": "tool", + "name": "packaging", + "version": "26.3", + "url": "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", + "sha256": "d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c" + }, + { + "kind": "tool", + "name": "requests", + "version": "2.34.2", + "url": "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", + "sha256": "2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0" + }, + { + "kind": "tool", + "name": "rich", + "version": "15.0.0", + "url": "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", + "sha256": "33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb" + }, + { + "kind": "tool", + "name": "attrs", + "version": "26.1.0", + "url": "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", + "sha256": "c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309" + }, + { + "kind": "tool", + "name": "idna", + "version": "3.20", + "url": "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", + "sha256": "ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c" + }, + { + "kind": "tool", + "name": "jsonschema-specifications", + "version": "2025.9.1", + "url": "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", + "sha256": "98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe" + }, + { + "kind": "tool", + "name": "markdown-it-py", + "version": "4.2.0", + "url": "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", + "sha256": "9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a" + }, + { + "kind": "tool", + "name": "Pygments", + "version": "2.21.0", + "url": "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", + "sha256": "2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9" + }, + { + "kind": "tool", + "name": "referencing", + "version": "0.37.0", + "url": "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", + "sha256": "381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231" + }, + { + "kind": "tool", + "name": "urllib3", + "version": "2.8.0", + "url": "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", + "sha256": "0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3" + }, + { + "kind": "tool", + "name": "mdurl", + "version": "0.1.2", + "url": "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", + "sha256": "84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8" + }, + { + "kind": "tool", + "name": "typing_extensions", + "version": "4.16.0", + "url": "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", + "sha256": "481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8" + }, + { + "kind": "tool", + "name": "tomli", + "version": "2.5.0", + "url": "https://files.pythonhosted.org/packages/60/3f/3e3f8fd0919249b0200c80fbc4f9a1e70be19f9883da71dfb7f8b9ab8aca/tomli-2.5.0-py3-none-any.whl", + "sha256": "32a7b79ac57a2e83670ce329ccf675798bc5a2094783a63676866b70503f2e2b" + }, + { + "kind": "tool", + "name": "charset-normalizer", + "version": "3.5.2", + "url": "https://files.pythonhosted.org/packages/fc/ad/d07d7862a62ffa6d79d68074d14823243dd235a77c45262acbf6adeb28bf/charset_normalizer-3.5.2-py3-none-any.whl", + "sha256": "b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685" + }, + { + "kind": "tool", + "name": "certifi", + "version": "2026.10.19", + "url": "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", + "sha256": "42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b" + }, + { + "kind": "tool", + "name": "rpds-py", + "version": "2026.9.1", + "url": "https://files.pythonhosted.org/packages/42/ff/bf7d54f362748fd6a49277b9d6531c394074110fedf50ad791ec59133fbb/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "sha256": "2693b2728bbcc48d09a981a356954b0c47c53ff25b545856f28a889ea619f69a" + }, + { + "kind": "tool", + "name": "rpds-py", + "version": "2026.9.1", + "url": "https://files.pythonhosted.org/packages/1d/6d/b26eb1e75395925b3a142ed351cbed2ec8212f5c9d937aee3df32c701baa/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "sha256": "addeda51556dac7c1a2f14cda62db8b621cd12afba3091d03a96c72932387eab" + }, + { + "kind": "tool", + "name": "rpds-py", + "version": "2026.9.1", + "url": "https://files.pythonhosted.org/packages/72/e2/5873bc4643c250db9e05d48dc0c93763d4aa68bc3b81164cb1af3b45b284/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "sha256": "07deecbfce94c78473018bc7d10b337cc651d12df87a1eb2cb3e4024bc9c33d0" + }, + { + "kind": "tool", + "name": "rpds-py", + "version": "2026.9.1", + "url": "https://files.pythonhosted.org/packages/22/9d/d08a1128ab199b2f0cf25bfeb0639bd05119fff4b7c47bec24ef9a8ec23f/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "sha256": "eac2f5dbafd585dfe31f86a23ebf0d3ba480a9d49ebc87947267b5608d4ea0cd" + }, + { + "kind": "source", + "name": "pip-source", + "version": "26.2.2", + "url": "https://github.com/pypa/pip/archive/b3b696fe2122f2339d782517559bfade2cb6b879.tar.gz", + "sha256": "5202e6f34650ed74b8cfc74a4808af45a46ebd994b187e160a92d426e1bb3e0f" + }, + { + "kind": "tool", + "name": "pip", + "version": "26.2.2", + "url": "https://files.pythonhosted.org/packages/3f/06/3e4420302363fbb71be3c7f35ce042f54bfca330e3e66b3a2e3c4cca579f/pip-26.2.2-py3-none-any.whl", + "sha256": "fd7f4dfa8190e014013b9f08f3e973f4e4c26ffd179a204895533a7254813a70" + } + ] +} diff --git a/scripts/pip-security/test-pip-overlay.py b/scripts/pip-security/test-pip-overlay.py new file mode 100644 index 00000000..a2c62651 --- /dev/null +++ b/scripts/pip-security/test-pip-overlay.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Focused streaming regression, CLI and TLS controls for the installed wheel.""" +import argparse +import contextlib +import http.client +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import io +import json +from pathlib import Path +import ssl +import subprocess +import sys +import tempfile +import threading + +import pip +from pip._internal.network.utils import response_chunks +from pip._vendor import msgpack, requests, urllib3 +from pip._vendor.urllib3.exceptions import ProtocolError + + +class TrackedBody(io.BytesIO): + largest_readline = 0 + def readline(self, size=-1): + data = super().readline(size) + self.largest_readline = max(self.largest_readline, len(data)) + return data + + +class Socket: + def __init__(self, body): + self.body = body + def makefile(self, *_): + return self.body + + +def streamed(body): + wire = TrackedBody(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n" + body) + origin = http.client.HTTPResponse(Socket(wire), method="GET") + origin.begin() + response = requests.Response() + response.status_code = 200 + response.raw = urllib3.response.HTTPResponse( + body=origin, original_response=origin, headers=dict(origin.headers.items()), + preload_content=False) + return response, wire + + +def boundary_tests(expect_fixed): + outcome = [] + for label, body in ( + ("chunk-size", b"f" * 131072), + ("chunk-trailer", b"3\r\nfoo\r\n0\r\n" + b"X" * 131072), + ): + response, wire = streamed(body) + error_description = "no exception" + try: + list(response_chunks(response)) + except Exception as error: + error_description = repr(error) + if expect_fixed: + assert isinstance(error, ProtocolError), repr(error) + assert "exceeded maximum allowed length" in str(error), str(error) + if expect_fixed: + assert error_description != "no exception", "malicious chunk line was accepted" + assert wire.largest_readline <= 65537, wire.largest_readline + else: + assert wire.largest_readline == 131072, (wire.largest_readline, error_description) + outcome.append({"path": label, "max_readline": wire.largest_readline}) + for body in (b"3\r\nfoo\r\n0\r\n\r\n", b"3;" + b"x" * 65532 + b"\r\nfoo\r\n0\r\n\r\n"): + response, _ = streamed(body) + assert b"".join(response_chunks(response)) == b"foo" + return outcome + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--expect-vulnerable", action="store_true") + args = parser.parse_args() + result = {"pip": pip.__version__, "urllib3": urllib3.__version__, + "streaming": boundary_tests(not args.expect_vulnerable)} + if args.expect_vulnerable: + print(json.dumps(result)); return + assert pip.__version__ == "26.2.2+pasturestack.1" + assert urllib3.__version__ == "2.8.0" + assert msgpack.Unpacker.__module__ == "pip._vendor.msgpack.fallback" + import importlib.util + assert importlib.util.find_spec("pip._vendor.msgpack._cmsgpack") is None + unpacker = msgpack.Unpacker() + unpacker.feed(b"\x83\x73\xc4\x00") + try: + unpacker.unpack() + except (ValueError, msgpack.OutOfData): + pass + try: + unpacker.skip() + except (ValueError, msgpack.OutOfData): + pass + with tempfile.TemporaryDirectory(prefix="pip-tls-") as name: + root = Path(name) + cert, key = root / "cert.pem", root / "key.pem" + subprocess.run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", + "-days", "1", "-subj", "/CN=localhost", "-addext", + "subjectAltName=DNS:localhost", "-keyout", str(key), "-out", str(cert)], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = b"trusted fixture" + self.send_response(200); self.send_header("Content-Length", str(len(body))) + self.end_headers(); self.wfile.write(body) + def log_message(self, *_): + pass + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.minimum_version = ssl.TLSVersion.TLSv1_2 + context.load_cert_chain(cert, key) + server.socket = context.wrap_socket(server.socket, server_side=True) + thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start() + try: + port = server.server_address[1] + session = requests.Session(); session.trust_env = False + assert session.get(f"https://localhost:{port}/", verify=str(cert), timeout=5).status_code == 200 + for url, verify in ((f"https://localhost:{port}/", True), + (f"https://127.0.0.1:{port}/", str(cert))): + try: + session.get(url, verify=verify, timeout=5) + except requests.exceptions.SSLError: + pass + else: + raise AssertionError("untrusted or wrong-host TLS was accepted") + finally: + server.shutdown(); server.server_close(); thread.join() + subprocess.run([sys.executable, "-m", "pip", "--version"], check=True) + subprocess.run([sys.executable, "-m", "pip", "check"], check=True) + result.update(tls_trusted="accepted", tls_untrusted="rejected", tls_wrong_host="rejected", + msgpack_implementation=msgpack.Unpacker.__module__, cli="passed") + print(json.dumps(result)) + + +if __name__ == "__main__": + if sys.flags.optimize: + raise RuntimeError("Security regression tests must not run with Python optimization") + main() diff --git a/scripts/pip-security/test-pip-vex.py b/scripts/pip-security/test-pip-vex.py new file mode 100644 index 00000000..9baa3e1c --- /dev/null +++ b/scripts/pip-security/test-pip-vex.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Negative controls for the exact pip-vendor VEX boundary.""" +from copy import deepcopy +import importlib.util +from pathlib import Path +import unittest + +spec = importlib.util.spec_from_file_location("pip_vex", Path(__file__).with_name("filter-pip-vendor-findings.py")) +vex = importlib.util.module_from_spec(spec) +spec.loader.exec_module(vex) + + +class VexBoundary(unittest.TestCase): + def setUp(self): + self.proof = {"verifiedWheelSha256": vex.WHEEL_SHA, "pip": "26.2.2+pasturestack.1", + "urllib3": "2.8.0", "msgpackImplementation": "pip._vendor.msgpack.fallback", + "setuptoolsPackageAbsent": True, "vendorFilesByteCompared": 299, + "notAffected": [{"id": key, "package": value[0], "version": value[1], + "detail": "Test fixture only"} for key, value in vex.RULES.items()]} + self.raw = {"Results": [{"Type": "python-pkg", "Packages": [ + {"Name": name, "Version": version, "AnalyzedBy": "sbom"} + for name, version in set(vex.RULES.values())], "Vulnerabilities": [ + {"VulnerabilityID": key, "PkgName": value[0], "InstalledVersion": value[1]} + for key, value in vex.RULES.items()]}]} + + def test_exact_slice_only(self): + original = deepcopy(self.raw) + result, document = vex.apply(self.raw, self.proof, "urn:test:exact-image") + self.assertEqual(result["Results"][0]["Vulnerabilities"], []) + self.assertEqual(len(document["statements"]), 4) + self.assertEqual(self.raw, original, "raw evidence must not be mutated") + + def test_invalid_proof_fails_closed(self): + for key, value in [("verifiedWheelSha256", "0" * 64), ("urllib3", "2.7.0"), + ("vendorFilesByteCompared", 300), ("setuptoolsPackageAbsent", False)]: + proof = dict(self.proof, **{key: value}) + with self.subTest(key=key), self.assertRaises(ValueError): + vex.apply(self.raw, proof, "urn:test:image") + + def test_full_distribution_is_not_excluded(self): + for package in self.raw["Results"][0]["Packages"]: + package["AnalyzedBy"] = "python-pkg" + result, document = vex.apply(self.raw, self.proof, "urn:test:image") + self.assertEqual(len(result["Results"][0]["Vulnerabilities"]), 4) + self.assertEqual(document["statements"], []) + + def test_missing_packages_is_not_excluded(self): + self.raw["Results"][0]["Packages"] = [] + result, _ = vex.apply(self.raw, self.proof, "urn:test:image") + self.assertEqual(len(result["Results"][0]["Vulnerabilities"]), 4) + + def test_wrong_version_is_not_excluded(self): + for finding in self.raw["Results"][0]["Vulnerabilities"]: + finding["InstalledVersion"] = "999.0" + result, document = vex.apply(self.raw, self.proof, "urn:test:image") + self.assertEqual(len(result["Results"][0]["Vulnerabilities"]), 4) + self.assertEqual(document["statements"], []) + + def test_unknown_advisory_and_urllib3_are_retained(self): + unknown = {"VulnerabilityID": "CVE-TEST-UNKNOWN", "PkgName": "setuptools", "InstalledVersion": "70.3.0"} + urllib = {"VulnerabilityID": "CVE-2026-97689", "PkgName": "urllib3", "InstalledVersion": "2.7.0"} + self.raw["Results"][0]["Vulnerabilities"] += [unknown, urllib] + result, _ = vex.apply(self.raw, self.proof, "urn:test:image") + self.assertEqual(result["Results"][0]["Vulnerabilities"], [unknown, urllib]) + + +if __name__ == "__main__": + unittest.main()