diff --git a/.github/workflows/exact-artifact-sbom-attestation.yml b/.github/workflows/exact-artifact-sbom-attestation.yml index b038c5478e..198392aa0c 100644 --- a/.github/workflows/exact-artifact-sbom-attestation.yml +++ b/.github/workflows/exact-artifact-sbom-attestation.yml @@ -78,19 +78,39 @@ jobs: - name: Materialize immutable trusted verifier uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - # job.workflow_repository/workflow_sha are not real Actions context - # properties (actionlint-flagged); this always resolved to an empty - # repository/ref, silently defaulting checkout away from the pinned - # trusted verifier source. ContextualWisdomLab/.github is this - # workflow's own repository; github.workflow_sha is the real, - # documented property for its pinned commit. + # Independently reviewed helper snapshot, not caller/called workflow SHA. repository: ContextualWisdomLab/.github - ref: ${{ github.workflow_sha }} + # Reviewed helper revision; intentionally distinct from workflow revision. + ref: 00c6551183cca101cfc97c43656a17cc2491c1b4 path: trusted-intake persist-credentials: false - sparse-checkout: scripts/ci/verify_exact_artifact_sbom_handoff.py + sparse-checkout: | + scripts/ci/ + requirements-strix-ci-hashes.txt sparse-checkout-cone-mode: false + - name: Verify fixed helper checkout identity + env: + HELPER_ROOT: trusted-intake + CALLER_WORKFLOW_SHA: ${{ github.workflow_sha }} + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + expected=00c6551183cca101cfc97c43656a17cc2491c1b4 + test "$(git -C "$HELPER_ROOT" rev-parse HEAD)" = "$expected" + origin="$(git -C "$HELPER_ROOT" remote get-url origin)" + case "$origin" in + https://github.com/ContextualWisdomLab/.github|https://github.com/ContextualWisdomLab/.github.git) ;; + *) echo "Foreign helper repository" >&2; exit 1 ;; + esac + test "$(git -C "$HELPER_ROOT" rev-parse HEAD:scripts/ci)" = bf26d3eefdb71fe79b855d941ffb46eb432b2f76 + test "$(git -C "$HELPER_ROOT" rev-parse HEAD:requirements-strix-ci-hashes.txt)" = 9e705850b5ce53c7fe836bc3df3a18771151e3f6 + git -C "$HELPER_ROOT" diff --exit-code HEAD -- scripts/ci requirements-strix-ci-hashes.txt + test -f "$HELPER_ROOT/scripts/ci/release_dependency_gate.py" + test -f "$HELPER_ROOT/scripts/ci/verify_exact_artifact_sbom_handoff.py" + test -f "$HELPER_ROOT/requirements-strix-ci-hashes.txt" + printf 'helper_repository=ContextualWisdomLab/.github helper_sha=%s caller_workflow_sha=%s\n' "$expected" "$CALLER_WORKFLOW_SHA" + + - name: Verify immutable same-run artifact metadata env: GH_TOKEN: ${{ github.token }} @@ -177,19 +197,39 @@ jobs: - name: Materialize immutable trusted verifier uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - # job.workflow_repository/workflow_sha are not real Actions context - # properties (actionlint-flagged); this always resolved to an empty - # repository/ref, silently defaulting checkout away from the pinned - # trusted verifier source. ContextualWisdomLab/.github is this - # workflow's own repository; github.workflow_sha is the real, - # documented property for its pinned commit. + # Independently reviewed helper snapshot, not caller/called workflow SHA. repository: ContextualWisdomLab/.github - ref: ${{ github.workflow_sha }} + # Reviewed helper revision; intentionally distinct from workflow revision. + ref: 00c6551183cca101cfc97c43656a17cc2491c1b4 path: trusted-signer persist-credentials: false - sparse-checkout: scripts/ci/verify_exact_artifact_sbom_handoff.py + sparse-checkout: | + scripts/ci/ + requirements-strix-ci-hashes.txt sparse-checkout-cone-mode: false + - name: Verify fixed helper checkout identity + env: + HELPER_ROOT: trusted-signer + CALLER_WORKFLOW_SHA: ${{ github.workflow_sha }} + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + expected=00c6551183cca101cfc97c43656a17cc2491c1b4 + test "$(git -C "$HELPER_ROOT" rev-parse HEAD)" = "$expected" + origin="$(git -C "$HELPER_ROOT" remote get-url origin)" + case "$origin" in + https://github.com/ContextualWisdomLab/.github|https://github.com/ContextualWisdomLab/.github.git) ;; + *) echo "Foreign helper repository" >&2; exit 1 ;; + esac + test "$(git -C "$HELPER_ROOT" rev-parse HEAD:scripts/ci)" = bf26d3eefdb71fe79b855d941ffb46eb432b2f76 + test "$(git -C "$HELPER_ROOT" rev-parse HEAD:requirements-strix-ci-hashes.txt)" = 9e705850b5ce53c7fe836bc3df3a18771151e3f6 + git -C "$HELPER_ROOT" diff --exit-code HEAD -- scripts/ci requirements-strix-ci-hashes.txt + test -f "$HELPER_ROOT/scripts/ci/release_dependency_gate.py" + test -f "$HELPER_ROOT/scripts/ci/verify_exact_artifact_sbom_handoff.py" + test -f "$HELPER_ROOT/requirements-strix-ci-hashes.txt" + printf 'helper_repository=ContextualWisdomLab/.github helper_sha=%s caller_workflow_sha=%s\n' "$expected" "$CALLER_WORKFLOW_SHA" + + - name: Verify immutable same-run artifact metadata env: GH_TOKEN: ${{ github.token }} @@ -275,9 +315,7 @@ jobs: - name: Verify online and prepare offline bundles env: GH_TOKEN: ${{ github.token }} - # job.workflow_repository is not a real Actions context property - # (actionlint-flagged); ContextualWisdomLab/.github is this workflow's - # own repository, matching the pinned checkout above. + # Signer repository is fixed independently of the caller identity. SIGNER_REPOSITORY: ContextualWisdomLab/.github PREDICATE_TYPE: ${{ inputs.predicate_type }} SOURCE_REPOSITORY: ${{ inputs.source_repository }} @@ -400,4 +438,4 @@ jobs: name: exact-artifact-sbom-offline-verification path: offline-attestation-evidence if-no-files-found: error - retention-days: 90 \ No newline at end of file + retention-days: 90 diff --git a/.github/workflows/release-dependency-license-strix-gate.yml b/.github/workflows/release-dependency-license-strix-gate.yml new file mode 100644 index 0000000000..2d31a6835a --- /dev/null +++ b/.github/workflows/release-dependency-license-strix-gate.yml @@ -0,0 +1,631 @@ +name: Release Dependency License and Strix Gate + +# Central pre-publish dependency gate (issue #2342). A release workflow calls +# this BEFORE it publishes anything. There is no neutral outcome: the gate +# either succeeds or the release is refused. The organization's scheduled SBOM +# roll-up (scripts/ci/sbom_inventory_aggregator.py) is informational governance +# reporting and is deliberately not reused here. +# +# On success the job's outputs are exactly the inputs of +# .github/workflows/exact-artifact-sbom-attestation.yml, so the caller chains: +# +# gate: +# uses: ContextualWisdomLab/.github/.github/workflows/release-dependency-license-strix-gate.yml@ +# secrets: inherit +# attest: +# needs: gate +# uses: ContextualWisdomLab/.github/.github/workflows/exact-artifact-sbom-attestation.yml@ +# with: +# source_repository: ${{ needs.gate.outputs.source_repository }} +# ... +# +# Provenance then covers exactly the bytes the gate examined. + +on: + workflow_call: + inputs: + source_repository: + description: Release repository in owner/name form. + required: true + type: string + source_sha: + description: Exact release head commit SHA. + required: true + type: string + ecosystems: + description: Comma-separated ecosystems to enumerate (python and/or cargo). + required: true + type: string + python_lock_path: + description: Hash-pinned Python lock installed into the build environment. + required: false + type: string + default: "" + cargo_manifest_path: + description: Release Cargo.toml whose Cargo.lock and build graph are gated. + required: false + type: string + default: "" + build_artifact_name: + description: Caller artifact holding the exact wheel and sdist to publish. + required: true + type: string + build_artifact_id: + description: Immutable upload-artifact ID from this caller run. + required: true + type: string + build_artifact_digest: + description: Expected sha256-prefixed artifact digest from the producer upload. + required: true + type: string + wheel_filename: + description: Exact wheel filename inside the build artifact. + required: true + type: string + sdist_filename: + description: Exact source distribution filename inside the build artifact. + required: true + type: string + evidence_artifact_name: + description: Unique per-call sealed evidence name; also namespaces diagnostic reports. The default retains legacy report names. + required: false + type: string + default: release-dependency-sealed-evidence + # The five provider credentials are declared optional so the licence stage, + # which needs none of them, can run on a review-only negative fixture that + # never reaches Strix. Optional is not lenient: the Strix stage refuses to + # start unless all five are present (STRIX_CREDENTIALS_ABSENT), so an allowed + # input that reaches Strix without credentials fails closed rather than being + # skipped, neutralized, or passed. `required: false` exists so a caller that + # passes no secrets fails on the *licence decision* rather than on + # `workflow_call` schema validation — a schema error is not evidence of a + # licence denial or of Strix being blocked. It is not an invitation to supply + # dummy secrets, and it does not widen any caller's secret exposure. + secrets: + BYTEZ_API_KEY: + required: false + NVIDIA_NIM_API_KEY: + required: false + NVIDIA_NIM_API_KEY_SUB: + required: false + OPENROUTER_API_KEY: + required: false + OPENAI_API_KEY: + required: false + outputs: + source_repository: + description: Gated release repository. + value: ${{ jobs.gate.outputs.source_repository }} + source_sha: + description: Gated release head SHA. + value: ${{ jobs.gate.outputs.source_sha }} + evidence_artifact_id: + description: Immutable same-run sealed evidence artifact ID. + value: ${{ jobs.gate.outputs.evidence_artifact_id }} + evidence_artifact_name: + description: Sealed evidence artifact name. + value: ${{ jobs.gate.outputs.evidence_artifact_name }} + evidence_artifact_digest: + description: Sealed evidence artifact digest in sha256: form. + value: ${{ jobs.gate.outputs.evidence_artifact_digest }} + wheel_filename: + description: Exact gated wheel filename. + value: ${{ jobs.gate.outputs.wheel_filename }} + wheel_sha256: + description: SHA-256 of the exact gated wheel. + value: ${{ jobs.gate.outputs.wheel_sha256 }} + wheel_sbom_filename: + description: CycloneDX SBOM filename for the gated wheel. + value: ${{ jobs.gate.outputs.wheel_sbom_filename }} + wheel_sbom_sha256: + description: SHA-256 of the gated wheel's CycloneDX SBOM. + value: ${{ jobs.gate.outputs.wheel_sbom_sha256 }} + sdist_filename: + description: Exact gated source distribution filename. + value: ${{ jobs.gate.outputs.sdist_filename }} + sdist_sha256: + description: SHA-256 of the exact gated source distribution. + value: ${{ jobs.gate.outputs.sdist_sha256 }} + sdist_sbom_filename: + description: CycloneDX SBOM filename for the gated source distribution. + value: ${{ jobs.gate.outputs.sdist_sbom_filename }} + sdist_sbom_sha256: + description: SHA-256 of the gated source distribution's CycloneDX SBOM. + value: ${{ jobs.gate.outputs.sdist_sbom_sha256 }} + source_identity_sha256: + description: SHA-256 of the sealed source-identity.json. + value: ${{ jobs.gate.outputs.source_identity_sha256 }} + checksum_sha256: + description: SHA-256 of the sealed checksums.sha256. + value: ${{ jobs.gate.outputs.checksum_sha256 }} + predicate_type: + description: Canonical CycloneDX in-toto predicate type. + value: ${{ jobs.gate.outputs.predicate_type }} + cyclonedx_schema: + description: Canonical CycloneDX 1.7 schema URL. + value: ${{ jobs.gate.outputs.cyclonedx_schema }} + +permissions: + contents: read + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +jobs: + gate: + name: Gate every resolved dependency before publishing + runs-on: ubuntu-24.04 + timeout-minutes: 360 + permissions: + contents: read + actions: read + outputs: + source_repository: ${{ steps.seal.outputs.source_repository }} + source_sha: ${{ steps.seal.outputs.source_sha }} + evidence_artifact_id: ${{ steps.sealed-evidence.outputs.artifact-id }} + evidence_artifact_name: ${{ steps.seal.outputs.evidence_artifact_name }} + evidence_artifact_digest: sha256:${{ steps.sealed-evidence.outputs.artifact-digest }} + wheel_filename: ${{ steps.seal.outputs.wheel_filename }} + wheel_sha256: ${{ steps.seal.outputs.wheel_sha256 }} + wheel_sbom_filename: ${{ steps.seal.outputs.wheel_sbom_filename }} + wheel_sbom_sha256: ${{ steps.seal.outputs.wheel_sbom_sha256 }} + sdist_filename: ${{ steps.seal.outputs.sdist_filename }} + sdist_sha256: ${{ steps.seal.outputs.sdist_sha256 }} + sdist_sbom_filename: ${{ steps.seal.outputs.sdist_sbom_filename }} + sdist_sbom_sha256: ${{ steps.seal.outputs.sdist_sbom_sha256 }} + source_identity_sha256: ${{ steps.seal.outputs.source_identity_sha256 }} + checksum_sha256: ${{ steps.seal.outputs.checksum_sha256 }} + predicate_type: ${{ steps.seal.outputs.predicate_type }} + cyclonedx_schema: ${{ steps.seal.outputs.cyclonedx_schema }} + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - name: Materialize immutable trusted gate + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: ContextualWisdomLab/.github + # Reviewed helper revision; intentionally distinct from workflow revision. + ref: 00c6551183cca101cfc97c43656a17cc2491c1b4 + path: trusted-gate + persist-credentials: false + # The whole scripts/ci tree, not an enumerated file list: the trusted + # Strix gate, the orchestrator sidecar and the token loader each source + # siblings by their own directory (strix_model_utils.sh, + # sanitize_contextual_orchestrator_sidecar_stream.py, + # install_strix_timeout_compat.py, strix_timeout_compat.py, …), and an + # enumeration silently breaks the moment one of them gains another. + sparse-checkout: | + scripts/ci/ + requirements-strix-ci-hashes.txt + sparse-checkout-cone-mode: false + + - name: Verify fixed helper checkout identity + env: + HELPER_ROOT: trusted-gate + CALLER_WORKFLOW_SHA: ${{ github.workflow_sha }} + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + expected=00c6551183cca101cfc97c43656a17cc2491c1b4 + test "$(git -C "$HELPER_ROOT" rev-parse HEAD)" = "$expected" + origin="$(git -C "$HELPER_ROOT" remote get-url origin)" + case "$origin" in + https://github.com/ContextualWisdomLab/.github|https://github.com/ContextualWisdomLab/.github.git) ;; + *) echo "Foreign helper repository" >&2; exit 1 ;; + esac + test "$(git -C "$HELPER_ROOT" rev-parse HEAD:scripts/ci)" = bf26d3eefdb71fe79b855d941ffb46eb432b2f76 + test "$(git -C "$HELPER_ROOT" rev-parse HEAD:requirements-strix-ci-hashes.txt)" = 9e705850b5ce53c7fe836bc3df3a18771151e3f6 + git -C "$HELPER_ROOT" diff --exit-code HEAD -- scripts/ci requirements-strix-ci-hashes.txt + test -f "$HELPER_ROOT/scripts/ci/release_dependency_gate.py" + test -f "$HELPER_ROOT/scripts/ci/verify_exact_artifact_sbom_handoff.py" + test -f "$HELPER_ROOT/requirements-strix-ci-hashes.txt" + printf 'helper_repository=ContextualWisdomLab/.github helper_sha=%s caller_workflow_sha=%s\n' "$expected" "$CALLER_WORKFLOW_SHA" + + - name: Validate the exact release identity before anything else runs + env: + SOURCE_REPOSITORY: ${{ inputs.source_repository }} + SOURCE_SHA: ${{ inputs.source_sha }} + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + # `workflow_call` can only type these inputs as `string`, so a branch + # name or a short SHA would otherwise be accepted here and only caught + # by the gate's own 40-hex check after Strix had already run. The shape + # is therefore checked by the trusted gate before the release head is + # even fetched, and long before any credential is materialized. + python3 -I trusted-gate/scripts/ci/release_dependency_gate.py validate-inputs \ + --source-repository "$SOURCE_REPOSITORY" \ + --source-sha "$SOURCE_SHA" + + - name: Check out the exact release head + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: ${{ inputs.source_repository }} + ref: ${{ inputs.source_sha }} + path: release-source + persist-credentials: false + + - name: Set up the release build interpreter + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + + - name: Verify immutable same-run build artifact metadata + env: + GH_TOKEN: ${{ github.token }} + SOURCE_REPOSITORY: ${{ inputs.source_repository }} + ARTIFACT_ID: ${{ inputs.build_artifact_id }} + ARTIFACT_NAME: ${{ inputs.build_artifact_name }} + ARTIFACT_DIGEST: ${{ inputs.build_artifact_digest }} + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + # Reuse the exact-artifact attestation's same-run metadata boundary. + test "$SOURCE_REPOSITORY" = "$GITHUB_REPOSITORY" + [[ "$ARTIFACT_ID" =~ ^[1-9][0-9]*$ ]] + [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]] + artifact_json="$(gh api "/repos/${SOURCE_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")" + jq -e \ + --arg id "$ARTIFACT_ID" \ + --arg name "$ARTIFACT_NAME" \ + --arg digest "$ARTIFACT_DIGEST" \ + --argjson run_id "$GITHUB_RUN_ID" \ + '(.id | tostring) == $id and .name == $name and .digest == $digest and .workflow_run.id == $run_id and .expired == false' \ + <<<"$artifact_json" >/dev/null + + - name: Download the exact distributions the caller intends to publish + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + artifact-ids: ${{ inputs.build_artifact_id }} + digest-mismatch: error + path: release-distributions + + - name: Collect the release closure without installing or executing it + env: + ECOSYSTEMS: ${{ inputs.ecosystems }} + PYTHON_LOCK_PATH: ${{ inputs.python_lock_path }} + CARGO_MANIFEST_PATH: ${{ inputs.cargo_manifest_path }} + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + python_lock="" + cargo_manifest="" + if [ -n "$PYTHON_LOCK_PATH" ]; then + python_lock="${PWD}/release-source/${PYTHON_LOCK_PATH}" + fi + if [ -n "$CARGO_MANIFEST_PATH" ]; then + cargo_manifest="${PWD}/release-source/${CARGO_MANIFEST_PATH}" + fi + bash trusted-gate/scripts/ci/release_dependency_capture_raw.sh \ + --raw-root "${RUNNER_TEMP}/raw" \ + --capture-root "${RUNNER_TEMP}/capture" \ + --ecosystems "$ECOSYSTEMS" \ + --python-lock "$python_lock" \ + --download-root "${RUNNER_TEMP}/collected" \ + --cargo-manifest "$cargo_manifest" + + - name: Assemble per-dependency evidence and isolated synthetic fixtures + env: + SOURCE_REPOSITORY: ${{ inputs.source_repository }} + SOURCE_SHA: ${{ inputs.source_sha }} + ECOSYSTEMS: ${{ inputs.ecosystems }} + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + jq -n \ + --arg repository "$SOURCE_REPOSITORY" \ + --arg sha "$SOURCE_SHA" \ + --arg ecosystems "$ECOSYSTEMS" \ + '{source_repository: $repository, source_sha: $sha, + ecosystems: ($ecosystems | split(","))}' \ + > "${RUNNER_TEMP}/capture/release.json" + python3 -I trusted-gate/scripts/ci/release_dependency_gate.py capture \ + --raw "${RUNNER_TEMP}/raw" \ + --capture "${RUNNER_TEMP}/capture" + + - name: Refuse a denied or unverifiable licence before any credential exists + id: license-stage + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + # The licence determination runs here, ahead of every credentialed and + # model step, using the *same* evaluate_dependency_license path the final + # gate uses — so a GPL/LGPL/AGPL dependency, an UNKNOWN licence, or an + # `OR` expression with no recorded permissive selection refuses the + # release before a provider secret is ever read. `capture` alone does not + # reject a licence; it only assembles evidence and fixtures. This stage + # also performs the full-set scope comparison, so an ecosystem whose + # membership cannot be established fails here too. + python3 -I trusted-gate/scripts/ci/release_dependency_gate.py prescreen \ + --capture "${RUNNER_TEMP}/capture" \ + --report "${RUNNER_TEMP}/license-report.json" + + - name: Install the prescreened closure into a lock-only environment + if: ${{ inputs.python_lock_path != '' }} + env: + PYTHON_LOCK_PATH: ${{ inputs.python_lock_path }} + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + # Installing runs dependency code, so it happens only after the licence + # stage above has passed, and only from the bytes that stage judged: + # --no-index --find-links over the collected distributions, with + # --require-hashes so pip proves each file against the lock. Nothing is + # re-resolved or re-downloaded, so the installed bytes are the inspected + # bytes even when the lock records several hashes for a project. + # --without-pip keeps the environment's contents exactly what the lock + # installed, so LOCK_ENV_MISMATCH means a real disagreement. + python3 -m venv --without-pip "${RUNNER_TEMP}/gate-venv" + bash trusted-gate/scripts/ci/release_dependency_capture_raw.sh \ + --install-gated \ + --python-lock "${PWD}/release-source/${PYTHON_LOCK_PATH}" \ + --python-interpreter "${RUNNER_TEMP}/gate-venv/bin/python" \ + --capture-root "${RUNNER_TEMP}/capture" \ + --download-root "${RUNNER_TEMP}/collected" \ + --license-report "${RUNNER_TEMP}/license-report.json" + + - name: Require every Strix provider credential before the Strix stage starts + env: + BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} + NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + # The secrets are optional on the contract so the licence stage above can + # run without them. An allowed input that gets this far must still be + # scanned, so absence is a refusal with STRIX_CREDENTIALS_ABSENT. This is + # deliberately not an `if:` condition: a condition would *skip* the Strix + # stage and let the release proceed unscanned. The reason code names only + # the absent variables and never echoes or measures a present value. + python3 -I trusted-gate/scripts/ci/release_dependency_gate.py require-strix-credentials + + - name: Provision the zero-cost review gateway for Strix + env: + BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} + NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + bash trusted-gate/scripts/ci/contextual_orchestrator_review_sidecar.sh + + # Scope boundary, recorded rather than left implicit. The steps below install + # the *gate's own* toolchain — this repository's hash-pinned + # requirements-strix-ci-hashes.txt, materialized from github.workflow_sha, and + # the orchestrator sidecar's own pinned lock. They are a different trust domain + # from the caller's release closure: they are pinned and reviewed in this + # repository, and the licence stage that judges the closure cannot judge the + # scanner it has to run first without a cycle. They are therefore NOT covered by + # the prescreen above, and that is a stated limit, not an exemption: bringing + # the gate's own dependencies under a licence verdict is an owner decision, + # tracked separately, and nothing here may be read as evidence that it happened. + - name: Install the pinned Strix toolchain + working-directory: trusted-gate + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + # Mirrors .github/workflows/strix.yml's install invariants: a private + # umask so the credential-bearing console script is not group-writable, + # --no-deps because strix-agent declares cryptography<49 against this + # repository's cryptography==50.0.0 security pin (see + # requirements-strix-ci-overrides.txt, #952), and an absolute, + # non-symlinked executable inside the interpreter's own scripts root. + umask 022 + python3 -m pip install --disable-pip-version-check --no-cache-dir \ + --require-hashes --no-deps -r requirements-strix-ci-hashes.txt + strix_executable="" + if command -v strix >/dev/null 2>&1; then + strix_executable="$(command -v strix)" + fi + if [ -z "$strix_executable" ] || [[ "$strix_executable" != /* ]] \ + || [ ! -f "$strix_executable" ] || [ -L "$strix_executable" ] \ + || [ ! -x "$strix_executable" ]; then + echo "::error::Pinned Strix installation did not produce a trusted absolute executable path." + exit 1 + fi + case "$strix_executable" in + "$GITHUB_WORKSPACE"/*|"$RUNNER_TEMP"/*) + echo "::error::Refusing a Strix executable from a workspace or runner-temp path." + exit 1 + ;; + esac + strix_scripts_root="$(python3 -c 'import sysconfig; print(sysconfig.get_path("scripts"))')" + if [ -z "$strix_scripts_root" ] || [[ "$strix_scripts_root" != /* ]] \ + || [ ! -d "$strix_scripts_root" ] || [ -L "$strix_scripts_root" ]; then + echo "::error::Pinned Strix installation did not produce a trusted absolute scripts root." + exit 1 + fi + case "$strix_executable" in + "$strix_scripts_root"/*) ;; + *) + echo "::error::Pinned Strix executable is outside the trusted scripts root." + exit 1 + ;; + esac + chmod go-w -- "$strix_scripts_root" "$strix_executable" + { + printf 'STRIX_EXECUTABLE_PATH=%s\n' "$strix_executable" + printf 'STRIX_EXECUTABLE_ROOT=%s\n' "$strix_scripts_root" + printf 'STRIX_EXECUTABLE_SHA256=%s\n' \ + "$(sha256sum "$strix_executable" | cut -d' ' -f1)" + } >> "$GITHUB_ENV" + + - name: Bind the zero-cost model, key, and API base for Strix + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + source trusted-gate/scripts/ci/load_contextual_orchestrator_token.sh + sanitized="$(printf '%s' "${CONTEXTUAL_ORCHESTRATOR_TOKEN:-}" | tr -d '\r\n')" + trimmed="$(printf '%s' "$sanitized" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [ -z "$trimmed" ]; then + echo '::error::CONTEXTUAL_ORCHESTRATOR_TOKEN is required for Strix scans.' + exit 1 + fi + echo "::add-mask::${trimmed}" + sidecar_base="${CONTEXTUAL_ORCHESTRATOR_BASE_URL:-}" + if [ "$sidecar_base" != "http://127.0.0.1:18080" ]; then + echo '::error::Strix sidecar base URL is not the pinned local gateway origin.' + exit 1 + fi + umask 077 + printf '%s' "$trimmed" > "${RUNNER_TEMP}/llm_api_key.txt" + printf '%s/v1' "${sidecar_base%/}" > "${RUNNER_TEMP}/llm_api_base.txt" + printf '%s' 'orchestrator/free' > "${RUNNER_TEMP}/strix_llm.txt" + { + printf 'LLM_API_KEY_FILE=%s\n' "${RUNNER_TEMP}/llm_api_key.txt" + printf 'LLM_API_BASE_FILE=%s\n' "${RUNNER_TEMP}/llm_api_base.txt" + printf 'STRIX_LLM_FILE=%s\n' "${RUNNER_TEMP}/strix_llm.txt" + } >> "$GITHUB_ENV" + + - name: Run Strix against one isolated synthetic fixture per dependency + env: + STRIX_LLM_DEFAULT_PROVIDER: contextual_orchestrator + STRIX_REASONING_EFFORT: none + STRIX_FALLBACK_MODELS: "" + STRIX_FAIL_ON_PROVIDER_SIGNAL: "1" + STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM + STRIX_DISABLE_PR_SCOPING: "1" + # Narrow the scan to the fixture subdirectory. The trusted gate + # requires its evidence binder at $STRIX_REPO_ROOT/scripts/ci on + # current main, so the workspace root necessarily holds that module; + # scanning "./" would pentest the trusted binder itself and fail every + # dependency on a finding against code this PR did not ship. + STRIX_TARGET_PATH: fixture + STRIX_SOURCE_DIRS: "." + # strix_quick_gate.sh treats PR_NUMBER/PR_BASE_SHA/PR_HEAD_SHA as + # optional and defaults IS_PR_EVIDENCE_RUN to false; this is a release + # scan, not PR evidence, so the value is stated rather than implied. + IS_PR_EVIDENCE_RUN: "false" + NPM_CONFIG_IGNORE_SCRIPTS: "true" + PNPM_CONFIG_IGNORE_SCRIPTS: "true" + YARN_ENABLE_SCRIPTS: "false" + BUN_CONFIG_IGNORE_SCRIPTS: "true" + SOURCE_SHA: ${{ inputs.source_sha }} + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + # docs/product-goal-directive.md section 8: elapsed inference time is + # never converted into a model-failure verdict. + export LLM_TIMEOUT=0 + export STRIX_MEMORY_COMPRESSOR_TIMEOUT=0 + export STRIX_PROCESS_TIMEOUT_SECONDS=0 + export STRIX_TOTAL_TIMEOUT_SECONDS=0 + trusted_gate_root="${PWD}/trusted-gate" + mkdir -p "${RUNNER_TEMP}/capture/strix/bindings" + for fixture in "${RUNNER_TEMP}"/capture/strix/fixtures/*.json; do + slug="$(basename "$fixture" .json)" + workspace="${RUNNER_TEMP}/strix-workspace/${slug}" + mkdir -p "$workspace/scripts/ci" "$workspace/fixture" + cp "$fixture" "$workspace/fixture/fixture.json" + # sanitize_remediation_evidence_claims returns 2 when + # $REPO_ROOT/scripts/ci/strix_evidence_binding.py is absent, and + # REPO_ROOT is STRIX_REPO_ROOT on current main (its own script + # directory once #2291 lands). The trusted binder is therefore copied + # into each isolated workspace so both resolutions hold without + # editing strix_quick_gate.sh, which #2291 owns. It sits beside the + # scan target, never inside it. + cp "$trusted_gate_root/scripts/ci/strix_evidence_binding.py" \ + "$workspace/scripts/ci/strix_evidence_binding.py" + ( + cd "$workspace" && + STRIX_REPO_ROOT="$workspace" \ + bash "$trusted_gate_root/scripts/ci/strix_quick_gate.sh" + ) + vulnerabilities="" + if [ -d "$workspace/strix_runs" ]; then + vulnerabilities="$(find "$workspace/strix_runs" -type f \ + -name 'vulnerabilities.json' -print -quit)" + fi + if [ -z "$vulnerabilities" ]; then + echo "::warning::Strix produced no machine-readable findings file for ${slug};" \ + "the gate will refuse the release with STRIX_BINDING_MISSING." + continue + fi + # Normalize Strix's findings document to an array without inventing + # a result: an array is used as-is, an object carrying a + # `vulnerabilities` array contributes that array, and any other + # shape yields null so no binding is written at all and the gate + # refuses with STRIX_BINDING_MISSING. A textual summary can never + # become a pass here. + findings_json="$(jq -c ' + if type == "array" then . + elif type == "object" and (.vulnerabilities? | type) == "array" then .vulnerabilities + else null end' "$vulnerabilities")" + if [ "$findings_json" = "null" ]; then + echo "::warning::Strix findings for ${slug} are not a structured array;" \ + "the gate will refuse the release with STRIX_BINDING_MISSING." + continue + fi + jq -n \ + --argjson fixture "$(cat "$fixture")" \ + --argjson findings "$findings_json" \ + --arg sha "$SOURCE_SHA" \ + --arg digest "$(cat "${fixture%.json}.sha256")" \ + '{ + schema: "cwl.release-dependency-strix-binding/1", + dependency: $fixture.dependency, + fixture: { + id: ($fixture.dependency.ecosystem + "/" + $fixture.dependency.name + + "@" + $fixture.dependency.version), + sha256: $digest, + scenarios: ($fixture.scenarios | keys) + }, + source_sha: $sha, + findings: $findings, + verdict: (if ($findings | length) == 0 + then "no_exploitable_findings" else "findings_present" end) + }' > "${RUNNER_TEMP}/capture/strix/bindings/${slug}.json" + done + + - name: Refuse the release unless every dependency passes + id: full-stage + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + python3 -I trusted-gate/scripts/ci/release_dependency_gate.py gate \ + --capture "${RUNNER_TEMP}/capture" \ + --report "${RUNNER_TEMP}/gate-report.json" + + - name: Seal exactly the gated bytes for attestation + id: seal + env: + EVIDENCE_ARTIFACT_NAME: ${{ inputs.evidence_artifact_name }} + WHEEL_FILENAME: ${{ inputs.wheel_filename }} + SDIST_FILENAME: ${{ inputs.sdist_filename }} + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + python3 -I trusted-gate/scripts/ci/release_dependency_gate.py seal \ + --report "${RUNNER_TEMP}/gate-report.json" \ + --wheel "release-distributions/${WHEEL_FILENAME}" \ + --sdist "release-distributions/${SDIST_FILENAME}" \ + --evidence-root "${RUNNER_TEMP}/sealed-evidence" \ + --evidence-artifact-name "$EVIDENCE_ARTIFACT_NAME" + + - name: Export the sealed evidence as one immutable same-run artifact + id: sealed-evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0 + with: + name: ${{ inputs.evidence_artifact_name }} + path: ${{ runner.temp }}/sealed-evidence + if-no-files-found: error + + # Failure evidence has to survive the failure that produced it. A default + # (success-conditional) upload is skipped exactly when the gate refuses the + # release, which is when the report matters most. Each upload is therefore + # bound to the step that writes its report: it runs whether that step passed + # or failed, but not when that step never ran, and not on cancellation. This + # is narrower than a blanket `always()` on purpose — with + # `if-no-files-found: error`, a report that its own producing step should + # have written but did not stays a failure instead of being masked. Neither + # upload can rescue the run: a refusing stage has already failed the job, and + # a later successful step does not undo that. + - name: Export the pre-credential licence report + if: ${{ !cancelled() && steps.license-stage.conclusion != 'skipped' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0 + with: + name: ${{ inputs.evidence_artifact_name == 'release-dependency-sealed-evidence' && 'release-dependency-license-report' || format('release-dependency-license-report--{0}', inputs.evidence_artifact_name) }} + path: ${{ runner.temp }}/license-report.json + if-no-files-found: error + + - name: Export the per-dependency gate report + if: ${{ !cancelled() && steps.full-stage.conclusion != 'skipped' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0 + with: + name: ${{ inputs.evidence_artifact_name == 'release-dependency-sealed-evidence' && 'release-dependency-gate-report' || format('release-dependency-gate-report--{0}', inputs.evidence_artifact_name) }} + path: ${{ runner.temp }}/gate-report.json + if-no-files-found: error diff --git a/CHANGELOG.d/20260923-release-dependency-license-strix-gate.md b/CHANGELOG.d/20260923-release-dependency-license-strix-gate.md new file mode 100644 index 0000000000..784333b9c3 --- /dev/null +++ b/CHANGELOG.d/20260923-release-dependency-license-strix-gate.md @@ -0,0 +1,186 @@ +### Central pre-publish dependency gate: parsed license denial, resolved-graph reconciliation, per-dependency Strix bindings + +- `origin/main` had **no** fail-closed pre-publish dependency gate. The only license signal was + `scripts/ci/sbom_inventory_aggregator.py`, a *scheduled, informational* org SBOM roll-up that + flags GPL/AGPL/NOASSERTION for governance: it is not per-dependency, not fail-closed, and not + bound to a release head. That gap blocked fast-mlsirm's 0.11.5 PyPI release and + contextual-orchestrator's VCS-pin removal (#2342). +- New reusable `workflow_call` workflow `.github/workflows/release-dependency-license-strix-gate.yml` + runs **before** a release workflow publishes. It has no `continue-on-error`, no `if: always()`, + no neutral outcome, and no bypass; `permissions` is `contents: read` at both workflow and job + scope, and every action is pinned to the same commits `exact-artifact-sbom-attestation.yml` uses. + The decision code is materialized from `ContextualWisdomLab/.github` at `github.workflow_sha` + into `trusted-gate/`, so a caller's tree can never supply it. +- New `scripts/ci/spdx_license_policy.py` is a recursive-descent SPDX 2.3 expression parser + (`AND`/`OR`/`WITH`/parentheses/legacy `+`). Policy is applied to the parsed tree, never by + substring matching: GPL, LGPL, and AGPL are denied in every version and in both the `-only` and + `-or-later` spellings, an exception never rescues a denied base (`GPL-2.0-only WITH + Classpath-exception-2.0` stays denied), and `missing`, `NOASSERTION`, `NONE`, `UNKNOWN`, + `custom`, `LicenseRef-*`, and any unparseable expression fail closed. A dual-licensed dependency + passes only when a non-denied operand is explicitly selected with a written rationale, which is + copied into the artifact provenance as a CycloneDX component property. Bundled `LICENSE`, + `COPYING`, and `NOTICE` text *is* substring-scanned — correct for prose — so metadata claiming + MIT while shipping GPL text fails as a disagreement. +- New `scripts/ci/release_dependency_gate.py` enumerates both ecosystems and refuses any + asymmetry: the hash-pinned Python lock against `pip inspect` of the build environment + (`LOCK_ENV_MISMATCH`), and `Cargo.lock` against the full resolved build graph including + build-dependencies and every `cfg()`-gated target (`CARGO_LOCK_GRAPH_MISMATCH`, + `CARGO_CHECKSUM_MISSING`). It records name, version, source hash, license, license source, and + distribution inclusion per dependency; verifies the captured source hash against the pin + (`SOURCE_HASH_MISMATCH`); evaluates static and dynamic linking targets of shipped native + libraries against an explicit, auditable platform-runtime soname allowlist (glibc, the GCC + runtime-library-exception libraries, `libpython`) so a real compiled wheel can pass at all; and + runs deterministic archive-escape and install-hook detectors (`ARCHIVE_PATH_ESCAPE`, + `INSTALL_HOOK`). +- Strix evidence is accepted **only** as a machine-readable binding, one isolated synthetic + fixture per dependency, simulating file parsing, install hooks, archive traversal, native library + loading, credential/network attempts, and known-vulnerability surface. A textual "0 findings" or + "No exploitable vulnerabilities detected" is rejected (`STRIX_TEXTUAL_PASS_REJECTED`), and a + missing or malformed binding is a failure, never neutral (`STRIX_BINDING_MISSING`, + `STRIX_BINDING_MALFORMED`, `STRIX_BINDING_UNBOUND`). The trusted binder is resolved next to the + gate script's **own** directory, adopting `strix_quick_gate.sh`'s trusted-path semantics in new + code without touching that file (PR #2291 owns its one-line repair). +- On success the gate seals exactly the six members + `scripts/ci/verify_exact_artifact_sbom_handoff.py` expects — wheel, sdist, their CycloneDX 1.7 + SBOMs, `source-identity.json`, `checksums.sha256` — and emits all 17 inputs of + `exact-artifact-sbom-attestation.yml` as workflow outputs, so provenance covers exactly the bytes + that were gated. `tests/test_release_dependency_gate_capture_and_seal.py` proves the sealed + directory is accepted verbatim by that verifier. +- Strix itself is invoked through the organization's existing trusted entry point + `scripts/ci/strix_quick_gate.sh`, once per isolated fixture workspace via `STRIX_REPO_ROOT`, + with `strix.yml`'s bootstrap invariants mirrored verbatim (private install umask, + `--require-hashes --no-deps` against the unmodified `requirements-strix-ci-hashes.txt`, absolute + non-symlinked executable inside the interpreter's scripts root, `chmod go-w`, digest pinned into + `GITHUB_ENV`, sidecar-provided `LLM_API_KEY_FILE`/`LLM_API_BASE_FILE`/`STRIX_LLM_FILE`, and + `orchestrator/free` as the only accepted model). The trusted binder is copied into each fixture + workspace so the gate's binder lookup resolves both on current `main` and after #2291, without + editing that file. `strix_runs/**/vulnerabilities.json` is normalized to an array only when it + already is one (or carries a `vulnerabilities` array); any other shape writes no binding, so the + gate refuses with `STRIX_BINDING_MISSING` rather than inventing a result. +- `scripts/ci/release_dependency_capture_raw.sh` runs the runner-only tools (`pip inspect`, + `pip download`, `cargo metadata --locked`, `cargo fetch`, archive listing, `readelf -d`) and + writes their output verbatim; every decision lives in the unit-tested Python that reads it. It + inspects a `python3 -m venv --without-pip` environment holding exactly the lock, so the + no-exemption lock/environment rule is not defeated by setup-python's preinstalled `pip`, and it + fetches by exact pin with hash checking deliberately disabled so `SOURCE_HASH_MISMATCH` is + observable rather than pre-empted by pip. The gate adds no Python dependency and does not touch + any `anyio` pin or `requirements-strix-ci*` (#2278 owns that lane). Refs #2342. +- The gate now runs in **two stages**, so the licence determination precedes every credential and + model step. `release_dependency_gate.py prescreen` (`stage: license`) enumerates the full + dependency scope and applies the *same* `evaluate_dependency_license` decision the final gate + uses, reading no Strix binding and requiring no provider credential: a GPL/LGPL/AGPL dependency, + an `UNKNOWN`/missing licence, or an `OR` expression with no recorded permissive selection refuses + the release before a secret is read. `capture` alone never rejected a licence — it only assembles + evidence and fixtures — so making the secrets optional would not by itself have produced a + pre-Strix rejection. The five provider secrets are therefore declared `required: false`, which is + not leniency: `require-strix-credentials` refuses the Strix stage with `STRIX_CREDENTIALS_ABSENT` + when any is absent, as a failing command rather than an `if:` condition, because a condition would + *skip* the scan and let the release proceed unscanned. The reason code names only the absent + variables and never echoes or measures a present value. Only a `full`-stage report may be sealed, + so a passing prescreen can never stand in for the Strix stage. +- Dependency **scope is compared as a whole set**, per ecosystem, with `expected_count`, + `enumerated_count`, `collected_count`, and `matched_count` recorded in the report and equality + required. CO#1226 accepted coverage because one component of one ecosystem existed; an ecosystem + this gate cannot enumerate is now `SCOPE_UNVERIFIABLE` rather than silently skipped, a collected + set that is a subset of the producer's declared set is `SCOPE_SET_MISMATCH`, and so is capture + material for something no declared ecosystem expects. Scope is direct, transitive, build, dev, + optional and platform: `resolve_cargo_graph` walks every `resolve.nodes` edge regardless of + `dep_kind` or target `cfg`, so a UEFI-only crate such as `r-efi` is an expected member and gets no + target-based exemption. +- Licence metadata is read from **each fetched distribution's own** `METADATA`/`PKG-INFO`, by the + trusted gate's `distribution-metadata`, which also re-checks that the archive declares the pinned + project and version. It cannot come from `pip inspect` of the lock-only environment any more, + because no such environment exists yet when the licence is judged; the enumeration is built from + the same fetched set, in the `pip inspect` shape the lock/environment reconciliation already reads, + so identity and licence stay consistent by construction and an entry that is not present exactly + once is an error rather than a default. The previous metadata step ran `python3 -m pip show` + without the `--python` target its neighbours carried, so it inspected the *runner's* global + interpreter where the release dependencies are not installed at all. +- The exact release identity is shape-checked **first**. `workflow_call` can only type + `source_sha` as `string`, and the gate's own 40-hex check was reached only after Strix had run, so + `validate-inputs` now refuses a branch name or a short SHA before the release head is fetched. +- Failure evidence survives the failure that produced it: each report upload is bound to the step + that writes it, running whether that step passed or failed but not when it never ran and not on + cancellation. This is deliberately narrower than a blanket `always()`, and with + `if-no-files-found: error` a report that should have been written but was not stays a failure + instead of being masked. Neither upload can rescue the run. Refs #2342. +- **Install and capture now resolve from the same validated sources.** `pip install -r ` reads + the real lock and honors `--index-url`, `--extra-index-url` and `--find-links` in it, while the + capture step's `pip download` used a reconstructed plain requirements file built with + `grep -oE '^[A-Za-z0-9._-]+==[^ ;]+'`, which dropped every `-`-prefixed directive. Collection could + therefore resolve from a different source than install, and any release lock using a private or + extra index failed capture outright. The fix never forwards what the lock says: `lock-source-options` + parses each directive, validates it, and only then emits an explicit option list, reusing the + trusted-origin and bounded-path policy `materialize_base_python_requirements.py` already applies + (HTTPS, default port, host allowlist, no userinfo; normalized relative path with no `.`/`..` and + none of `\\ : ? #`). An unlisted origin is `LOCK_SOURCE_ORIGIN_DENIED`, a URL carrying userinfo is + `LOCK_SOURCE_CREDENTIAL_IN_URL` and withholds the whole URL from both the message and the report, a + path leaving the release tree is `LOCK_SOURCE_PATH_ESCAPE`, and a nested `-r`/`-c` include, an + environment marker, or any other directive form is `LOCK_SOURCE_UNSUPPORTED`. Nothing is dropped + silently, because silent dropping was the defect. The supported dialect is deliberately narrow and + this organization's own `requirements-*-hashes.txt` files use none of these forms. The options are + read into a bash array with the validator's exit status checked explicitly — *not* through + `mapfile < <(…)`, where `set -e` discards a refusal and it would read as "no options" and resolve + from the default index anyway. Source resolution decides only where pip looks: the hash pin still + decides what is acceptable, so `SOURCE_HASH_MISMATCH` remains observable and an offline + `--find-links` root cannot substitute different bytes. Refs #2342. +- **Nothing is installed before it has been adjudicated.** The gate's premise is that a denied, + unknown or untrusted dependency is refused before any of it runs, but the workflow installed the + whole release closure in a step that preceded *both* the lock-source validation and the licence + prescreen. A GPL/LGPL/AGPL or `UNKNOWN` dependency therefore reached the environment first, and a + lock pointing at an untrusted index had its directives honoured by that install while only the + later capture validated them — so the first network action of the run was the unvalidated one. The + order is now: validate the lock's sources (no network), collect the closure with + `pip download --no-deps --only-binary=:all:` (wheels only, because `pip download` executes an + sdist's build backend for metadata even with `--no-deps`), judge the licence, and only then + install. The install is `--require-hashes --only-binary=:all: --no-index --find-links ` + over the very bytes that were inspected, so nothing is re-resolved or re-downloaded and the + installed bytes are the judged bytes even where the lock records several hashes for one project — + which a second hash-less download could not have established. `install-authorized` refuses the + install unless a prescreen report records a passed `license` stage, so a missing, malformed or + failing report fails closed instead of defaulting to permitted. + One consequence is stated plainly rather than papered over: `LOCK_ENV_MISMATCH` is now evaluated + against the *collected* closure, because no installed environment exists when the gate reads its + capture. Agreement between that closure and the environment is enforced at install time instead, + by pip itself: `--require-hashes` with `--no-index --find-links ` can only install a + file from the collected root that matches a hash the lock records, so a disagreement fails the + install rather than being reported by a later inspect. + `tests/test_release_dependency_install_ordering.py` pins the wiring rather than the parser: with + `RELEASE_GATE_PIP` pointed at a recorder, a refused lock directive performs **no** pip call at all, + an unauthorized licence stage performs **no** `install`, an authorized release performs exactly one + offline hash-checked `install` from the collected root, and the workflow's step order is asserted + because the defect lived there. Refs #2342. +- **Three release-blocking defects found by independent review of `03ba1777`, each with its own + regression.** (1) *A permissive declaration was accepted as licence evidence.* The decision + allowed the declared SPDX expression and then only looked for a **denied** title in the bundled + text, so `scan_license_text` returning `None` was read as "the text is fine" — it only means no + GPL/LGPL/AGPL title was found. Reproduced: MIT metadata with `license_texts = {}`, with + `LICENSE = UNKNOWN`, and with `LICENSE = Commercial redistribution is prohibited.` each passed + the licence stage with an empty failure list. `recognize_license_text` is the positive half — + it returns the SPDX identifiers a body actually supports — so absent text is now + `LICENSE_TEXT_MISSING`, an unrecognizable body is `LICENSE_TEXT_UNVERIFIED`, and a recognized + body naming none of the declared identifiers is `LICENSE_TEXT_DISAGREEMENT`. Two of this + repository's own fixtures were declaring one licence while bundling another and are corrected. + (2) *The approval was not bound to what was installed.* `install_is_authorized` checked only + `stage` and `result`, and the install re-read the original lock, so a two-field report authorized + it and a lock recording several hashes for one project let `--require-hashes` accept an artifact + whose licence and contents were never judged. The verdict now records `python_lock_sha256`, and + `bind-install` refuses unless that lock still digests to what the verdict read, every judged + artifact is present in the collected root **by digest**, and the root holds no other + distribution; it then writes a requirements file pinning each project to the one judged digest, + which is what the install reads. A swapped artifact, an extra unjudged wheel, an edited lock and + a failing report each install nothing. (3) *The install could never run.* `python3 -m venv` + symlinks `bin/python` on POSIX, and the interpreter guard refused symlinks outright, so a normal + virtual environment exited 2 before pip was reached. The guard now resolves the link and requires + the resolved target to be a regular executable file, which a real venv satisfies while a dangling + link and a directory still fail. Refs #2342. +- **The gate's own toolchain is out of the prescreen's scope, and that limit is now written down + instead of being implicit.** The same review noted that the pinned Strix toolchain + (`requirements-strix-ci-hashes.txt`, materialized from `github.workflow_sha`) and the orchestrator + sidecar's own lock are installed without passing through the licence stage. They are a different + trust domain from the caller's release closure — pinned and reviewed in this repository — and the + stage that judges the closure cannot judge the scanner it must run first without a cycle. The + workflow says so at the install step: not an automatic exception for CI/build/dev dependencies, + but a stated limit whose removal is an owner decision tracked separately. Nothing in this gate's + output may be read as evidence that the gate's own dependencies were licence-judged. Refs #2342. + diff --git a/docs/doctoring/20260924-release-gate-negative-fixture-verification-plan.md b/docs/doctoring/20260924-release-gate-negative-fixture-verification-plan.md new file mode 100644 index 0000000000..6db5f96e41 --- /dev/null +++ b/docs/doctoring/20260924-release-gate-negative-fixture-verification-plan.md @@ -0,0 +1,205 @@ +# Negative-fixture verification plan for the central release dependency gate (#2342, #2347) + +Prepared, **not approved to run**. No hosted run, publish, merge, approval or re-run is authorized +by this document. It closes the two written gaps the coordinator required alongside the reviewable +exact head, and it records one refutation the owner must decide on before any run is scheduled. + +Central head this describes: `65727fa8411ec92672e03b1c6447b3a47d2616fc` on +`feat/release-dependency-license-strix-gate-2342`, on top of the reviewed +`3c3ca9b1445d4a73a9d47216ff88996f012f1757`. + +Two claims are kept apart throughout, and must stay apart in any report that cites this file: + +- **Reason codes verified locally.** Unit results from `pytest`, which prove a decision outcome and + its reason code and nothing else. +- **Run-level facts.** "The Strix step did not start", "the gate job concluded `failure`", "the + publish job did not start". None of these is established here. A failing unit-test wrapper is + never a real release-gate failure, and a passing test is never a release PASS. + +## Gap 1 — how a negative case enters the real gate + +### The path, by step, subcommand and function + +| # | Workflow step (`release-dependency-license-strix-gate.yml`) | Runs | Decides | +|---|---|---|---| +| 1 | `Validate the exact release identity before anything else runs` | `release_dependency_gate.py validate-inputs` | `validate_release_identity` — 40-hex `source_sha`, `owner/name` repository | +| 2 | `Install the release dependency closure into a lock-only environment` | `pip install --require-hashes --only-binary=:all: -r release-source/` into a `--without-pip` venv | — | +| 3 | `Download the exact distributions the caller intends to publish` | `actions/download-artifact` → `release-distributions/` | — | +| 4 | `Collect raw resolved-dependency evidence from both ecosystems` | `release_dependency_capture_raw.sh` | — (writes tool output verbatim; `capture_python` derives each `metadata.json` from `python/installed.json`) | +| 5 | `Assemble per-dependency evidence and isolated synthetic fixtures` | `release_dependency_gate.py capture` | `capture` → `build_evidence` → `evidence/.json`, `strix/fixtures/.json` | +| 6 | `Refuse a denied or unverifiable licence before any credential exists` | `release_dependency_gate.py prescreen` | **`gate(stage="license")` → `evaluate_dependency_license` → `declared_license_expression` → `spdx_license_policy.evaluate_license_expression`** | +| 7 | `Require every Strix provider credential before the Strix stage starts` | `release_dependency_gate.py require-strix-credentials` | `require_strix_credentials` → `STRIX_CREDENTIALS_ABSENT` | +| 8–11 | gateway, toolchain, credential binding, Strix | `strix_quick_gate.sh` per fixture workspace | — | +| 12 | `Refuse the release unless every dependency passes` | `release_dependency_gate.py gate` | `gate(stage="full")` — the same licence decision **plus** `validate_strix_binding` | +| 13 | `Seal exactly the gated bytes for attestation` | `release_dependency_gate.py seal` | `seal` — refuses a non-`PASS` **and** a non-`full` report | + +The licence decision in step 6 is the same function the final gate calls in step 12. There is one +decision implementation, not a prescreen copy of one. + +`needs` path traversed: the gate is a single `workflow_call` job (`jobs.gate`). A caller composes +`gate` → `attest`, and any mock job models only the edge out of `jobs.gate`. + +### No collection-bypass input exists + +Verified by reading the current source: + +- The workflow's `workflow_call` inputs are the release identity, ecosystems, lock/manifest paths, + artifact and filenames. **None of them skips capture, skips the licence stage, or injects a + verdict.** `test_workflow_is_reusable_and_never_branch_selectable` and + `test_gate_has_no_bypass_of_any_kind` pin the absence of a bypass shape. +- Step 4 always runs; step 5 always runs; step 6 always runs. The only `if:` conditions in the + workflow are the lock-only install guard and the two evidence-retention uploads + (`test_failure_evidence_survives_the_failure_that_produced_it` asserts there are exactly three). +- A hand-written `evidence/.json` cannot manufacture a case. The expected set comes from the + producer's own lock (`_enumerate_python`) and `Cargo.lock` (`_enumerate_cargo`), and + `_scope_rows` refuses collected material that no declared ecosystem expects with + `SCOPE_SET_MISMATCH`. Test: `test_collected_material_outside_every_expected_set_is_a_scope_mismatch`. + +**Consequence, stated plainly: a denial case cannot be fed in as a bare JSON blob.** It must arrive +as something the real capture path genuinely collects — a distribution present in the lock, with a +real `sha256`, whose own metadata carries the case. + +### The fixture-distribution shape: **refuted as currently specified** + +The proposed shape — a locally authored fixture wheel in the caller's artifact whose metadata +declares `GPL-3.0-only` (or omits `License` entirely for `LICENSE_MISSING`) — is the right *idea*, +because the case then rides on metadata the real collection reads. It does **not** work against the +unmodified capture script, for a reason the owner must decide on before any run is scheduled: + +- `parse_python_lock` skips every directive line (`if not line or line.startswith("-")`), so a lock + may carry `--find-links ./wheels` without the gate's parser objecting — this part is verified by + reading the parser. Step 2's `pip install` reads the **real lock**, and pip's documented + requirements-file behavior is to honor an in-file `--find-links` under `--require-hashes + --only-binary=:all:`. That last point is **pip's documented behavior, not executed here**; no step + of this workflow has been run. +- `release_dependency_capture_raw.sh:151` reconstructs a *plain* requirements file for `pip + download` with `grep -oE '^[A-Za-z0-9._-]+==[^ ;]+'`, which **drops every `-`-prefixed + directive**. Step 4's `pip download --no-deps --only-binary=:all:` therefore resolves against the + default index only. + +So a locally authored wheel installs in step 2 and then fails to download in step 4 +(`ERROR: no fetched distribution for ==`). The run would fail in collection, before +the licence decision — which is *not* the licence rejection the fixture is meant to demonstrate. + +This is also a latent production defect independent of the fixture: a real release whose lock +carries `--index-url`, `--extra-index-url` or `--find-links` has those dropped for the download, so +step 4 either fails or fetches from the wrong index while step 2 installed from the right one. +**Reported as a follow-up, not fixed here** — preserving index/find-links directives into the plain +requirements file changes production collection semantics and needs the owner's decision. + +Until that is resolved, the only collectible negative case is one whose distribution the default +index already serves, which conflicts with "never fetch or install any GPL/LGPL/AGPL package". + +### A GPL-declaring fixture package is rejected + +The coordinator has **retracted** the idea of authoring or installing a fixture package whose +metadata declares a copyleft identifier. It is not to be built. The two evidence classes are split +instead: + +- **Per-reason denial codes stay unit-level.** A self-authored, **data-only** SPDX string is a valid + input to the production decision functions, and `LICENSE_DENIED_GPL`, `LICENSE_DENIED_LGPL`, + `LICENSE_DENIED_AGPL`, `LICENSE_UNPARSEABLE`, `LICENSE_UNRECOGNIZED`, `LICENSE_MISSING`, + `LICENSE_SELECTION_REQUIRED` and `LICENSE_SELECTION_INVALID` are proven exactly there, by direct + calls to `evaluate_dependency_license` / `declared_license_expression` in the existing + `tests/test_release_dependency_gate.py`. Those SPDX strings are **not** extended into the real + package-install path. +- **The real capture path is exercised with a self-authored artifact containing no forbidden + source**, verified through a `LICENSE_MISSING` rejection. Nothing copyleft is fetched, declared or + installed at any point. + +Note that the refutation above still applies to the `LICENSE_MISSING` fixture, because it is also a +locally authored distribution: until the dropped-directive defect is decided, `pip download` in step +4 cannot fetch it. + +### Naming discipline for the eventual run + +What such a run can prove, and the only way it may be described: +**real collection → licence-missing rejection → Strix blocked → `mock_publish` gated by `needs`.** + +It is **not** a "GPL real-collection-refusal E2E" and must never be called one. The per-reason +copyleft denials are unit-level decision evidence and belong in a separate, separately labelled +section of any report. The link between the real capture path and the decision function is for the +coordinator to review from the exact-head source; it is not established by this prose. + +## Gap 2 — the `mock_publish` job's contract and its limits + +Name: **`mock_publish`**. Never `publish`, `release`, or `deploy`, so no reader or later script +mistakes it for the release job. + +What it verifies: **only that the gating edge behaves as the real caller's publish job would.** Its +`needs` and `if` must be character-identical to the real release workflow's publish job, and both +must be quoted side by side in the run's evidence. It models the *edge*, nothing else: it does not +show that a real publish job would not start, because it is not that job and does not share its +environment, permissions or triggers. + +Prohibited in `mock_publish`, and unnecessary for the contract: any `permissions:` beyond +`contents: read`, any token or secret, any `environment:`, any tag creation, any release creation, +any registry credential, any upload to a registry. Its steps are `echo` only. + +**The real publish job's `needs`/`if` cannot be quoted here.** The caller workflow is FMLS-owned and +is not present at this head, so the two conditions must be quoted from the FMLS caller at its exact +SHA when the run is proposed. This plan does not invent them. + +### Judgement rule and the exact fields to read + +A skipped job can still carry a `started_at` in GitHub's payload, so **nothing may be inferred from +an absent or present `started_at` alone.** Judge from the raw payload plus whether steps actually +executed: + +From `GET /repos/{owner}/{repo}/actions/runs/{run_id}` — `id`, `head_sha` (must equal the fixture +commit exactly), `status`, `conclusion`. + +From `GET /repos/{owner}/{repo}/actions/runs/{run_id}/jobs` per job — `name`, `status`, +`conclusion`, and the full `steps[]` array, reading each step's `name`, `status`, `conclusion` and +`number`. + +Decision rules: + +- **Gate refused**: the `gate` job has `conclusion == "failure"`, and the step named + `Refuse a denied or unverifiable licence before any credential exists` has + `conclusion == "failure"`. The reason code is read from the + `release-dependency-license-report` artifact's `failures[].code`, not from log prose. +- **Strix never started**: every step from `Provision the zero-cost review gateway for Strix` + through `Run Strix against one isolated synthetic fixture per dependency` has + `conclusion == "skipped"`. A step that ran and failed is a different outcome and must not be + reported as "did not start". +- **Credentials were never required for the licence decision**: the step + `Require every Strix provider credential before the Strix stage starts` also has + `conclusion == "skipped"`, which places it after the licence refusal. +- **`mock_publish` did not execute**: its `conclusion == "skipped"` **and** its `steps[]` is empty or + every entry has `conclusion == "skipped"`. `started_at` is recorded verbatim and explicitly **not** + used as evidence either way. +- **Evidence survived the failure**: the `release-dependency-license-report` artifact exists on the + failed run, which is the behavior the bound-to-producing-step upload condition exists to provide. + +Anything not on this list stays unverified. + +## Remaining end-to-end verification scope + +Splitting the evidence into unit-level denials and one `LICENSE_MISSING` collection run does **not** +shrink the requirement, and nothing here may be marked fully complete. Still unproven, with the kind +of run that would prove each: + +| Unproven | What would prove it | +|---|---| +| A copyleft dependency is refused by the **real collection path** | A run whose collected metadata carries a denied licence. No such run is planned, because authoring or installing a copyleft-declaring package is rejected. This gap stays open by policy. | +| `release_dependency_capture_raw.sh` executes at all | Any hosted run that reaches step 4. No step of that script has ever executed, here or in CI. | +| A locally authored fixture distribution is collectible | Resolution of the dropped-directive defect above, then one non-deploy run. | +| Strix succeeds and produces a real binding | A credentialed run that reaches step 12 with `verdict` and `findings` from an actual scan. | +| `seal` output is accepted by the real attestation workflow on real bytes | A run composing `gate` → `attest` on a real wheel and sdist. Locally only the *shape* is checked, against a synthetic sealed directory. | +| Capture/hash/metadata/artifact binding agree end to end | The same composed run, comparing `wheel_sha256` and `sdist_sha256` against the published artifact digests. | +| A real publish job would not start | Nothing planned proves this. `mock_publish` models the gating edge only. | + +## What remains unverified without a hosted run + +Verified locally by execution: every reason code above, produced by the production functions through +the existing harness in `tests/test_release_dependency_gate.py` and its siblings. + +Not verified, and not claimable until a single approved non-deploy run exists: that the gate job +concludes `failure` on a real runner; that the Strix steps report `skipped`; that `mock_publish` +does not execute; that the capture script's real `pip inspect`/`pip download`/`cargo metadata` +invocations behave as read (no step of `release_dependency_capture_raw.sh` has ever been executed, +here or in CI); that `seal` and `exact-artifact-sbom-attestation.yml` agree on real bytes; and that +the fixture distribution is collectible at all, which the refutation above says it currently is not. + +Refs #2342, #2347. diff --git a/docs/doctoring/release-build-artifact-identity.md b/docs/doctoring/release-build-artifact-identity.md new file mode 100644 index 0000000000..bf00920ee3 --- /dev/null +++ b/docs/doctoring/release-build-artifact-identity.md @@ -0,0 +1,62 @@ +# Immutable same-run build artifact intake + +Base: 1916e95a8ee3b0dbd1c84011d88fc580700430e3. Previously the dependency +gate selected the caller's build artifact by name only. The gate now requires +`build_artifact_id` and `build_artifact_digest` as well as the expected name. +The producer must pass its upload result ID and `sha256:`-prefixed digest. +Missing values cannot fall back to name selection. + +The shipped shell reuses the existing exact-artifact-sbom-attestation workflow's +same-repository/same-run metadata comparison and the same pinned download action. +It also checks the returned ID explicitly and requires canonical positive +decimal ID and sha256 digest input. Metadata identity, name, digest, run and +unexpired status must agree before download. An API error blocks consumption. +No second generic verifier, new token, or helper revision is introduced. + +The gate requests only the additional `actions: read` permission required by +the metadata API. The caller must grant it; a called workflow cannot elevate +the caller's token permissions. Existing name-only callers must provide both +new required inputs when adopting this revision. Repository-local inspection +finds no executable caller of this reusable workflow; external caller adoption +is not exhaustively verified. FMLS's trusted matrix aggregation remains unwired. + +## Pinned download action contract, source inspection only + +The actual pinned revision is +`actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c`. +Its action.yml lines42–46 declare `digest-mismatch: error` as the default; +this candidate explicitly selects error. Its src/download-artifact.ts +lines94–136 select immutable IDs from current-run artifacts, lines171–183 pass +the selected artifact's digest as expectedHash, and lines215–231 throw and fail +the action on mismatch. A single requested ID avoids the multi-ID partial-match +warning path. A single selected artifact uses the existing destination root. + +Primary sources opened directly: +https://raw.githubusercontent.com/actions/download-artifact/3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c/action.yml +https://raw.githubusercontent.com/actions/download-artifact/3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c/src/download-artifact.ts + +The pre-download API comparison is not an independent hash of downloaded +bytes. Byte verification relies on this pinned action's implementation; its +download library/bundled execution and hosted transport are not executed in +the local tests. No warning-only revision is treated as equivalent. The API +record and downloaded record must refer to the same immutable ID; this does +not establish a release-source build proof or trusted gate success by itself. + +## Scoped evidence and remaining holds + +Tests execute the actual shell with inert gh output and installed jq. They +cover a valid record, same-name different ID, different run, absent/modified +digest, expiry, absent ID, invalid expected digest, different repository and +API failure. Rejected cases cannot reach the next-step marker. Static checks +bind download to ID and error-on-digest-mismatch; no real download occurs. +The first test run failed10 cases because its declaration extractor split at +child indentation; after anchoring the next input key correctly, the same +cases execute the shipped shell. This is a test harness repair, not acceptance +of the failed run. + +Source/control equality, fixed helper00c655, full licence/Strix policy, +platform closure, same-run trusted success aggregation, resource bounds, and +final R5 HOLD remain unchanged. API rate exhaustion prevents a fresh broad +external ownership/Project census; no inference of absent external callers or +approval follows. CodeGraph indexed38 workflow files with0nodes/edges, so all +workflow call paths are inspected as source rather than graph completeness. diff --git a/docs/doctoring/release-fixed-helper-source.md b/docs/doctoring/release-fixed-helper-source.md new file mode 100644 index 0000000000..f93ef9f0c1 --- /dev/null +++ b/docs/doctoring/release-fixed-helper-source.md @@ -0,0 +1,34 @@ +# Fixed helper source for release gates + +The three trusted checkouts use literal repository `ContextualWisdomLab/.github` +and reviewed helper revision `00c6551183cca101cfc97c43656a17cc2491c1b4`. +This is an independent helper revision, not an assertion that helper and called +workflow revisions are equal. A future workflow change does not silently update +these helper bytes. Updating the pin and content identities requires review. + +All three checkouts materialize `scripts/ci/` and +`requirements-strix-ci-hashes.txt`, preserving helper siblings. Before executing +helpers they verify checkout HEAD, canonical origin URL, the scripts tree +`bf26d3eefdb71fe79b855d941ffb46eb432b2f76`, lock blob +`9e705850b5ce53c7fe836bc3df3a18771151e3f6`, tracked-file cleanliness and required +entrypoints. Missing, foreign or mismatched source rejects. The step reports +helper repository/SHA separately from caller workflow SHA. The latter is only +provenance context, never a checkout selector or authorization input. + +GitHub's [current context reference](https://docs.github.com/en/actions/reference/workflows-and-actions/contexts#job-context) +documents called-job workflow identity fields, but actionlint 1.7.12 and the +inspected upstream main strict schema do not yet support them. This alternative +uses neither those expressions nor an ignored diagnostic or permissive schema. +It changes the contract from called-self checkout to an explicitly adopted +fixed helper snapshot. The earlier called-self candidate remains separate. + +Local synthetic guards exercise valid identity, another caller SHA, missing +git source, foreign origin, wrong HEAD/tree, dirty tracked files and a missing +entrypoint. They do not perform checkout or network access. Hosted checkout and +attestation behavior remain unexecuted. + +`SOURCE_SHA == GITHUB_SHA`, full licence/Strix authorization, twelve-platform +closure and complete resource intake budgets remain unresolved independently. +No condition is relaxed by this source-selection fix. The pinned snapshot +retains its existing licence/tool-dependency limitations; exact-byte provenance +does not imply policy acceptance. diff --git a/docs/doctoring/release-license-archive-binding-20260924.md b/docs/doctoring/release-license-archive-binding-20260924.md new file mode 100644 index 0000000000..745e843727 --- /dev/null +++ b/docs/doctoring/release-license-archive-binding-20260924.md @@ -0,0 +1,64 @@ +# Archive-bound license evidence candidate + +Base: `a78b1c9f788d1a89fd7c8ab39d6347152b7e3065`. + +## Reproduced defect + +`/private/tmp/pr2347-archive-binding-repro.py` exercises the real capture, license +gate and install-lock binder. Its synthetic wheel contains academic-only terms, +while the separately supplied raw `licenses/LICENSE` contains the reviewed pytest +MIT text. The wheel and lock SHA-256 both equal +`659169bde33b6d27bf4cf927c01d69418cc97241bf728627448542f781c2771d`. +The base gate returns PASS and the binder writes that restrictive archive's hash. +This is a synthetic exploit, not a claim about any upstream package. +The raw receipt is `/private/tmp/pr2347-archive-binding-repro.md`. + +## Candidate contract + +- Raw capture retains `source.archive` for both wheel and crate inputs. +- One bounded byte read supplies both the archive digest and in-memory license + extraction. No archive extraction or package execution occurs in this helper. +- Conventional license/notice names at every depth and declared custom + `License-File` / Cargo `license-file` members are inspected. Missing declared + members, duplicate normalized paths, traversal, archive links, invalid text + and malformed archives refuse the capture. +- Separate raw license sidecars no longer supply the license decision. +- The gate reopens the retained archive and compares the source digest, full + license text mapping and raw member SHA-256 mapping against the evidence. +- The install binder verifies those member hashes against the collected wheel + and repeats the license decision on its actual member bytes before writing + the pinned install lock. Forged permissive report fields cannot authorize + an unrecognized restrictive body. + +## Verification + +The existing eight targeted test files plus +`tests/test_release_dependency_archive_binding.py` produce **296 passed, +1 skipped**, raw exit **0**, in 4.67 seconds. The skip is the existing GNU-find +Linux capture integration case; it is not a new skip. + +The 16 added cases cover Python/Cargo sidecar forgery, evidence alteration, +archive replacement, archive absence, duplicate archive members, nested/raw-byte +hash preservation, custom wheel license paths and missing declarations, plus +two real shell install-binder refusals. A fake pip recorder establishes zero +install calls in both new install-negative cases. No real install or download +is used. Existing fixture archives now carry the same source-bound full texts +used by their license tests; one formerly permissive missing-archive capture +expectation changes to an explicit refusal. + +`git diff --check` and shell syntax checking also return 0. This is a selected +regression result, not full-suite, coverage, hosted execution or release approval. + +## Remaining boundaries + +Raw shell capture still performs its existing extraction/native/hook collection +before assembly. Those scanners and separate metadata are not all reconstructed +from the retained snapshot by this patch. This candidate closes the demonstrated +license-sidecar binding defect; it does not certify every captured evidence field +or shell extraction as safe. The final install trusts the protected workflow +workspace to prevent mutation between binding and pip's hash-checked read. + +Only the previously reviewed exact full texts are recognized. Unsupported texts, +MPL/BSL complex provenance, NumPy bundle questions, tools/sidecar pre-install +coverage and complete dependency closure remain held. No license exception or +legal conclusion follows from this candidate. diff --git a/docs/doctoring/release-license-fixture-recovery-20260924.md b/docs/doctoring/release-license-fixture-recovery-20260924.md new file mode 100644 index 0000000000..ed010bf912 --- /dev/null +++ b/docs/doctoring/release-license-fixture-recovery-20260924.md @@ -0,0 +1,52 @@ +# 기존 18실패의 원문 근거 복원 + +기준 `4fe66efdcee6bb6b68e5a6c386feea7280ecea8d`에서 새 브랜치 `codex/pr2347-source-fixture-recovery-20260924`를 사용한다. 기존 여섯 원문의 JSON 값과 provenance 첫 여섯 행은 Git blob 대조로 불변을 확인한다. + +공통 fixture는 Python MIT→실제 pytest 전체 MIT, Cargo Apache→실제 atheris 전체 Apache로 연결한다. 패키지 자체에 atheris라는 이름을 붙이거나 metadata 누락을 고친다는 주장이 아니라, synthetic gate fixture의 동일 SPDX 본문을 검증 가능한 전체 원문으로 복원한다. Cargo를 MIT로 바꾸지 않는다. 원문 hash는 기존 provenance에 있다. + +## 18개 before/after 기대와 근거 + +아래 이름은 `test_release_dependency_license_text_evidence.py` 기준이다. P=Python 원문, C=Cargo 원문이다. 기존 실패의 별도 C UNVERIFIED는 실제 Apache 전체 원문으로 해소한다. 검사의 핵심 실패 사유를 삭제하지 않는다. + +|번호|사례|before expected|after expected 및 변경 근거| +|---|---|---|---| +|1|no_bundled_text|MISSING 1개|동일. C만 전체 Apache로 복원| +|2|unrecognizable[unknown]|UNVERIFIED 1개|동일. UNKNOWN 입력 유지| +|3|unrecognizable[commercial-prohibited]|UNVERIFIED 1개|동일. 상업 금지 입력 유지| +|4|unrecognizable[empty]|UNVERIFIED 1개|동일. 빈 원문 유지| +|5|unrecognizable[pointer]|UNVERIFIED 1개|동일. 링크-only 입력 유지| +|6|unrecognizable[all-rights-reserved]|UNVERIFIED 1개|동일. 권리 유보 입력 유지| +|7|recognized_text_contradicts_declaration|DISAGREEMENT 1개|동일. MIT 선언에 실제 전체 Apache를 넣음| +|8|denied_title_disagreement|DISAGREEMENT 1개|동일. 정상 LICENSE는 실제 MIT, 별도 GPL COPYING 유지| +|9|matching_declaration|실패 없음|동일. 전체 pytest MIT 사용| +|10|permissive_family[Apache]|실패 없음|동일. 전체 atheris Apache 사용| +|11|permissive_family[BSD3]|실패 없음|동일. 전체 colorama BSD3 사용| +|12|permissive_family[ISC]|실패 없음|동일. 전체 libloading ISC 사용| +|13|permissive_family[MPL]|실패 없음|제목-only 원문은 UNVERIFIED. unsupported_title_only[MPL]로 목적을 명시하며 동일 입력을 유지. 실제 hypothesis 복합 적용 범위는 아래 별도 HOLD| +|14|permissive_family[BSL]|실패 없음|제목-only 원문은 UNVERIFIED. unsupported_title_only[BSL]로 목적을 명시하며 동일 입력을 유지. 기존 cache 인벤토리에 BSL 원문 mapping이 없음| +|15|permissive_family[Unlicense]|실패 없음|동일. memchr의 실제 전체 UNLICENSE 사용| +|16|dual_selection_disagreement|DISAGREEMENT 1개|동일. BSD/GPL 선언·BSD 선택에 실제 전체 MIT를 넣어 불일치 유지| +|17|recognizer_positive_half|MIT 포함|동일. 전체 MIT를 사용하고 UNKNOWN 반환 검사는 유지| +|18|install_binding cargo_only_release|실패 없음, lock digest 없음|동일. C에 전체 Apache를 연결. lock/hash 구현은 변경하지 않음| + +BSD 선택 양성과 sealed SBOM 선택 rationale 테스트도 동일 colorama BSD3 전체 원문으로 복원한다. 앞 후보의 부분 Apache 거부 테스트는 공통 fixture 변경에 영향받지 않도록 그 테스트에서 부분 Apache를 명시한다. 기존 음성을 정상으로 변경하지 않는다. + +## 추가 실제 원문과 보류 + +memchr2.8.3 `.crate` SHA256 `cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98`, member `memchr-2.8.3/UNLICENSE`, 원시 `7e12e5df4bae12cb21581ba157ced20e1986a0508dd10d0e8a4ab9a4cf94e85c`, 정규화 `2069c208cba553e43cd0b730df8a0c10bf1b1101b96f661e2f1307c73b9722e3`다. 전체 본문에서 copy/modify/publish/use/compile/sell/distribute, commercial or non-commercial, public-domain dedication 및 면책을 직접 읽는다. 다른 추가 조건을 발견하지 않는다. 이 파일만 registry에 추가하며 crate의 COPYING pointer·MIT/Unlicense 선택 전체를 자동 수용하지 않는다. + +hypothesis6.156.6 wheel SHA256 `b4e66aaa7385538a5d617174d47c198ee807f06de99e282a67c6cb724c69340d`, member `hypothesis-6.156.6.dist-info/licenses/LICENSE.txt`, raw `ac89037bac63550644dce8cf32c6765e5fab9dc1a1ce94b89f8a805f341a6750`이다. 전체 1–10절과 Exhibits A/B를 읽는다. 앞부분은 명시된 예외 외 MPL 적용, 다른 프로젝트 코드의 원래 license와 수정 dual license를 설명한다. METADATA의 License-Expression=MPL-2.0/License-File=LICENSE.txt이며, archive의 license/copying/notice 이름 member는 이 파일 하나다. 개별 코드의 다른 원래 라이선스 적용 범위는 이 한 파일로 확정되지 않아 후속 mapping 검토가 필요하다. + +1.12절의 GPL/LGPL/AGPL 명칭은 Secondary License 정의다. 그 이름만으로 실제 금지 의존성이나 선택된 copyleft라고 판정하지 않는다. 기존 `scan_license_text`가 이 명칭에서 거부하는 문제는 의미 구분이 없는 별도 한계다. 이번 원문은 `unsupported-hypothesis.json`에 원시 hash와 함께 보존하고 recognizer UNKNOWN을 확인하며, keyword 거부를 실제 GPL 확정 판정으로 승인하지 않는다. 이번 범위에서 scanner 예외를 새로 허용하지 않는다. + +## 실행 + +``` +PYTHONDONTWRITEBYTECODE=1 PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python3 -B -m pytest --noconftest -p no:cacheprovider -o addopts= tests/test_release_dependency_license_text_evidence.py tests/test_release_dependency_install_binding.py tests/test_release_dependency_install_ordering.py tests/test_release_dependency_full_text_contract.py tests/test_release_dependency_reviewed_artifact_texts.py tests/test_spdx_license_policy.py tests/test_release_dependency_gate.py tests/test_release_dependency_gate_capture_and_seal.py -q --tb=short +280 passed, 1 skipped in 2.94s +raw exit 0 +``` + +최초 실행은 sealed SBOM 사례의 REVIEWED_TEXTS import 누락으로 1 failed/279 passed/1 skipped/exit1이다. 실제 해당 함수 import를 고쳐 위 결과를 얻는다. 기존 skip은 macOS의 GNU find capture 경로이며 새 skip을 추가하지 않는다. 검사 삭제 없이 MPL/BSL 두 parameter를 별도 UNKNOWN 음성으로 유지하고, 실원문과 추가 제한·다중 파일·미선언 거부 회귀를 함께 실행한다. 마지막 unused import 제거는 실행 경로와 무관하다. + +`git diff --check` exit0. 기존6개 텍스트/provenance 불변 대조 true. 전체 suite·coverage·hosted·tooldeps·현재 release closure는 이번 소형 성공으로 수용하지 않으며 HOLD다. diff --git a/docs/doctoring/release-license-six-artifact-texts-20260924.md b/docs/doctoring/release-license-six-artifact-texts-20260924.md new file mode 100644 index 0000000000..c477bcb52c --- /dev/null +++ b/docs/doctoring/release-license-six-artifact-texts-20260924.md @@ -0,0 +1,30 @@ +# 실제 artifact 원문 여섯 개의 증분 인식 + +이 후보는 `4329ebb84ddac1752e5c4149fd3c94731b262f2e` 뒤에 여섯 전체 원문 hash를 추가한다. 일반적인 라이선스 판별기나 전체 배포 closure 승인으로 확대하지 않는다. production 변경은 기존 `_VERIFIED_LICENSE_TEXT_DIGESTS` 추가뿐이다. + +## 출처와 직접 읽은 범위 + +자료는 기존 로컬 FMLS license evidence archive다. artifact filename/SHA256, 내부 member, 원시 SHA256, 정규화 SHA256, SPDX 대응은 `tests/fixtures/release_license_texts/provenance.json`에 기록한다. 원문은 기억에서 재작성하지 않고 archive member를 UTF-8로 읽는다. 전체 본문을 줄 생략 없이 확인한다. `texts.json`의 각 문자열을 UTF-8로 인코딩한 바이트가 원시 member hash와 일치하는지 테스트한다. `fixture` 필드는 이 JSON 안의 키다. 끝 개행이 없는 원문도 그대로 보존한다. + +|실제 원문|읽은 허용·조건과 경계| +|---|---| +|pytest9.1.1 LICENSE / MIT|전체 grant에 use/copy/modify/merge/publish/distribute/sublicense/sell과 without restriction이 있다. copyright·permission notice 보존 및 보증 면책을 읽는다. 정확한 Holger Krekel 머리말도 hash에 포함한다. 다른 MIT 머리말을 자동 인정하지 않는다.| +|atheris3.1.0 LICENSE / Apache-2.0|1–9절과 적용 부록 전체다. 2절 copyright grant, 3절 patent grant·소송 종료조건, 4절 재배포·수정·고지, 5절 contribution, 6절 trademark, 7–9절 보증·책임을 읽는다. 별도 NC/학술 전용 부속문구는 없다. 이는 파일의 인식이며 실제 atheris의 metadata 선언 누락은 계속 HOLD다.| +|Rust numpy0.29.0 LICENSE / BSD-2-Clause|source·binary 재배포 허용, 두 고지 보존 조건, 전체 면책을 읽는다. PyPI NumPy 복합 원문과 다른 파일이다.| +|colorama0.4.6 LICENSE.txt / BSD-3-Clause|source·binary 재배포 허용, 고지 보존 두 조건, 이름을 허가 없이 endorsement에 사용하지 않는 세 번째 조건과 면책을 읽는다. 추가 상업 이용 금지는 없다.| +|libloading0.8.9 LICENSE / ISC|any purpose with or without fee의 use/copy/modify/distribute 허용, copyright·permission notice 보존, 전체 면책을 읽는다. Simonas Kazlauskas 머리말을 포함한다.| +|foldhash0.2.0 LICENSE / Zlib|any purpose including commercial applications, alter/redistribute 허용과 출처 오인 금지·변형 표시·고지 제거 금지 세 조건을 읽는다. 전체 면책도 포함한다.| + +모두 저장된 실제 소스에 대한 인식 지원이다. 상용 제품의 모든 법적 의무 충족을 확정하는 판단이 아니다. 선택된 OR의 pointer 문서, PyPI NumPy의 GPL/LGPL 복합 원문, upstream 16개, 다른 copyright/원문 변형, 수집 누락은 별도 HOLD다. 추가 파일마다 기존 consumer의 검사가 계속 적용된다. + +## 검사와 실패 보존 + +명령 공통 환경: `PYTHONDONTWRITEBYTECODE=1 PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python3 -B -m pytest --noconftest -p no:cacheprovider -o addopts=`. + +- `tests/test_release_dependency_reviewed_artifact_texts.py tests/test_release_dependency_full_text_contract.py tests/test_spdx_license_policy.py -q`: 99 passed, raw exit0. 여섯 실제 원문의 hash·정상 consumer, 앞/뒤/중간 추가 조건, 별도 NOTICE 제한, atheris 선언 누락 유지가 포함된다. +- 첫 fixture 생성은 끝 개행을 추가해 원시 hash 6건이 실패(6 failed/93 passed)하고, 첫 보정은 개행 없는 2개 파일 끝 문자를 훼손해 4 failed/95 passed다. 원문 문자열을 JSON에 그대로 보존하는 방식으로 보정하고 모든 원시 hash를 다시 확인한다. 정규화 hash만 일치한다는 이유로 이 실패를 무시하지 않는다. +- 기존 비교군 `test_release_dependency_license_text_evidence.py`, `test_release_dependency_install_binding.py`, `test_release_dependency_install_ordering.py`: 18 failed/27 passed/1 skipped, raw exit1. 이전과 같은 미지원 부분 원문 기대값 실패를 보존한다. 기존 fixture 수정은 없다. + +이전18건 중 잘못된 양성 기대는 짧은 MIT/Apache/BSD/ISC/MPL/BSL/Unlicense 제목·일부 grant를 완전 원문으로 취급하는 부분이다. 정상 corpus를 바꾸려면 그 라이선스의 실제 전체 자료와 hash를 같은 SPDX로 연결하는 별도 diff가 필요하다. Cargo Apache 제목-only fixture도 Apache 전체 원문으로 복원해야 하며, MIT로 바꾸는 방식은 허용하지 않는다. 이번 변경은 그 기대값을 편의상 고치지 않는다. + +전체 suite·coverage·hosted·tooldeps 검사와 publish는 미실행·HOLD다. 여섯 원문 추가로 범위 전체를 수용하지 않는다. diff --git a/docs/doctoring/release-license-whole-text-candidate-20260924.md b/docs/doctoring/release-license-whole-text-candidate-20260924.md new file mode 100644 index 0000000000..ffa9db7855 --- /dev/null +++ b/docs/doctoring/release-license-whole-text-candidate-20260924.md @@ -0,0 +1,34 @@ +# PR2347 전체 원문 확인 중간 후보 + +기준은 `48caafec7160dd0cb9bafc58b28a884dc4c35cbb`이다. 원문 제목/부분 문자열만으로 허용하는 P1을 닫는 로컬 후보이며, 전체 의존성 정책 구현 완료나 병합·배포 수용을 뜻하지 않는다. + +## 근거와 지원 경계 + +직접 읽은 저장소 `LICENSE` 전체를 근거로 삼는다. 원시 SHA256은 `08f1fd81fb120bc468b69dc3e58ea0dc23c216305c766e45e107f56c76559e3f`이다. ASCII 공백·탭·CR·LF만 연속 공백 하나로 정규화한 전체 본문 SHA256은 `f5ac0308cf2b3f96a0f49a8c0c9e4a2a02c483afc72a646af8de1f356983de06`이다. + +검증 지원은 이 MIT 원문 한 개이며 Copyright 문구까지 포함한다. 다른 저작권자 머리말도 아직 UNKNOWN이다. 임의 머리말·추가 조건·접미사·유니코드 제어 문자를 지우지 않는다. SPDX 선언, 제목, 허용 구절만으로 확인된 원문이 되지 않는다. 이 레지스트리는 새로운 의존성을 라이선스 이름만으로 승인하는 수단이 아니다. + +실제 closure에 필요한 BSD, Apache, CC0 및 다른 라이선스 원문·변형 지원은 미완료다. 각 원문과 전체 일치 계약을 독립 검토한 뒤 별도 증분으로 추가해야 한다. 현재 인벤토리 전체 PASS는 불가능하다. 설치 전 도구 의존성 검사는 별도 미해결이다. + +`recognize_license_text`의 production caller는 `release_dependency_gate.evaluate_dependency_license`다. CodeGraph는 해당 트리에 index가 없다고 반환하며, 소스 참조를 직접 대조한다. caller가 파일마다 판정하므로 허용 LICENSE와 별도 제한 NOTICE를 함께 넣어도 거부한다. + +## 검증 + +모든 명령은 이 별도 작업 트리에서 실행한다. 환경은 `PYTHONDONTWRITEBYTECODE=1 PYTEST_DISABLE_PLUGIN_AUTOLOAD=1`, 공통 인자는 `python3 -B -m pytest --noconftest -p no:cacheprovider -o addopts=`다. + +1. `tests/test_release_dependency_full_text_contract.py tests/test_spdx_license_policy.py -q`: **68 passed, raw exit 0**. 최초 실행은 새 테스트에서 상수 소유 모듈을 잘못 적어 1 failed/67 passed/exit1이다. `policy.LICENSE_TEXT_DISAGREEMENT`를 실제 소유자 `gate`로 고친 뒤 위 결과를 얻는다. +2. 기존 소형 비교군 `tests/test_release_dependency_license_text_evidence.py tests/test_release_dependency_install_binding.py tests/test_release_dependency_install_ordering.py -q --tb=no`: **18 failed, 27 passed, 1 skipped, raw exit 1**. 기존 fixture는 수정하지 않는다. +3. `git diff --check`: exit0. + +새 회귀는 실제 gate의 MIT 추가 상업 제한과 CC0/NonCommercial 반례를 거부하고, 완전한 확인 MIT 원문은 동일 dependency caller에서 통과시킨다. gate 전체의 정상 Python 사례에서도 기존 Cargo Apache fixture가 UNKNOWN으로 남아 전체 통과를 주장하지 않는다. GPL 별도 파일, 추가 NOTICE, 본문 변조·앞뒤 조건·NUL·zero-width suffix도 확인한다. + +### 보존하는 기존 실패 분류 + +- 원문 누락/UNKNOWN/별도 GPL 등 기존 원인 자체는 계속 거부한다. 같은 capture의 Cargo Apache 제목-only fixture가 추가 `LICENSE_TEXT_UNVERIFIED`를 내므로 기존 exact-single-failure 기대와 다르다. +- MIT·Apache·BSD·ISC·MPL·BSL·Unlicense 부분 원문을 정상으로 기대한 사례와 recognizer 직접 호출의 부분 MIT 기대는 더 이상 충족하지 않는다. +- 기존 Apache-vs-MIT 제목-only 불일치는 확인된 Apache가 아니므로 UNKNOWN으로 분류한다. 지원하지 않는 본문에서 라이선스 종류를 확정하지 않는다. +- Cargo-only binding 테스트는 그 Apache fixture 원문의 미확인 때문에 실패한다. lock hash 결속 구현의 변경은 아니다. + +실제 gate 분류 재확인: Python 원문 없음은 MISSING + Cargo UNVERIFIED, Python UNKNOWN/부분 MIT/부분 Apache는 각각 Python UNVERIFIED + Cargo UNVERIFIED다. 합성 Cargo를 MIT로 바꿔 실패를 숨기지 않는다. + +전체 suite·coverage·hosted CI·Linux capture·원문 수집 완전성·tooldeps 설치 전 검사는 이번 수용 밖이며 HOLD다. 이 후보는 공개 push 없이 다른 reviewer의 검토에 인계한다. diff --git a/scripts/ci/release_dependency_capture_raw.sh b/scripts/ci/release_dependency_capture_raw.sh new file mode 100755 index 0000000000..b1390ac750 --- /dev/null +++ b/scripts/ci/release_dependency_capture_raw.sh @@ -0,0 +1,391 @@ +#!/usr/bin/env bash +# Collect raw pre-publish dependency evidence for the central release gate (#2342). +# +# This script only *runs tools and writes their output verbatim*. Every decision +# — license policy, lock/environment reconciliation, archive-escape and +# install-hook detection, Strix binding validation — lives in the unit-tested +# scripts/ci/release_dependency_gate.py, which reads what this writes. Keeping +# the split that way means no untested shell ever decides whether a release may +# publish. +# +# It requires a runner: pip, cargo, readelf, and network access to the indexes. +# It is therefore exercised in GitHub Actions only; see +# .github/workflows/release-dependency-license-strix-gate.yml. +# +# Output layout (consumed by `release_dependency_gate.py capture` and `gate`): +# +# /python/lock.txt the hash-pinned lock that was collected +# /python/installed.json declared identity/licence per fetched +# distribution, in `pip inspect` shape +# /cargo/Cargo.lock the committed Cargo lock +# /cargo/metadata.json cargo metadata --format-version 1 --locked +# //metadata.json declared identity + license fields +# //source.sha256 sha256 of the distribution as fetched +# //members.txt "\t\t" per member +# //licenses/* bundled LICENSE/COPYING/NOTICE verbatim +# //hooks/* setup.py / build.rs sources verbatim +# //native.json dynamic/static link targets per shipped .so +# //parsed_inputs.txt file names the dependency parses + +set -euo pipefail + +RAW_ROOT="" +CAPTURE_ROOT="" +ECOSYSTEMS="" +PYTHON_LOCK="" +PYTHON_INTERPRETER="" +CARGO_MANIFEST="" +DOWNLOAD_ROOT="" +LICENSE_REPORT="" +MODE="capture" + +while [ "$#" -gt 0 ]; do + case "$1" in + --raw-root) RAW_ROOT="$2"; shift 2 ;; + --capture-root) CAPTURE_ROOT="$2"; shift 2 ;; + --ecosystems) ECOSYSTEMS="$2"; shift 2 ;; + --python-lock) PYTHON_LOCK="$2"; shift 2 ;; + --python-interpreter) PYTHON_INTERPRETER="$2"; shift 2 ;; + --cargo-manifest) CARGO_MANIFEST="$2"; shift 2 ;; + --download-root) DOWNLOAD_ROOT="$2"; shift 2 ;; + --license-report) LICENSE_REPORT="$2"; shift 2 ;; + --install-gated) MODE="install"; shift ;; + *) echo "ERROR: unknown argument $1" >&2; exit 2 ;; + esac +done + +if [ "$MODE" = "install" ]; then + if [ -z "$PYTHON_LOCK" ] || [ ! -f "$PYTHON_LOCK" ] || [ -z "$DOWNLOAD_ROOT" ]; then + echo "ERROR: --install-gated requires --python-lock and --download-root." >&2 + exit 2 + fi + if [ -z "$LICENSE_REPORT" ] || [ -z "$CAPTURE_ROOT" ]; then + echo "ERROR: --install-gated requires --license-report and --capture-root." >&2 + exit 2 + fi +else + if [ -z "$RAW_ROOT" ] || [ -z "$CAPTURE_ROOT" ] || [ -z "$ECOSYSTEMS" ]; then + echo "ERROR: --raw-root, --capture-root and --ecosystems are required." >&2 + exit 2 + fi + mkdir -p "$RAW_ROOT" "$CAPTURE_ROOT" +fi + +# The pip entry point is a variable only so the wiring can be regression-tested +# without a network: a test points RELEASE_GATE_PIP at a recorder and asserts +# which pip invocations happened, and in what order, for a refused release. +PIP=(python3 -m pip) +if [ -n "${RELEASE_GATE_PIP:-}" ]; then + PIP=("${RELEASE_GATE_PIP}") +fi + +# Resolved from this script's own directory, never from the caller's cwd or an +# environment variable, so the trusted gate cannot be swapped by a PR. +GATE_SCRIPT="$(cd -- "$(dirname -- "$0")" && pwd)/release_dependency_gate.py" + +# pip's global --python re-executes pip against another interpreter, which is how +# the lock-only virtual environment is installed into by the gated install mode. +PIP_TARGET_ARGS=() +if [ -n "$PYTHON_INTERPRETER" ]; then + # `python3 -m venv` uses symlinks by default on POSIX, so a normal virtual + # environment's bin/python *is* a symlink; refusing symlinks outright rejected + # every real venv and made this path unreachable. What must be refused is a + # target that is not a regular executable file, or a dangling link, so the link + # is resolved and the resolved target is checked. + resolved_interpreter="$(cd -- "$(dirname -- "$PYTHON_INTERPRETER")" 2>/dev/null && pwd -P)/$(basename -- "$PYTHON_INTERPRETER")" + while [ -L "$resolved_interpreter" ]; do + link_target="$(readlink -- "$resolved_interpreter")" + case "$link_target" in + /*) resolved_interpreter="$link_target" ;; + *) resolved_interpreter="$(dirname -- "$resolved_interpreter")/$link_target" ;; + esac + done + if [ ! -f "$resolved_interpreter" ] || [ ! -x "$resolved_interpreter" ]; then + echo "ERROR: --python-interpreter must resolve to a regular executable interpreter." >&2 + exit 2 + fi + PIP_TARGET_ARGS=(--python "$PYTHON_INTERPRETER") +fi + +# Record one archive's members as "\t\t". Symlink and +# hardlink targets are preserved verbatim so the gate can detect escapes. +record_members() { + local archive="$1" destination="$2" + case "$archive" in + *.whl | *.zip) + unzip -Z1 "$archive" | while IFS= read -r member; do + printf 'file\t%s\t\n' "$member" + done + ;; + *) + tar -tvf "$archive" | while IFS= read -r line; do + local permissions name link type + permissions="${line%% *}" + name="$(printf '%s' "$line" | sed -E 's/^.* [0-9]{2}:[0-9]{2} //')" + link="" + type="file" + case "$permissions" in + l*) type="symlink"; link="${name#* -> }"; name="${name%% -> *}" ;; + h*) type="hardlink"; link="${name#* link to }"; name="${name%% link to *}" ;; + d*) type="directory" ;; + esac + printf '%s\t%s\t%s\n' "$type" "$name" "$link" + done + ;; + esac >"$destination" +} + +# Record every bundled license-like file verbatim, flattened into one directory. +record_license_files() { + local root="$1" destination="$2" + mkdir -p "$destination" + find "$root" -maxdepth 4 -type f \ + \( -iname 'LICENSE*' -o -iname 'COPYING*' -o -iname 'NOTICE*' \) -print0 | + while IFS= read -r -d '' found; do + cp -- "$found" "$destination/$(printf '%s' "${found#"$root"/}" | tr '/' '_')" + done +} + +# Record install/build hook sources verbatim so the gate can inspect them. +record_hook_sources() { + local root="$1" destination="$2" + mkdir -p "$destination" + find "$root" -maxdepth 3 -type f \ + \( -name 'setup.py' -o -name 'build.rs' -o -name 'conanfile.py' \) -print0 | + while IFS= read -r -d '' found; do + cp -- "$found" "$destination/$(printf '%s' "${found#"$root"/}" | tr '/' '_')" + done +} + +# Record dynamic NEEDED entries and shipped static archives for native libraries. +record_native_libraries() { + local root="$1" destination="$2" + local entries="[]" + while IFS= read -r library; do + local needed + needed="$(readelf -d "$library" 2>/dev/null | + sed -n 's/.*(NEEDED).*\[\(.*\)\]/\1/p' | + jq -R . | jq -s .)" + entries="$(jq --arg path "${library#"$root"/}" --argjson needed "${needed:-[]}" \ + '. + [{"path": $path, "needed": $needed, "static_archives": []}]' <<<"$entries")" + done < <(find "$root" -type f \( -name '*.so' -o -name '*.so.*' -o -name '*.pyd' \)) + printf '%s\n' "$entries" >"$destination" +} + +capture_python() { + local lock="$1" + mkdir -p "$CAPTURE_ROOT/python" "$DOWNLOAD_ROOT" + cp -- "$lock" "$CAPTURE_ROOT/python/lock.txt" + + local plain_requirements + plain_requirements="$DOWNLOAD_ROOT/pins-without-hashes.txt" + # Fetch by exact pin with hash checking deliberately disabled, then hash the + # bytes here and compare against the lock in the gate. Downloading *with* + # --require-hashes would make pip itself reject a tampered distribution, so + # the gate could never observe SOURCE_HASH_MISMATCH. The install of these same + # bytes happens later, offline and *with* --require-hashes, in install_gated. + sed -E 's/\\$//' "$lock" | grep -oE '^[A-Za-z0-9._-]+==[^ ;]+' \ + >"$plain_requirements" + # The real lock may carry --index-url, --extra-index-url or --find-links, + # while this reconstructed plain file has none of them. Dropping them silently + # made collection resolve from a different source than install. The trusted + # gate therefore parses and *validates* those directives — allowed HTTPS + # origin, no userinfo, bounded relative path — and emits them one per line; + # anything unsupported or untrusted fails here rather than being dropped. + # mapfile keeps each value a single argv element, so no lock content is ever + # word-split or re-interpreted by this shell. This runs before the first + # network action, so a refused directive means nothing was ever fetched. + local -a source_options=() + if [ ! -f "$GATE_SCRIPT" ] || [ -L "$GATE_SCRIPT" ]; then + echo "ERROR: trusted gate script is missing beside this script." >&2 + exit 2 + fi + # Deliberately not `mapfile < <(python3 ...)`: inside process substitution the + # validator's exit status is discarded by set -e, so a refusal would be read as + # "no options" and collection would continue from the default index — the same + # silent drop this fix exists to remove. The status is checked explicitly. + local options_file="$DOWNLOAD_ROOT/validated-source-options.txt" + if ! python3 -I "$GATE_SCRIPT" lock-source-options \ + --lock "$lock" --permitted-root "$(dirname -- "$lock")" >"$options_file"; then + echo "ERROR: lock source directives failed validation; refusing to collect." >&2 + exit 2 + fi + mapfile -t source_options <"$options_file" + # --only-binary=:all: is not only a build-hook guard for the gate environment: + # `pip download` executes an sdist's build backend to get its metadata even + # with --no-deps, so a wheel-only collection is what keeps unadjudicated + # dependency code from running before the licence stage. + "${PIP[@]}" download --no-deps --only-binary=:all: \ + "${source_options[@]}" \ + --dest "$DOWNLOAD_ROOT" -r "$plain_requirements" >/dev/null + + # The enumeration and the licence fields both come from the *fetched + # distributions*, never from `pip inspect` of an installed environment: the + # closure is not installed yet at this point, and must not be until the + # licence stage has passed. The file keeps the `pip inspect` shape the gate + # already reconciles against the lock. + local installed="$CAPTURE_ROOT/python/installed.json" + printf '{"installed": []}\n' >"$installed" + while IFS= read -r pin; do + [ -n "$pin" ] || continue + local name version slug target distribution extracted + name="${pin%%==*}" + version="${pin#*==}" + slug="pypi__$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]' | tr '._' '--')__$version" + target="$RAW_ROOT/$slug" + mkdir -p "$target" + distribution="$(find "$DOWNLOAD_ROOT" -maxdepth 1 -type f \ + -iname "$(printf '%s' "$name" | tr '.-' '__')-${version}*" | head -n 1)" + if [ -z "$distribution" ]; then + echo "ERROR: no fetched distribution for ${name}==${version}" >&2 + exit 2 + fi + cp -- "$distribution" "$target/source.archive" + sha256sum "$target/source.archive" | cut -d' ' -f1 >"$target/source.sha256" + record_members "$distribution" "$target/members.txt" + extracted="$(mktemp -d)" + case "$distribution" in + *.whl) unzip -qq -o "$distribution" -d "$extracted" ;; + *) tar -xf "$distribution" -C "$extracted" ;; + esac + record_license_files "$extracted" "$target/licenses" + record_hook_sources "$extracted" "$target/hooks" + record_native_libraries "$extracted" "$target/native.json" + find "$extracted" -maxdepth 3 -type f -name '*.py' -printf '%P\n' | + LC_ALL=C sort >"$target/parsed_inputs.txt" + printf '{}\n' >"$target/bundled_library_licenses.json" + # Licence metadata is read out of the distribution's own METADATA/PKG-INFO + # by the trusted gate, which also re-checks that the archive declares the + # pinned name and version. A file whose metadata names another project + # fails here instead of being adjudicated under the wrong identity. + python3 -I "$GATE_SCRIPT" distribution-metadata \ + --distribution "$distribution" --name "$name" --version "$version" \ + >"$target/metadata.json" + jq --slurpfile declared "$target/metadata.json" \ + '.installed += [{"metadata": $declared[0]}]' "$installed" \ + >"$installed.next" + mv -- "$installed.next" "$installed" + rm -rf "${extracted:?}" + done <"$plain_requirements" +} + +# Install exactly the distributions the licence stage already judged: offline, +# from the collected bytes, with --require-hashes so pip itself proves each file +# matches the lock. No index is consulted and nothing is re-resolved or +# re-downloaded, so the installed bytes are the inspected bytes by construction — +# which a second hash-less download could not establish for a multi-hash lock. +install_gated() { + local lock="$1" + if [ -z "$LICENSE_REPORT" ]; then + echo "ERROR: --install-gated requires --license-report." >&2 + exit 2 + fi + if [ ! -f "$GATE_SCRIPT" ] || [ -L "$GATE_SCRIPT" ]; then + echo "ERROR: trusted gate script is missing beside this script." >&2 + exit 2 + fi + if [ ! -d "$DOWNLOAD_ROOT" ]; then + echo "ERROR: no collected distributions to install from: $DOWNLOAD_ROOT" >&2 + exit 2 + fi + if [ -z "$CAPTURE_ROOT" ]; then + echo "ERROR: --install-gated requires --capture-root to bind the judged lock." >&2 + exit 2 + fi + # The report alone is not permission: bind-install refuses unless the lock still + # digests to what the verdict read, every judged artifact is present in the + # collected root by digest, and the root holds nothing else. It then pins each + # project to the single judged digest, so a lock recording several hashes for one + # project cannot admit an artifact whose licence and contents were never judged. + local bound_requirements="$DOWNLOAD_ROOT/gated-requirements.txt" + if ! python3 -I "$GATE_SCRIPT" bind-install \ + --report "$LICENSE_REPORT" \ + --capture "$CAPTURE_ROOT" \ + --download-root "$DOWNLOAD_ROOT" \ + --output "$bound_requirements" >/dev/null; then + echo "ERROR: the licence verdict does not authorize installing these bytes." >&2 + exit 2 + fi + "${PIP[@]}" "${PIP_TARGET_ARGS[@]}" install \ + --require-hashes --only-binary=:all: --no-index \ + --find-links "$DOWNLOAD_ROOT" \ + -r "$bound_requirements" +} + +capture_cargo() { + local manifest="$1" manifest_dir + manifest_dir="$(dirname -- "$manifest")" + mkdir -p "$CAPTURE_ROOT/cargo" + cp -- "$manifest_dir/Cargo.lock" "$CAPTURE_ROOT/cargo/Cargo.lock" + cargo metadata --format-version 1 --locked --manifest-path "$manifest" \ + >"$CAPTURE_ROOT/cargo/metadata.json" + cargo fetch --locked --manifest-path "$manifest" >/dev/null + + while IFS=$'\t' read -r name version license; do + local slug target crate extracted + slug="cargo__${name}__${version}" + target="$RAW_ROOT/$slug" + mkdir -p "$target" + crate="$(find "${CARGO_HOME:-$HOME/.cargo}/registry/cache" -type f \ + -name "${name}-${version}.crate" | head -n 1)" + if [ -z "$crate" ]; then + echo "ERROR: no fetched crate for ${name} ${version}" >&2 + exit 2 + fi + cp -- "$crate" "$target/source.archive" + sha256sum "$target/source.archive" | cut -d' ' -f1 >"$target/source.sha256" + record_members "$crate" "$target/members.txt" + extracted="$(mktemp -d)" + tar -xf "$crate" -C "$extracted" + record_license_files "$extracted" "$target/licenses" + record_hook_sources "$extracted" "$target/hooks" + printf '[]\n' >"$target/native.json" + printf '{}\n' >"$target/bundled_library_licenses.json" + find "$extracted" -maxdepth 3 -type f -name '*.rs' -printf '%P\n' | + LC_ALL=C sort >"$target/parsed_inputs.txt" + jq -n --arg name "$name" --arg version "$version" --arg license "$license" '{ + ecosystem: "cargo", + name: $name, + version: $version, + license_expression: $license, + license: "", + classifiers: [], + distribution_inclusion: ["wheel"], + known_vulnerabilities: [] + }' >"$target/metadata.json" + rm -rf "${extracted:?}" + done < <(jq -r '.packages[] | select(.source != null) | [.name, .version, (.license // "")] | @tsv' \ + "$CAPTURE_ROOT/cargo/metadata.json") +} + +if [ "$MODE" = "install" ]; then + install_gated "$PYTHON_LOCK" + echo "Installed the prescreened release closure from collected bytes." + exit 0 +fi + +case ",${ECOSYSTEMS}," in +*,python,*) + if [ -z "$PYTHON_LOCK" ] || [ ! -f "$PYTHON_LOCK" ]; then + echo "ERROR: --python-lock must name the hash-pinned release lock." >&2 + exit 2 + fi + if [ -z "$DOWNLOAD_ROOT" ]; then + echo "ERROR: --download-root is required so the gated install reuses these bytes." >&2 + exit 2 + fi + capture_python "$PYTHON_LOCK" + ;; +esac + +case ",${ECOSYSTEMS}," in +*,cargo,*) + if [ -z "$CARGO_MANIFEST" ] || [ ! -f "$CARGO_MANIFEST" ]; then + echo "ERROR: --cargo-manifest must name the release Cargo.toml." >&2 + exit 2 + fi + capture_cargo "$CARGO_MANIFEST" + ;; +esac + +echo "Raw dependency capture complete: $(find "$RAW_ROOT" -mindepth 1 -maxdepth 1 -type d | wc -l) dependencies." diff --git a/scripts/ci/release_dependency_gate.py b/scripts/ci/release_dependency_gate.py new file mode 100644 index 0000000000..8989bfea84 --- /dev/null +++ b/scripts/ci/release_dependency_gate.py @@ -0,0 +1,2380 @@ +#!/usr/bin/env python3 +"""Fail-closed pre-publish dependency gate for org releases (issue #2342). + +``scripts/ci/sbom_inventory_aggregator.py`` is a *scheduled, informational* org +SBOM roll-up: it flags GPL/AGPL/NOASSERTION components for governance, but it +is not per-dependency, not fail-closed, and not bound to a release head. This +module is the missing gate. It runs in +``.github/workflows/release-dependency-license-strix-gate.yml`` **before** a +release workflow publishes anything, and it either exits ``0`` or refuses the +release. There is no neutral outcome, no allow-failure, and no bypass. + +Design: the gate is a pure function over *captured* inputs. Workflow steps run +``pip inspect``, ``cargo metadata --locked``, archive listing, ``readelf -d``, +and Strix; each writes a file into a capture directory. This module only reads +files. That split keeps every deterministic decision unit-testable without a +runner and makes the Actions-only parts explicit instead of simulated. + +Capture layout (produced by the workflow, consumed here):: + + / + release.json source repository/SHA + artifact names + python/lock.txt the hash-pinned lock that was installed + python/installed.json `pip inspect` of the build environment + cargo/Cargo.lock the committed Cargo lock + cargo/metadata.json `cargo metadata --format-version 1 --locked` + evidence/.json per-dependency captured evidence + strix/bindings/.json per-dependency Strix structured binding + license-selections.json optional dual-license selections + +Every resolved dependency of both ecosystems must appear in the lock *and* in +the environment/build graph; any asymmetry fails ``LOCK_ENV_MISMATCH`` or +``CARGO_LOCK_GRAPH_MISMATCH``. No dependency is exempt: bootstrap tools such as +``pip`` are pinned in this organization's own ``*-hashes.txt`` files, so a lock +that omits an installed distribution is a defect, not a special case. + +Strix evidence is accepted **only** as a machine-readable binding. A textual +"0 findings" or "No exploitable vulnerabilities detected" is rejected +(``STRIX_TEXTUAL_PASS_REJECTED``), and a missing or malformed binding is a +failure rather than a neutral result. The binding's fail-closed shape and error +type follow ``scripts/ci/strix_evidence_binding.py``, which is imported from +this script's **own** directory so the gate behaves identically wherever the +trusted verifier is materialized. +""" + +from __future__ import annotations + +import argparse +import email.parser +import hashlib +import io +import json +import os +import re +import sys +import stat +import tarfile +import tomllib +import urllib.parse +import uuid +import zipfile +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath +from typing import Any, Iterable, Mapping, Sequence + +try: + from scripts.ci.spdx_license_policy import ( + LICENSE_MISSING, + LICENSE_TEXT_MISSING, + LICENSE_TEXT_UNVERIFIED, + LicenseDecision, + evaluate_license_expression, + recognize_license_text, + scan_license_text, + spdx_from_classifiers, + ) +except ImportError: # pragma: no cover - direct `python3 -I