From 62892c62c1ce491b73cf6959744c2a2a331a5d48 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:23:30 +0000 Subject: [PATCH 1/3] Run vulnerability remediation fallback agent with Pi and no write credentials --- .../workflows/vuln-remediation-self-test.yml | 8 + .github/workflows/vuln-remediation.yml | 311 +++++++++++------- .../workflows/vuln-remediation/fix-prompt.md | 4 +- scripts/test_vuln_remediation_workflow.py | 239 ++++++++++++++ 4 files changed, 449 insertions(+), 113 deletions(-) create mode 100644 scripts/test_vuln_remediation_workflow.py diff --git a/.github/workflows/vuln-remediation-self-test.yml b/.github/workflows/vuln-remediation-self-test.yml index 8e0ccc2..62d22ae 100644 --- a/.github/workflows/vuln-remediation-self-test.yml +++ b/.github/workflows/vuln-remediation-self-test.yml @@ -8,6 +8,7 @@ on: - '.github/workflows/vuln-remediation/**' - 'scripts/vuln_remediation.py' - 'scripts/test_vuln_remediation.py' + - 'scripts/test_vuln_remediation_workflow.py' push: branches: - main @@ -17,6 +18,7 @@ on: - '.github/workflows/vuln-remediation/**' - 'scripts/vuln_remediation.py' - 'scripts/test_vuln_remediation.py' + - 'scripts/test_vuln_remediation_workflow.py' workflow_dispatch: inputs: run-live-remediation: @@ -36,6 +38,12 @@ jobs: - name: Run unit and fixture tests run: python3 -m unittest scripts.test_vuln_remediation + - name: Run workflow credential-boundary tests + run: | + python3 -m venv /tmp/vuln-remediation-tests + /tmp/vuln-remediation-tests/bin/pip install PyYAML==6.0.3 + /tmp/vuln-remediation-tests/bin/python -m unittest discover -s scripts -p test_vuln_remediation_workflow.py + self-remediation: name: Manual self remediation if: github.event_name == 'workflow_dispatch' && inputs['run-live-remediation'] diff --git a/.github/workflows/vuln-remediation.yml b/.github/workflows/vuln-remediation.yml index 3026858..f5b0877 100644 --- a/.github/workflows/vuln-remediation.yml +++ b/.github/workflows/vuln-remediation.yml @@ -34,19 +34,22 @@ on: required: false default: 1 +permissions: + contents: read + jobs: scan: runs-on: ubuntu-latest steps: - name: Generate app token id: app-token - uses: actions/create-github-app-token@v1 + uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 with: app-id: ${{ secrets.ADMIN_APP_ID }} private-key: ${{ secrets.ADMIN_APP_PRIVATE_KEY }} - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: token: ${{ steps.app-token.outputs.token }} @@ -117,7 +120,7 @@ jobs: fi - name: Upload scan report - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: socket-report path: socket-report.json @@ -130,7 +133,7 @@ jobs: --output remediation-input.json - name: Upload normalized remediation input - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: remediation-input path: remediation-input.json @@ -140,10 +143,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Download normalized remediation input - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: remediation-input @@ -180,7 +183,7 @@ jobs: cp remediation-context.json triage-result.json - name: Upload triage results - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: triage-result path: | @@ -191,56 +194,61 @@ jobs: fix: needs: triage runs-on: ubuntu-latest + outputs: + publish: ${{ steps.result.outputs.publish }} + base_sha: ${{ steps.branch.outputs.base_sha }} + update_sha: ${{ steps.branch.outputs.update_sha }} steps: - - name: Generate app token - id: app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.ADMIN_APP_ID }} - private-key: ${{ secrets.ADMIN_APP_PRIVATE_KEY }} - - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 - token: ${{ steps.app-token.outputs.token }} + persist-credentials: false - name: Download triage results - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: triage-result + path: ${{ runner.temp }}/remediation - name: Check for fixable alerts id: check run: | - if ! [ -f remediation-context.json ]; then - echo "skip=true" >> $GITHUB_OUTPUT + CONTEXT="$RUNNER_TEMP/remediation/remediation-context.json" + if ! [ -f "$CONTEXT" ]; then + echo "skip=true" >> "$GITHUB_OUTPUT" exit 0 fi - FIX_COUNT=$(jq '.fixes | length' remediation-context.json 2>/dev/null || echo "0") + FIX_COUNT=$(jq '.fixes | length' "$CONTEXT" 2>/dev/null || echo "0") if [ "$FIX_COUNT" = "0" ]; then - echo "skip=true" >> $GITHUB_OUTPUT + echo "skip=true" >> "$GITHUB_OUTPUT" else - echo "skip=false" >> $GITHUB_OUTPUT + echo "skip=false" >> "$GITHUB_OUTPUT" fi - - name: Configure git identity and private module access - if: steps.check.outputs.skip != 'true' - run: | - git config user.name "kernel-internal[bot]" - git config user.email "260533166+kernel-internal[bot]@users.noreply.github.com" - git config --global url."https://x-access-token:${{ steps.app-token.outputs.token }}@github.com/".insteadOf "https://github.com/" - env: - GOPRIVATE: github.com/kernel/* - - name: Reset remediation branch to main + id: branch if: steps.check.outputs.skip != 'true' run: | + echo "base_sha=$(git rev-parse origin/main)" >> "$GITHUB_OUTPUT" + if git show-ref --verify --quiet refs/remotes/origin/security/vuln-remediation; then + echo "update_sha=$(git rev-parse origin/security/vuln-remediation)" >> "$GITHUB_OUTPUT" + fi git checkout -B security/vuln-remediation origin/main + # Read-only and scoped to this repository. Revoked before the agent runs. + - name: Generate private module read token + id: read-token + if: steps.check.outputs.skip != 'true' + uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 + with: + app-id: ${{ secrets.ADMIN_APP_ID }} + private-key: ${{ secrets.ADMIN_APP_PRIVATE_KEY }} + permission-contents: read + - name: Setup Go if: steps.check.outputs.skip != 'true' && inputs.go-version-file != '' - uses: actions/setup-go@v5 + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 with: go-version-file: ${{ inputs.go-version-file }} env: @@ -248,15 +256,19 @@ jobs: - name: Setup Bun if: steps.check.outputs.skip != 'true' && inputs.setup-bun - uses: oven-sh/setup-bun@v2 + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 - name: Install JS dependencies if: steps.check.outputs.skip != 'true' && inputs.setup-bun + env: + GIT_CONFIG_COUNT: '1' + GIT_CONFIG_KEY_0: url.https://x-access-token:${{ steps.read-token.outputs.token }}@github.com/.insteadOf + GIT_CONFIG_VALUE_0: https://github.com/ run: bun install - name: Setup Python if: steps.check.outputs.skip != 'true' && inputs.setup-python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ inputs.python-version }} @@ -264,39 +276,35 @@ jobs: if: steps.check.outputs.skip != 'true' && inputs.setup-python run: pip install uv - - name: Install Cursor CLI + - name: Setup Node if: steps.check.outputs.skip != 'true' - run: | - curl https://cursor.com/install -fsS | bash - echo "$HOME/.cursor/bin" >> $GITHUB_PATH - - - name: Enable Cursor Max Mode - if: steps.check.outputs.skip != 'true' - run: | - CFG_DIR="$HOME/.cursor" - mkdir -p "$CFG_DIR" - echo '{"maxMode": true}' > "$CFG_DIR/cli-config.json" - echo "CURSOR_CONFIG_DIR=$CFG_DIR" >> "$GITHUB_ENV" + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22' - name: Install Socket CLI if: steps.check.outputs.skip != 'true' run: npm install -g @socketsecurity/cli - - name: Apply fixes + - name: Apply Socket fixes + id: socket-fix if: steps.check.outputs.skip != 'true' timeout-minutes: 10 env: - CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }} + GIT_CONFIG_COUNT: '1' + GIT_CONFIG_KEY_0: url.https://x-access-token:${{ steps.read-token.outputs.token }}@github.com/.insteadOf + GIT_CONFIG_VALUE_0: https://github.com/ GOPRIVATE: github.com/kernel/* SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_API_TOKEN }} run: | + WORK="$RUNNER_TEMP/remediation" set +e - FIX_IDS=$(jq -r '.fixes[].id' remediation-context.json | paste -sd, -) + FIX_IDS=$(jq -r '.fixes[].id' "$WORK/remediation-context.json" | paste -sd, -) socket fix . \ --id "$FIX_IDS" \ --no-major-updates \ --json \ - --output-file socket-fix-apply.json + --output-file "$WORK/socket-fix-apply.json" SOCKET_FIX_EXIT=$? set -e if [ $SOCKET_FIX_EXIT -ne 0 ]; then @@ -304,33 +312,87 @@ jobs: fi if git diff --quiet; then - export DATE="$(date -u +%Y-%m-%d)" - curl -fsSL https://raw.githubusercontent.com/kernel/security-workflows/${{ inputs.security-workflows-ref }}/.github/workflows/vuln-remediation/fix-prompt.md | envsubst '${GITHUB_REPOSITORY} ${DATE}' | agent -p --model ${{ vars.CURSOR_PREFERRED_MODEL }} --workspace . --trust --force --output-format=text + echo "run_agent=true" >> "$GITHUB_OUTPUT" fi - - name: Validate remediation diff + - name: Revoke private module read token + if: always() && steps.read-token.outcome == 'success' + env: + GH_TOKEN: ${{ steps.read-token.outputs.token }} + run: gh api --method DELETE /installation/token + + - name: Install Pi + if: steps.socket-fix.outputs.run_agent == 'true' + env: + PI_TELEMETRY: '0' + run: | + npm install -g --ignore-scripts @earendil-works/pi-coding-agent + pi --version + + # The agent has no GitHub credentials. Later steps validate its diff and + # the pr job publishes it. + - name: Run fallback agent + if: steps.socket-fix.outputs.run_agent == 'true' + timeout-minutes: 20 + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PI_PREFERRED_MODEL: ${{ vars.PI_PREFERRED_MODEL }} + PI_SKIP_VERSION_CHECK: '1' + PI_TELEMETRY: '0' + GOPRIVATE: github.com/kernel/* + PROMPT_URL: https://raw.githubusercontent.com/kernel/security-workflows/${{ inputs.security-workflows-ref }}/.github/workflows/vuln-remediation/fix-prompt.md + run: | + : "${PI_PREFERRED_MODEL:?Set the PI_PREFERRED_MODEL organization variable}" + cp "$RUNNER_TEMP/remediation/remediation-context.json" remediation-context.json + DATE="$(date -u +%Y-%m-%d)" + export DATE + # shellcheck disable=SC2016 # envsubst takes the variable names literally + curl -fsSL "$PROMPT_URL" | envsubst '${GITHUB_REPOSITORY} ${DATE}' > "$RUNNER_TEMP/fix-prompt.md" + pi --print --no-session --approve \ + --model "$PI_PREFERRED_MODEL" --thinking max \ + @"$RUNNER_TEMP/fix-prompt.md" + + - name: Snapshot remediation diff id: diff if: steps.check.outputs.skip != 'true' + env: + AGENT_BASE: ${{ steps.branch.outputs.base_sha }} run: | - curl -fsSL https://raw.githubusercontent.com/kernel/security-workflows/${{ inputs.security-workflows-ref }}/scripts/vuln_remediation.py -o /tmp/vuln_remediation.py - git diff --name-only > changed-files.txt - if [ ! -s changed-files.txt ]; then - echo '{"fixed": [], "reverted": [], "summary": "No dependency changes were produced."}' > fix-result.json - echo "has_changes=false" >> $GITHUB_OUTPUT + WORK="$RUNNER_TEMP/remediation" + # Capture tracked changes, committed or not, against the pre-agent base, + # then validate only that patch on a clean tree. + git reset -q --soft "$AGENT_BASE" + git add -u + git diff --cached --name-only --no-renames > "$WORK/changed-files.txt" + git diff --cached --binary --no-renames --no-ext-diff --no-textconv > "$WORK/changes.patch" + git reset -q --hard + git clean -fdq + if [ ! -s "$WORK/changed-files.txt" ]; then + echo '{"fixed": [], "reverted": [], "summary": "No dependency changes were produced."}' > "$WORK/fix-result.json" + echo "has_changes=false" >> "$GITHUB_OUTPUT" exit 0 fi - echo "has_changes=true" >> $GITHUB_OUTPUT + git apply --index "$WORK/changes.patch" + echo "has_changes=true" >> "$GITHUB_OUTPUT" + + - name: Validate remediation diff + if: steps.check.outputs.skip != 'true' && steps.diff.outputs.has_changes == 'true' + run: | + WORK="$RUNNER_TEMP/remediation" + curl -fsSL https://raw.githubusercontent.com/kernel/security-workflows/${{ inputs.security-workflows-ref }}/scripts/vuln_remediation.py -o /tmp/vuln_remediation.py python3 /tmp/vuln_remediation.py validate-diff \ --repo-root . \ - --context remediation-context.json \ - --changed-files changed-files.txt \ - --output diff-validation.json + --context "$WORK/remediation-context.json" \ + --changed-files "$WORK/changed-files.txt" \ + --output "$WORK/diff-validation.json" - name: Build post-fix Socket plan if: steps.check.outputs.skip != 'true' && steps.diff.outputs.has_changes == 'true' env: SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_API_TOKEN }} run: | + WORK="$RUNNER_TEMP/remediation" set +e socket fix . \ --all \ @@ -338,111 +400,136 @@ jobs: --no-major-updates \ --show-affected-direct-dependencies \ --json \ - --output-file post-fix-plan.json + --output-file "$WORK/post-fix-plan.json" POST_PLAN_EXIT=$? set -e - if [ $POST_PLAN_EXIT -ne 0 ] || ! jq empty post-fix-plan.json 2>/dev/null; then + if [ $POST_PLAN_EXIT -ne 0 ] || ! jq empty "$WORK/post-fix-plan.json" 2>/dev/null; then echo "::warning::Post-fix Socket planning failed; confirmation will fail closed." - echo '{"fixes":{"__post_fix_plan_failed__":{}}}' > post-fix-plan.json + echo '{"fixes":{"__post_fix_plan_failed__":{}}}' > "$WORK/post-fix-plan.json" fi - name: Confirm fixes if: steps.check.outputs.skip != 'true' && steps.diff.outputs.has_changes == 'true' run: | + WORK="$RUNNER_TEMP/remediation" set +e python3 /tmp/vuln_remediation.py confirm \ --repo-root . \ - --context remediation-context.json \ - --post-fix-plan post-fix-plan.json \ - --output confirmation-result.json + --context "$WORK/remediation-context.json" \ + --post-fix-plan "$WORK/post-fix-plan.json" \ + --output "$WORK/confirmation-result.json" CONFIRM_EXIT=$? set -e python3 /tmp/vuln_remediation.py summarize-fix \ - --context remediation-context.json \ - --confirmation confirmation-result.json \ - --output fix-result.json + --context "$WORK/remediation-context.json" \ + --confirmation "$WORK/confirmation-result.json" \ + --output "$WORK/fix-result.json" exit $CONFIRM_EXIT - - name: Commit and push validated changes + - name: Check for confirmed fixes + id: result if: steps.check.outputs.skip != 'true' && steps.diff.outputs.has_changes == 'true' run: | - if ! jq -e '.fixed | length > 0' fix-result.json >/dev/null; then - echo "No confirmed fixes to commit." - exit 0 + if jq -e '.fixed | length > 0' "$RUNNER_TEMP/remediation/fix-result.json" >/dev/null; then + echo "publish=true" >> "$GITHUB_OUTPUT" + else + echo "No confirmed fixes to publish." fi - git add --pathspec-from-file=changed-files.txt - git diff --cached --quiet && exit 0 - git commit -m "security: vulnerability remediation ($(date -u +%Y-%m-%d))" - git push --force-with-lease origin security/vuln-remediation - name: Upload fix results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: fix-result path: | - fix-result.json - confirmation-result.json - diff-validation.json - changed-files.txt - socket-fix-apply.json - post-fix-plan.json + ${{ runner.temp }}/remediation/fix-result.json + ${{ runner.temp }}/remediation/confirmation-result.json + ${{ runner.temp }}/remediation/diff-validation.json + ${{ runner.temp }}/remediation/changed-files.txt + ${{ runner.temp }}/remediation/changes.patch + ${{ runner.temp }}/remediation/socket-fix-apply.json + ${{ runner.temp }}/remediation/post-fix-plan.json if-no-files-found: ignore + # Publishes the validated patch. Never runs repository code, package managers, or the agent. pr: needs: [triage, fix] - if: always() + if: needs.fix.outputs.publish == 'true' runs-on: ubuntu-latest + env: + UPDATE_SHA: ${{ needs.fix.outputs.update_sha }} steps: - - name: Generate app token - id: app-token - uses: actions/create-github-app-token@v1 + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: - app-id: ${{ secrets.ADMIN_APP_ID }} - private-key: ${{ secrets.ADMIN_APP_PRIVATE_KEY }} + ref: ${{ needs.fix.outputs.base_sha }} + persist-credentials: false - name: Download triage results - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: triage-result - continue-on-error: true + path: ${{ runner.temp }}/remediation - name: Download fix results - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: fix-result - continue-on-error: true + path: ${{ runner.temp }}/remediation - - name: Render PR body + - name: Apply validated changes run: | + WORK="$RUNNER_TEMP/remediation" + git config user.name "kernel-internal[bot]" + git config user.email "260533166+kernel-internal[bot]@users.noreply.github.com" + git switch -c security/vuln-remediation + git apply --index "$WORK/changes.patch" + git diff --cached --name-only --no-renames > "$RUNNER_TEMP/publish-files.txt" + if ! cmp -s "$RUNNER_TEMP/publish-files.txt" "$WORK/changed-files.txt"; then + echo "::error::The patch does not match the files that were validated." + exit 1 + fi curl -fsSL https://raw.githubusercontent.com/kernel/security-workflows/${{ inputs.security-workflows-ref }}/scripts/vuln_remediation.py -o /tmp/vuln_remediation.py - [ -f triage-result.json ] || echo '{"deferred":[]}' > triage-result.json - [ -f fix-result.json ] || echo '{"fixed":[],"reverted":[]}' > fix-result.json - [ -f confirmation-result.json ] || echo '{"ok":false,"confirmed":[],"failures":[]}' > confirmation-result.json + python3 /tmp/vuln_remediation.py validate-diff \ + --repo-root . \ + --context "$WORK/remediation-context.json" \ + --changed-files "$RUNNER_TEMP/publish-files.txt" + if git diff --cached --raw --no-renames | awk '$2 !~ /^(100644|000000)$/ { found = 1 } END { exit !found }'; then + echo "::error::Validated changes may only contain regular, non-executable files." + exit 1 + fi + git commit -q -m "security: vulnerability remediation ($(date -u +%Y-%m-%d))" + + - name: Render PR body + run: | + cd "$RUNNER_TEMP/remediation" python3 /tmp/vuln_remediation.py render-pr-body \ --triage triage-result.json \ --fix-result fix-result.json \ --confirmation confirmation-result.json \ - --output pr-body.md + --output "$RUNNER_TEMP/pr-body.md" - - name: Create or update PR + - name: Generate app token + id: app-token + uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 + with: + app-id: ${{ secrets.ADMIN_APP_ID }} + private-key: ${{ secrets.ADMIN_APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + + - name: Push and create or update PR env: GH_TOKEN: ${{ steps.app-token.outputs.token }} run: | - if [ "${{ needs.fix.result }}" != "success" ]; then - echo "Fix job did not complete successfully; not creating or updating a remediation PR." - exit 0 - fi REPO="${{ github.repository }}" - if ! jq -e '.fixed | length > 0' fix-result.json >/dev/null 2>&1; then - echo "No confirmed fixes; not creating a PR." - exit 0 - fi + gh auth setup-git + git push --force-with-lease="security/vuln-remediation:$UPDATE_SHA" origin security/vuln-remediation EXISTING_PR=$(gh pr list --repo "$REPO" --head security/vuln-remediation --state open --json number --jq '.[0].number' 2>/dev/null || echo "") if [ -n "$EXISTING_PR" ] && [ "$EXISTING_PR" != "null" ]; then - gh pr edit "$EXISTING_PR" --repo "$REPO" --body-file pr-body.md --add-assignee ulziibay-kernel --add-reviewer ulziibay-kernel + gh pr edit "$EXISTING_PR" --repo "$REPO" --body-file "$RUNNER_TEMP/pr-body.md" --add-assignee ulziibay-kernel --add-reviewer ulziibay-kernel echo "Updated PR #${EXISTING_PR}" else gh pr create \ @@ -452,5 +539,5 @@ jobs: --title "security: vulnerability remediation" \ --assignee ulziibay-kernel \ --reviewer ulziibay-kernel \ - --body-file pr-body.md + --body-file "$RUNNER_TEMP/pr-body.md" fi diff --git a/.github/workflows/vuln-remediation/fix-prompt.md b/.github/workflows/vuln-remediation/fix-prompt.md index 500351a..49a23a0 100644 --- a/.github/workflows/vuln-remediation/fix-prompt.md +++ b/.github/workflows/vuln-remediation/fix-prompt.md @@ -2,7 +2,7 @@ You are a security engineer performing bounded fallback dependency remediation. IMPORTANT: The workflow has already asked Socket to plan and apply fixes. You are only invoked when Socket did not leave a dependency diff. Start by reading the local JSON files and then execute the smallest package-manager command needed. -Git is available only for inspection. You do not have authority to create commits, push branches, create PRs, force-push, or stage files. +Git is available only for inspection. You have no GitHub credentials. Do not create commits, push branches, create PRs, force-push, or stage files. The workflow validates and publishes your changes to tracked files. It discards untracked files, and a change to any file other than a dependency manifest or lockfile fails validation. # Context @@ -37,6 +37,8 @@ go mod tidy ``` If no target version is available, do not guess; record the fix as reverted with reason `No Socket target version available for fallback`. +Private `github.com/kernel/*` modules cannot be downloaded. If a fix needs one, revert it and record the download error as the reason. + ### npm (`package.json` / `package-lock.json` / `pnpm-lock.yaml`) `cd` into the directory containing `package.json`, then inspect `packageManager`. diff --git a/scripts/test_vuln_remediation_workflow.py b/scripts/test_vuln_remediation_workflow.py new file mode 100644 index 0000000..adf3b66 --- /dev/null +++ b/scripts/test_vuln_remediation_workflow.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Credential-boundary tests for .github/workflows/vuln-remediation.yml.""" +from __future__ import annotations + +import os +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = yaml.safe_load((ROOT / ".github/workflows/vuln-remediation.yml").read_text()) +JOBS = WORKFLOW["jobs"] +TOKEN_REF = re.compile(r"GH_TOKEN|GITHUB_TOKEN|steps\.[\w-]+\.outputs\.token|github\.token") + + +def step(job: str, name: str) -> dict: + for entry in JOBS[job]["steps"]: + if entry.get("name") == name: + return entry + raise AssertionError(f"missing {job} step: {name}") + + +def index(job: str, name: str) -> int: + return JOBS[job]["steps"].index(step(job, name)) + + +def git(cwd: Path, *args: str) -> str: + return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout + + +def init_repo(path: Path) -> None: + path.mkdir() + git(path, "init", "-q", "-b", "main") + git(path, "config", "user.email", "test@example.com") + git(path, "config", "user.name", "Test") + (path / "package.json").write_text('{"dependencies": {"left-pad": "1.0.0"}}\n') + (path / "package-lock.json").write_text('{"lockfileVersion": 3, "left-pad": "1.0.0"}\n') + (path / "index.js").write_text("console.log(1)\n") + git(path, "add", "-A") + git(path, "commit", "-q", "-m", "base") + + +def run_step(entry: dict, cwd: Path, env: dict[str, str]) -> subprocess.CompletedProcess: + return subprocess.run(["bash", "-e", "-c", entry["run"]], cwd=cwd, env={**os.environ, **env}, capture_output=True, text=True) + + +class WorkflowCredentialBoundary(unittest.TestCase): + def test_actions_are_pinned_and_checkouts_do_not_persist_credentials(self): + for job_name, job in JOBS.items(): + for entry in job["steps"]: + uses = entry.get("uses") + if not uses: + continue + self.assertRegex(uses, r"^[\w/-]+@[0-9a-f]{40}$", f"{job_name}: {uses}") + if job_name in ("fix", "pr") and uses.startswith("actions/checkout@"): + self.assertIs(entry["with"]["persist-credentials"], False) + self.assertNotIn("token", entry["with"]) + + def test_default_token_is_read_only(self): + self.assertEqual(WORKFLOW["permissions"], {"contents": "read"}) + + def test_fix_job_only_has_a_revoked_read_token(self): + tokens = [entry for entry in JOBS["fix"]["steps"] if "create-github-app-token" in entry.get("uses", "")] + self.assertEqual(len(tokens), 1) + read_token = tokens[0]["with"] + self.assertEqual(read_token["permission-contents"], "read") + self.assertFalse([key for key, value in read_token.items() if key.startswith("permission-") and value != "read"]) + self.assertNotIn("owner", read_token) + self.assertNotIn("repositories", read_token) + + revoke = step("fix", "Revoke private module read token") + self.assertEqual(revoke["run"].strip(), "gh api --method DELETE /installation/token") + self.assertEqual(revoke["env"]["GH_TOKEN"], "${{ steps.read-token.outputs.token }}") + self.assertIn("always()", revoke["if"]) + + token_users = [ + entry["name"] + for entry in JOBS["fix"]["steps"] + if "steps.read-token.outputs.token" in yaml.safe_dump(entry) + ] + self.assertEqual(token_users, ["Install JS dependencies", "Apply Socket fixes", "Revoke private module read token"]) + for name in token_users: + self.assertLess(index("fix", name), index("fix", "Install Pi")) + for entry in JOBS["fix"]["steps"]: + self.assertNotIn("git config --global", entry.get("run", "")) + + def test_agent_step_has_only_model_credentials(self): + agent = step("fix", "Run fallback agent") + env = agent["env"] + secrets = sorted(key for key, value in env.items() if "secrets." in str(value)) + self.assertEqual(secrets, ["ANTHROPIC_API_KEY", "OPENAI_API_KEY"]) + self.assertEqual(env["PI_PREFERRED_MODEL"], "${{ vars.PI_PREFERRED_MODEL }}") + self.assertNotRegex(yaml.safe_dump(agent), TOKEN_REF) + self.assertIn("security-workflows-ref", env["PROMPT_URL"]) + self.assertIn("pi --print --no-session --approve", agent["run"]) + self.assertIn('@"$RUNNER_TEMP/fix-prompt.md"', agent["run"]) + self.assertNotIn("CURSOR", yaml.safe_dump(JOBS)) + for name in ("Snapshot remediation diff", "Validate remediation diff", "Confirm fixes"): + self.assertLess(index("fix", "Run fallback agent"), index("fix", name)) + + def test_publish_job_never_runs_repository_code(self): + pr = JOBS["pr"] + self.assertEqual(pr["needs"], ["triage", "fix"]) + self.assertEqual(pr["if"], "needs.fix.outputs.publish == 'true'") + runs = "\n".join(entry.get("run", "") for entry in pr["steps"]) + self.assertNotRegex(runs, r"(?m)^\s*(go|bun|npm|npx|node|pi|make|socket|uv|pip)\b") + self.assertNotIn("secrets.ANTHROPIC_API_KEY", yaml.safe_dump(pr)) + self.assertNotIn("secrets.SOCKET_API_TOKEN", yaml.safe_dump(pr)) + for entry in pr["steps"]: + self.assertNotIn("continue-on-error", entry) + mint = index("pr", "Generate app token") + self.assertLess(index("pr", "Apply validated changes"), mint) + for entry in pr["steps"][:mint]: + self.assertNotRegex(yaml.safe_dump(entry), TOKEN_REF) + self.assertEqual(mint, len(pr["steps"]) - 2) + + +class SnapshotStep(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) + self.repo = self.root / "repo" + init_repo(self.repo) + self.base = git(self.repo, "rev-parse", "HEAD").strip() + self.work = self.root / "remediation" + self.work.mkdir() + self.output = self.root / "output" + self.env = {"RUNNER_TEMP": str(self.root), "GITHUB_OUTPUT": str(self.output), "AGENT_BASE": self.base} + + def tearDown(self): + self.tmp.cleanup() + + def snapshot(self) -> subprocess.CompletedProcess: + return run_step(step("fix", "Snapshot remediation diff"), self.repo, self.env) + + def test_captures_committed_and_uncommitted_tracked_changes_on_a_clean_tree(self): + (self.repo / "package.json").write_text('{"dependencies": {"left-pad": "1.3.0"}}\n') + git(self.repo, "commit", "-qam", "agent commit") + (self.repo / "package-lock.json").write_text('{"lockfileVersion": 3, "left-pad": "1.3.0"}\n') + (self.repo / "bun.lock").write_text("untracked\n") + (self.repo / "remediation-context.json").write_text("{}\n") + + result = self.snapshot() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("has_changes=true", self.output.read_text()) + self.assertEqual((self.work / "changed-files.txt").read_text().split(), ["package-lock.json", "package.json"]) + self.assertEqual(git(self.repo, "rev-parse", "HEAD").strip(), self.base) + self.assertFalse((self.repo / "bun.lock").exists()) + self.assertFalse((self.repo / "remediation-context.json").exists()) + self.assertEqual(git(self.repo, "diff", "--cached", "--name-only").split(), ["package-lock.json", "package.json"]) + + def test_no_changes(self): + (self.repo / "fix-result.json").write_text("{}\n") + result = self.snapshot() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("has_changes=false", self.output.read_text()) + self.assertIn("No dependency changes", (self.work / "fix-result.json").read_text()) + + +class PublishStep(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) + self.repo = self.root / "repo" + init_repo(self.repo) + self.work = self.root / "remediation" + self.work.mkdir() + (self.work / "remediation-context.json").write_text('{"fixes": []}\n') + bin_dir = self.root / "bin" + bin_dir.mkdir() + curl = bin_dir / "curl" + curl.write_text(f'#!/bin/sh\nwhile [ "$1" != "-o" ]; do shift; done\ncp {ROOT / "scripts/vuln_remediation.py"} "$2"\n') + curl.chmod(0o755) + self.env = {"RUNNER_TEMP": str(self.root), "PATH": f"{bin_dir}:{os.environ['PATH']}"} + + def tearDown(self): + self.tmp.cleanup() + + def stage(self, files: dict[str, str], executable: bool = False) -> None: + source = self.root / "source" + git(self.root, "clone", "-q", str(self.repo), str(source)) + for path, content in files.items(): + (source / path).write_text(content) + if executable: + (source / path).chmod(0o755) + git(source, "add", "-A") + (self.work / "changes.patch").write_text(git(source, "diff", "--cached", "--binary", "--no-renames")) + (self.work / "changed-files.txt").write_text(git(source, "diff", "--cached", "--name-only", "--no-renames")) + + def publish(self) -> subprocess.CompletedProcess: + script = step("pr", "Apply validated changes")["run"].replace("${{ inputs.security-workflows-ref }}", "main") + return run_step({"run": script}, self.repo, self.env) + + def test_commits_validated_dependency_changes(self): + self.stage({"package-lock.json": '{"lockfileVersion": 3, "left-pad": "1.3.0"}\n'}) + result = self.publish() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(git(self.repo, "branch", "--show-current").strip(), "security/vuln-remediation") + self.assertEqual(git(self.repo, "show", "--name-only", "--format=", "HEAD").split(), ["package-lock.json"]) + + def test_rejects_source_changes(self): + self.stage({"index.js": "console.log(2)\n"}) + result = self.publish() + self.assertNotEqual(result.returncode, 0) + self.assertIn("only dependency manifests and lockfiles may change", result.stderr) + + def test_rejects_executable_files(self): + self.stage({"package-lock.json": "#!/bin/sh\n"}, executable=True) + self.assertIn("100755", (self.work / "changes.patch").read_text()) + result = self.publish() + self.assertNotEqual(result.returncode, 0) + self.assertIn("executable files are not allowed", result.stderr) + + def test_rejects_symlinks(self): + self.stage({}) + source = self.root / "source" + (source / "package-lock.json").unlink() + (source / "package-lock.json").symlink_to("index.js") + git(source, "add", "-A") + (self.work / "changes.patch").write_text(git(source, "diff", "--cached", "--binary", "--no-renames")) + (self.work / "changed-files.txt").write_text(git(source, "diff", "--cached", "--name-only", "--no-renames")) + result = self.publish() + self.assertNotEqual(result.returncode, 0) + self.assertIn("regular, non-executable files", result.stdout + result.stderr) + + def test_rejects_patch_that_differs_from_validated_files(self): + self.stage({"package-lock.json": '{"lockfileVersion": 3}\n'}) + (self.work / "changed-files.txt").write_text("package.json\n") + result = self.publish() + self.assertNotEqual(result.returncode, 0) + self.assertIn("does not match", result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main() From 5e4d780f4f004e073f383b72a616b3e44a9523a5 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:51:42 +0000 Subject: [PATCH 2/3] Run the remediation agent in a job that references only model keys --- .github/workflows/vuln-remediation.yml | 227 +++++++++++++++--- .../workflows/vuln-remediation/fix-prompt.md | 2 +- scripts/test_vuln_remediation_workflow.py | 227 ++++++++++-------- 3 files changed, 320 insertions(+), 136 deletions(-) diff --git a/.github/workflows/vuln-remediation.yml b/.github/workflows/vuln-remediation.yml index f5b0877..7134ea6 100644 --- a/.github/workflows/vuln-remediation.yml +++ b/.github/workflows/vuln-remediation.yml @@ -191,13 +191,15 @@ jobs: remediation-context.json socket-fix-plan.json + # Runs Socket's fix with Socket and a read-only GitHub token. Never runs the agent. fix: needs: triage runs-on: ubuntu-latest outputs: - publish: ${{ steps.result.outputs.publish }} base_sha: ${{ steps.branch.outputs.base_sha }} update_sha: ${{ steps.branch.outputs.update_sha }} + has_changes: ${{ steps.snapshot.outputs.has_changes }} + run_agent: ${{ steps.snapshot.outputs.run_agent }} steps: - name: Checkout repository uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 @@ -236,7 +238,6 @@ jobs: fi git checkout -B security/vuln-remediation origin/main - # Read-only and scoped to this repository. Revoked before the agent runs. - name: Generate private module read token id: read-token if: steps.check.outputs.skip != 'true' @@ -276,18 +277,11 @@ jobs: if: steps.check.outputs.skip != 'true' && inputs.setup-python run: pip install uv - - name: Setup Node - if: steps.check.outputs.skip != 'true' - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 - with: - node-version: '22' - - name: Install Socket CLI if: steps.check.outputs.skip != 'true' run: npm install -g @socketsecurity/cli - name: Apply Socket fixes - id: socket-fix if: steps.check.outputs.skip != 'true' timeout-minutes: 10 env: @@ -311,28 +305,145 @@ jobs: echo "::warning::Socket fix failed; invoking bounded fallback agent." fi - if git diff --quiet; then + - name: Snapshot Socket changes + id: snapshot + if: steps.check.outputs.skip != 'true' + run: | + WORK="$RUNNER_TEMP/remediation" + git add -u + git diff --cached --name-only --no-renames > "$WORK/changed-files.txt" + git diff --cached --binary --no-renames --no-ext-diff --no-textconv > "$WORK/changes.patch" + if [ -s "$WORK/changed-files.txt" ]; then + echo "has_changes=true" >> "$GITHUB_OUTPUT" + else echo "run_agent=true" >> "$GITHUB_OUTPUT" fi - - name: Revoke private module read token - if: always() && steps.read-token.outcome == 'success' + # The agent job has no GitHub credentials, so hand it the private Go + # modules it may need as a module cache subtree. + - name: Download private Go modules + if: steps.snapshot.outputs.run_agent == 'true' && inputs.go-version-file != '' env: - GH_TOKEN: ${{ steps.read-token.outputs.token }} - run: gh api --method DELETE /installation/token + GIT_CONFIG_COUNT: '1' + GIT_CONFIG_KEY_0: url.https://x-access-token:${{ steps.read-token.outputs.token }}@github.com/.insteadOf + GIT_CONFIG_VALUE_0: https://github.com/ + GOPRIVATE: github.com/kernel/* + run: | + jq -r '.fixes[].manifests[]? | select(endswith("go.mod") or endswith("go.sum"))' \ + "$RUNNER_TEMP/remediation/remediation-context.json" \ + | xargs -r -n1 dirname | sort -u | while read -r dir; do + go -C "$dir" mod download || echo "::warning::Could not download Go modules for $dir" + done + MODULES="$(go env GOMODCACHE)/cache/download/github.com/kernel" + mkdir -p "$MODULES" + tar -czf "$RUNNER_TEMP/go-private-modules.tgz" -C "$MODULES" . + + - name: Upload private Go modules + if: steps.snapshot.outputs.run_agent == 'true' && inputs.go-version-file != '' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: go-private-modules + path: ${{ runner.temp }}/go-private-modules.tgz + retention-days: 1 + + - name: Upload Socket changes + if: steps.snapshot.outputs.has_changes == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: remediation-patch + path: | + ${{ runner.temp }}/remediation/changes.patch + ${{ runner.temp }}/remediation/changed-files.txt + + - name: Upload Socket fix output + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: socket-fix-apply + path: ${{ runner.temp }}/remediation/socket-fix-apply.json + if-no-files-found: ignore + + # Runs the fallback agent when Socket left no diff. This job must reference no + # secrets other than the model API keys: every secret a job references is in + # runner memory for the whole job, where the agent could read it. + agent: + needs: fix + if: needs.fix.outputs.run_agent == 'true' + runs-on: ubuntu-latest + outputs: + has_changes: ${{ steps.snapshot.outputs.has_changes }} + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + ref: ${{ needs.fix.outputs.base_sha }} + persist-credentials: false + + - name: Download triage results + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: triage-result + path: ${{ runner.temp }}/remediation + + - name: Reset remediation branch to main + run: git checkout -B security/vuln-remediation + + - name: Setup Go + if: inputs.go-version-file != '' + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache: false + env: + GOPRIVATE: github.com/kernel/* + + - name: Download private Go modules + if: inputs.go-version-file != '' + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: go-private-modules + path: ${{ runner.temp }} + + - name: Restore private Go modules + if: inputs.go-version-file != '' + run: | + MODULES="$(go env GOMODCACHE)/cache/download/github.com/kernel" + mkdir -p "$MODULES" + tar -xzf "$RUNNER_TEMP/go-private-modules.tgz" -C "$MODULES" + + - name: Setup Bun + if: inputs.setup-bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + no-cache: true + + - name: Install JS dependencies + if: inputs.setup-bun + run: bun install + + - name: Setup Python + if: inputs.setup-python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ inputs.python-version }} + + - name: Install uv + if: inputs.setup-python + run: pip install uv + + - name: Setup Node + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22' - name: Install Pi - if: steps.socket-fix.outputs.run_agent == 'true' env: PI_TELEMETRY: '0' run: | npm install -g --ignore-scripts @earendil-works/pi-coding-agent pi --version - # The agent has no GitHub credentials. Later steps validate its diff and - # the pr job publishes it. - name: Run fallback agent - if: steps.socket-fix.outputs.run_agent == 'true' timeout-minutes: 20 env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} @@ -353,33 +464,70 @@ jobs: --model "$PI_PREFERRED_MODEL" --thinking max \ @"$RUNNER_TEMP/fix-prompt.md" - - name: Snapshot remediation diff - id: diff - if: steps.check.outputs.skip != 'true' + - name: Snapshot agent changes + id: snapshot env: - AGENT_BASE: ${{ steps.branch.outputs.base_sha }} + AGENT_BASE: ${{ needs.fix.outputs.base_sha }} run: | WORK="$RUNNER_TEMP/remediation" - # Capture tracked changes, committed or not, against the pre-agent base, - # then validate only that patch on a clean tree. + # Capture tracked changes, committed or not, against the pre-agent base. git reset -q --soft "$AGENT_BASE" git add -u git diff --cached --name-only --no-renames > "$WORK/changed-files.txt" git diff --cached --binary --no-renames --no-ext-diff --no-textconv > "$WORK/changes.patch" - git reset -q --hard - git clean -fdq - if [ ! -s "$WORK/changed-files.txt" ]; then - echo '{"fixed": [], "reverted": [], "summary": "No dependency changes were produced."}' > "$WORK/fix-result.json" - echo "has_changes=false" >> "$GITHUB_OUTPUT" - exit 0 + if [ -s "$WORK/changed-files.txt" ]; then + echo "has_changes=true" >> "$GITHUB_OUTPUT" + else + echo "No dependency changes were produced." fi - git apply --index "$WORK/changes.patch" - echo "has_changes=true" >> "$GITHUB_OUTPUT" + + - name: Upload agent changes + if: steps.snapshot.outputs.has_changes == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: remediation-patch + path: | + ${{ runner.temp }}/remediation/changes.patch + ${{ runner.temp }}/remediation/changed-files.txt + + # Validates and confirms the patch from Socket or the agent on a fresh runner. + verify: + needs: [fix, agent] + if: >- + !cancelled() && needs.fix.result == 'success' && + (needs.fix.outputs.has_changes == 'true' || + (needs.agent.result == 'success' && needs.agent.outputs.has_changes == 'true')) + runs-on: ubuntu-latest + outputs: + publish: ${{ steps.result.outputs.publish }} + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + ref: ${{ needs.fix.outputs.base_sha }} + persist-credentials: false + + - name: Download triage results + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: triage-result + path: ${{ runner.temp }}/remediation + + - name: Download remediation patch + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: remediation-patch + path: ${{ runner.temp }}/remediation - name: Validate remediation diff - if: steps.check.outputs.skip != 'true' && steps.diff.outputs.has_changes == 'true' run: | WORK="$RUNNER_TEMP/remediation" + git apply --index "$WORK/changes.patch" + git diff --cached --name-only --no-renames > "$RUNNER_TEMP/applied-files.txt" + if ! cmp -s "$RUNNER_TEMP/applied-files.txt" "$WORK/changed-files.txt"; then + echo "::error::The patch does not match its file list." + exit 1 + fi curl -fsSL https://raw.githubusercontent.com/kernel/security-workflows/${{ inputs.security-workflows-ref }}/scripts/vuln_remediation.py -o /tmp/vuln_remediation.py python3 /tmp/vuln_remediation.py validate-diff \ --repo-root . \ @@ -387,8 +535,10 @@ jobs: --changed-files "$WORK/changed-files.txt" \ --output "$WORK/diff-validation.json" + - name: Install Socket CLI + run: npm install -g @socketsecurity/cli + - name: Build post-fix Socket plan - if: steps.check.outputs.skip != 'true' && steps.diff.outputs.has_changes == 'true' env: SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_API_TOKEN }} run: | @@ -409,7 +559,6 @@ jobs: fi - name: Confirm fixes - if: steps.check.outputs.skip != 'true' && steps.diff.outputs.has_changes == 'true' run: | WORK="$RUNNER_TEMP/remediation" set +e @@ -428,7 +577,6 @@ jobs: - name: Check for confirmed fixes id: result - if: steps.check.outputs.skip != 'true' && steps.diff.outputs.has_changes == 'true' run: | if jq -e '.fixed | length > 0' "$RUNNER_TEMP/remediation/fix-result.json" >/dev/null; then echo "publish=true" >> "$GITHUB_OUTPUT" @@ -447,14 +595,15 @@ jobs: ${{ runner.temp }}/remediation/diff-validation.json ${{ runner.temp }}/remediation/changed-files.txt ${{ runner.temp }}/remediation/changes.patch - ${{ runner.temp }}/remediation/socket-fix-apply.json ${{ runner.temp }}/remediation/post-fix-plan.json if-no-files-found: ignore # Publishes the validated patch. Never runs repository code, package managers, or the agent. pr: - needs: [triage, fix] - if: needs.fix.outputs.publish == 'true' + needs: [triage, fix, verify] + if: >- + !cancelled() && needs.verify.result == 'success' && + needs.verify.outputs.publish == 'true' runs-on: ubuntu-latest env: UPDATE_SHA: ${{ needs.fix.outputs.update_sha }} diff --git a/.github/workflows/vuln-remediation/fix-prompt.md b/.github/workflows/vuln-remediation/fix-prompt.md index 49a23a0..002d861 100644 --- a/.github/workflows/vuln-remediation/fix-prompt.md +++ b/.github/workflows/vuln-remediation/fix-prompt.md @@ -37,7 +37,7 @@ go mod tidy ``` If no target version is available, do not guess; record the fix as reverted with reason `No Socket target version available for fallback`. -Private `github.com/kernel/*` modules cannot be downloaded. If a fix needs one, revert it and record the download error as the reason. +Private `github.com/kernel/*` modules that are not already in the Go module cache cannot be downloaded. If a fix needs one, revert it and record the download error as the reason. ### npm (`package.json` / `package-lock.json` / `pnpm-lock.yaml`) diff --git a/scripts/test_vuln_remediation_workflow.py b/scripts/test_vuln_remediation_workflow.py index adf3b66..c45454b 100644 --- a/scripts/test_vuln_remediation_workflow.py +++ b/scripts/test_vuln_remediation_workflow.py @@ -12,9 +12,16 @@ import yaml ROOT = Path(__file__).resolve().parents[1] -WORKFLOW = yaml.safe_load((ROOT / ".github/workflows/vuln-remediation.yml").read_text()) -JOBS = WORKFLOW["jobs"] +WORKFLOW_PATH = ROOT / ".github/workflows/vuln-remediation.yml" TOKEN_REF = re.compile(r"GH_TOKEN|GITHUB_TOKEN|steps\.[\w-]+\.outputs\.token|github\.token") +MODEL_KEYS = {"ANTHROPIC_API_KEY", "OPENAI_API_KEY"} + + +def load_jobs(text: str | None = None) -> dict: + return yaml.safe_load(text if text is not None else WORKFLOW_PATH.read_text())["jobs"] + + +JOBS = load_jobs() def step(job: str, name: str) -> dict: @@ -28,11 +35,19 @@ def index(job: str, name: str) -> int: return JOBS[job]["steps"].index(step(job, name)) +def secrets_of(job: dict) -> set[str]: + return set(re.findall(r"secrets\.(\w+)", yaml.safe_dump(job))) + + +def agent_jobs(jobs: dict) -> list[str]: + return [name for name, job in jobs.items() if re.search(r"(?m)^\s*pi\b", "\n".join(s.get("run", "") for s in job["steps"]))] + + def git(cwd: Path, *args: str) -> str: return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout -def init_repo(path: Path) -> None: +def init_repo(path: Path) -> str: path.mkdir() git(path, "init", "-q", "-b", "main") git(path, "config", "user.email", "test@example.com") @@ -42,10 +57,12 @@ def init_repo(path: Path) -> None: (path / "index.js").write_text("console.log(1)\n") git(path, "add", "-A") git(path, "commit", "-q", "-m", "base") + return git(path, "rev-parse", "HEAD").strip() def run_step(entry: dict, cwd: Path, env: dict[str, str]) -> subprocess.CompletedProcess: - return subprocess.run(["bash", "-e", "-c", entry["run"]], cwd=cwd, env={**os.environ, **env}, capture_output=True, text=True) + script = entry["run"].replace("${{ inputs.security-workflows-ref }}", "main") + return subprocess.run(["bash", "-e", "-c", script], cwd=cwd, env={**os.environ, **env}, capture_output=True, text=True) class WorkflowCredentialBoundary(unittest.TestCase): @@ -56,60 +73,64 @@ def test_actions_are_pinned_and_checkouts_do_not_persist_credentials(self): if not uses: continue self.assertRegex(uses, r"^[\w/-]+@[0-9a-f]{40}$", f"{job_name}: {uses}") - if job_name in ("fix", "pr") and uses.startswith("actions/checkout@"): + if job_name in ("fix", "agent", "verify", "pr") and uses.startswith("actions/checkout@"): self.assertIs(entry["with"]["persist-credentials"], False) self.assertNotIn("token", entry["with"]) def test_default_token_is_read_only(self): - self.assertEqual(WORKFLOW["permissions"], {"contents": "read"}) - - def test_fix_job_only_has_a_revoked_read_token(self): - tokens = [entry for entry in JOBS["fix"]["steps"] if "create-github-app-token" in entry.get("uses", "")] - self.assertEqual(len(tokens), 1) - read_token = tokens[0]["with"] - self.assertEqual(read_token["permission-contents"], "read") - self.assertFalse([key for key, value in read_token.items() if key.startswith("permission-") and value != "read"]) - self.assertNotIn("owner", read_token) - self.assertNotIn("repositories", read_token) - - revoke = step("fix", "Revoke private module read token") - self.assertEqual(revoke["run"].strip(), "gh api --method DELETE /installation/token") - self.assertEqual(revoke["env"]["GH_TOKEN"], "${{ steps.read-token.outputs.token }}") - self.assertIn("always()", revoke["if"]) - - token_users = [ - entry["name"] - for entry in JOBS["fix"]["steps"] - if "steps.read-token.outputs.token" in yaml.safe_dump(entry) - ] - self.assertEqual(token_users, ["Install JS dependencies", "Apply Socket fixes", "Revoke private module read token"]) - for name in token_users: - self.assertLess(index("fix", name), index("fix", "Install Pi")) - for entry in JOBS["fix"]["steps"]: - self.assertNotIn("git config --global", entry.get("run", "")) - - def test_agent_step_has_only_model_credentials(self): - agent = step("fix", "Run fallback agent") - env = agent["env"] - secrets = sorted(key for key, value in env.items() if "secrets." in str(value)) - self.assertEqual(secrets, ["ANTHROPIC_API_KEY", "OPENAI_API_KEY"]) - self.assertEqual(env["PI_PREFERRED_MODEL"], "${{ vars.PI_PREFERRED_MODEL }}") + self.assertEqual(yaml.safe_load(WORKFLOW_PATH.read_text())["permissions"], {"contents": "read"}) + for job in JOBS.values(): + self.assertNotIn("permissions", job) + + def test_agent_job_references_only_model_keys(self): + self.assertEqual(agent_jobs(JOBS), ["agent"]) + agent = JOBS["agent"] + self.assertEqual(secrets_of(agent), MODEL_KEYS) self.assertNotRegex(yaml.safe_dump(agent), TOKEN_REF) - self.assertIn("security-workflows-ref", env["PROMPT_URL"]) + self.assertNotIn("create-github-app-token", yaml.safe_dump(agent)) + self.assertEqual(step("agent", "Run fallback agent")["env"]["PI_PREFERRED_MODEL"], "${{ vars.PI_PREFERRED_MODEL }}") + + def test_secret_check_fails_when_a_key_is_added_to_the_agent_job(self): + text = WORKFLOW_PATH.read_text() + anchor = " OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n" + self.assertEqual(text.count(anchor), 1) + for leaked in ("SOCKET_API_TOKEN", "ADMIN_APP_PRIVATE_KEY"): + jobs = load_jobs(text.replace(anchor, anchor + f" LEAK: ${{{{ secrets.{leaked} }}}}\n")) + self.assertNotEqual(secrets_of(jobs["agent"]), MODEL_KEYS) + + def test_agent_job_does_not_save_caches(self): + self.assertIs(step("agent", "Setup Go")["with"]["cache"], False) + self.assertIs(step("agent", "Setup Bun")["with"]["no-cache"], True) + + def test_agent_gets_the_prompt_and_private_modules_as_data(self): + agent = step("agent", "Run fallback agent") + self.assertIn("security-workflows-ref", agent["env"]["PROMPT_URL"]) self.assertIn("pi --print --no-session --approve", agent["run"]) self.assertIn('@"$RUNNER_TEMP/fix-prompt.md"', agent["run"]) - self.assertNotIn("CURSOR", yaml.safe_dump(JOBS)) - for name in ("Snapshot remediation diff", "Validate remediation diff", "Confirm fixes"): - self.assertLess(index("fix", "Run fallback agent"), index("fix", name)) + self.assertNotIn("CURSOR", WORKFLOW_PATH.read_text()) + self.assertEqual(step("agent", "Download private Go modules")["with"]["name"], "go-private-modules") + self.assertIn("cache/download/github.com/kernel", step("fix", "Download private Go modules")["run"]) + self.assertIn("cache/download/github.com/kernel", step("agent", "Restore private Go modules")["run"]) + + def test_credentialed_jobs_never_run_the_agent(self): + self.assertEqual(secrets_of(JOBS["fix"]), {"ADMIN_APP_ID", "ADMIN_APP_PRIVATE_KEY", "SOCKET_API_TOKEN"}) + self.assertEqual(secrets_of(JOBS["verify"]), {"SOCKET_API_TOKEN"}) + self.assertEqual(secrets_of(JOBS["pr"]), {"ADMIN_APP_ID", "ADMIN_APP_PRIVATE_KEY"}) + read_token = step("fix", "Generate private module read token")["with"] + self.assertEqual({k: v for k, v in read_token.items() if k.startswith("permission-")}, {"permission-contents": "read"}) + self.assertNotIn("owner", read_token) + self.assertNotIn("repositories", read_token) + self.assertEqual(JOBS["agent"]["needs"], "fix") + self.assertEqual(JOBS["verify"]["needs"], ["fix", "agent"]) + for name in ("fix", "verify", "pr"): + self.assertNotIn("pi-coding-agent", yaml.safe_dump(JOBS[name])) def test_publish_job_never_runs_repository_code(self): pr = JOBS["pr"] - self.assertEqual(pr["needs"], ["triage", "fix"]) - self.assertEqual(pr["if"], "needs.fix.outputs.publish == 'true'") + self.assertEqual(pr["needs"], ["triage", "fix", "verify"]) + self.assertIn("needs.verify.outputs.publish == 'true'", pr["if"]) runs = "\n".join(entry.get("run", "") for entry in pr["steps"]) self.assertNotRegex(runs, r"(?m)^\s*(go|bun|npm|npx|node|pi|make|socket|uv|pip)\b") - self.assertNotIn("secrets.ANTHROPIC_API_KEY", yaml.safe_dump(pr)) - self.assertNotIn("secrets.SOCKET_API_TOKEN", yaml.safe_dump(pr)) for entry in pr["steps"]: self.assertNotIn("continue-on-error", entry) mint = index("pr", "Generate app token") @@ -119,13 +140,12 @@ def test_publish_job_never_runs_repository_code(self): self.assertEqual(mint, len(pr["steps"]) - 2) -class SnapshotStep(unittest.TestCase): +class SnapshotSteps(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.root = Path(self.tmp.name) self.repo = self.root / "repo" - init_repo(self.repo) - self.base = git(self.repo, "rev-parse", "HEAD").strip() + self.base = init_repo(self.repo) self.work = self.root / "remediation" self.work.mkdir() self.output = self.root / "output" @@ -134,34 +154,45 @@ def setUp(self): def tearDown(self): self.tmp.cleanup() - def snapshot(self) -> subprocess.CompletedProcess: - return run_step(step("fix", "Snapshot remediation diff"), self.repo, self.env) + def run_snapshot(self, job: str) -> subprocess.CompletedProcess: + name = "Snapshot Socket changes" if job == "fix" else "Snapshot agent changes" + result = run_step(step(job, name), self.repo, self.env) + self.assertEqual(result.returncode, 0, result.stderr) + return result + + def test_socket_changes_skip_the_agent(self): + (self.repo / "package-lock.json").write_text('{"lockfileVersion": 3, "left-pad": "1.3.0"}\n') + (self.repo / "socket-fix-apply.json").write_text("{}\n") + self.run_snapshot("fix") + self.assertEqual(self.output.read_text(), "has_changes=true\n") + self.assertEqual((self.work / "changed-files.txt").read_text().split(), ["package-lock.json"]) + + def test_no_socket_changes_run_the_agent(self): + self.run_snapshot("fix") + self.assertEqual(self.output.read_text(), "run_agent=true\n") - def test_captures_committed_and_uncommitted_tracked_changes_on_a_clean_tree(self): + def test_agent_snapshot_captures_committed_and_uncommitted_tracked_changes(self): (self.repo / "package.json").write_text('{"dependencies": {"left-pad": "1.3.0"}}\n') git(self.repo, "commit", "-qam", "agent commit") (self.repo / "package-lock.json").write_text('{"lockfileVersion": 3, "left-pad": "1.3.0"}\n') (self.repo / "bun.lock").write_text("untracked\n") (self.repo / "remediation-context.json").write_text("{}\n") - - result = self.snapshot() - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("has_changes=true", self.output.read_text()) + self.run_snapshot("agent") + self.assertEqual(self.output.read_text(), "has_changes=true\n") self.assertEqual((self.work / "changed-files.txt").read_text().split(), ["package-lock.json", "package.json"]) - self.assertEqual(git(self.repo, "rev-parse", "HEAD").strip(), self.base) - self.assertFalse((self.repo / "bun.lock").exists()) - self.assertFalse((self.repo / "remediation-context.json").exists()) - self.assertEqual(git(self.repo, "diff", "--cached", "--name-only").split(), ["package-lock.json", "package.json"]) + patch = (self.work / "changes.patch").read_text() + self.assertNotIn("bun.lock", patch) + self.assertNotIn("remediation-context.json", patch) - def test_no_changes(self): + def test_agent_snapshot_without_changes(self): (self.repo / "fix-result.json").write_text("{}\n") - result = self.snapshot() - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("has_changes=false", self.output.read_text()) - self.assertIn("No dependency changes", (self.work / "fix-result.json").read_text()) + self.run_snapshot("agent") + self.assertFalse(self.output.exists()) -class PublishStep(unittest.TestCase): +class Gates(unittest.TestCase): + """Runs the verify job's and the publish job's real gate scripts against a fresh checkout.""" + def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.root = Path(self.tmp.name) @@ -180,59 +211,63 @@ def setUp(self): def tearDown(self): self.tmp.cleanup() - def stage(self, files: dict[str, str], executable: bool = False) -> None: + def stage(self, files: dict[str, str] | None = None, executable: bool = False, symlink: str | None = None) -> None: source = self.root / "source" git(self.root, "clone", "-q", str(self.repo), str(source)) - for path, content in files.items(): + for path, content in (files or {}).items(): (source / path).write_text(content) if executable: (source / path).chmod(0o755) + if symlink: + (source / symlink).unlink() + (source / symlink).symlink_to("index.js") git(source, "add", "-A") (self.work / "changes.patch").write_text(git(source, "diff", "--cached", "--binary", "--no-renames")) (self.work / "changed-files.txt").write_text(git(source, "diff", "--cached", "--name-only", "--no-renames")) - def publish(self) -> subprocess.CompletedProcess: - script = step("pr", "Apply validated changes")["run"].replace("${{ inputs.security-workflows-ref }}", "main") - return run_step({"run": script}, self.repo, self.env) - - def test_commits_validated_dependency_changes(self): + def run_gate(self, gate: str) -> subprocess.CompletedProcess: + fresh = self.root / f"fresh-{gate}" + git(self.root, "clone", "-q", str(self.repo), str(fresh)) + git(fresh, "checkout", "-q", "--detach") + git(fresh, "branch", "-q", "-D", "main") + entry = step("verify", "Validate remediation diff") if gate == "verify" else step("pr", "Apply validated changes") + self.fresh = fresh + return run_step(entry, fresh, self.env) + + def assert_both_reject(self, message: str) -> None: + for gate in ("verify", "pr"): + result = self.run_gate(gate) + self.assertNotEqual(result.returncode, 0, gate) + self.assertIn(message, result.stdout + result.stderr, gate) + + def test_publishes_validated_dependency_changes(self): self.stage({"package-lock.json": '{"lockfileVersion": 3, "left-pad": "1.3.0"}\n'}) - result = self.publish() + self.assertEqual(self.run_gate("verify").returncode, 0) + result = self.run_gate("pr") self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(git(self.repo, "branch", "--show-current").strip(), "security/vuln-remediation") - self.assertEqual(git(self.repo, "show", "--name-only", "--format=", "HEAD").split(), ["package-lock.json"]) + self.assertEqual(git(self.fresh, "branch", "--show-current").strip(), "security/vuln-remediation") + self.assertEqual(git(self.fresh, "show", "--name-only", "--format=", "HEAD").split(), ["package-lock.json"]) - def test_rejects_source_changes(self): + def test_rejects_files_outside_the_allowlist(self): self.stage({"index.js": "console.log(2)\n"}) - result = self.publish() - self.assertNotEqual(result.returncode, 0) - self.assertIn("only dependency manifests and lockfiles may change", result.stderr) + self.assert_both_reject("only dependency manifests and lockfiles may change") def test_rejects_executable_files(self): self.stage({"package-lock.json": "#!/bin/sh\n"}, executable=True) self.assertIn("100755", (self.work / "changes.patch").read_text()) - result = self.publish() - self.assertNotEqual(result.returncode, 0) - self.assertIn("executable files are not allowed", result.stderr) + self.assert_both_reject("executable files are not allowed") - def test_rejects_symlinks(self): - self.stage({}) - source = self.root / "source" - (source / "package-lock.json").unlink() - (source / "package-lock.json").symlink_to("index.js") - git(source, "add", "-A") - (self.work / "changes.patch").write_text(git(source, "diff", "--cached", "--binary", "--no-renames")) - (self.work / "changed-files.txt").write_text(git(source, "diff", "--cached", "--name-only", "--no-renames")) - result = self.publish() + def test_publish_mode_check_rejects_symlinks(self): + self.stage(symlink="package-lock.json") + self.assertEqual(self.run_gate("verify").returncode, 0) + result = self.run_gate("pr") self.assertNotEqual(result.returncode, 0) self.assertIn("regular, non-executable files", result.stdout + result.stderr) - def test_rejects_patch_that_differs_from_validated_files(self): + def test_rejects_patch_that_differs_from_its_file_list(self): self.stage({"package-lock.json": '{"lockfileVersion": 3}\n'}) (self.work / "changed-files.txt").write_text("package.json\n") - result = self.publish() - self.assertNotEqual(result.returncode, 0) - self.assertIn("does not match", result.stdout + result.stderr) + self.assert_both_reject("does not match") if __name__ == "__main__": From 08d907728a9984295cd53b7f821506e8be8ab2a5 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:01:36 +0000 Subject: [PATCH 3/3] Add job timeouts to the remediation jobs --- .github/workflows/vuln-remediation.yml | 4 ++++ scripts/test_vuln_remediation_workflow.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/vuln-remediation.yml b/.github/workflows/vuln-remediation.yml index 7134ea6..bfca9dd 100644 --- a/.github/workflows/vuln-remediation.yml +++ b/.github/workflows/vuln-remediation.yml @@ -195,6 +195,7 @@ jobs: fix: needs: triage runs-on: ubuntu-latest + timeout-minutes: 30 outputs: base_sha: ${{ steps.branch.outputs.base_sha }} update_sha: ${{ steps.branch.outputs.update_sha }} @@ -370,6 +371,7 @@ jobs: needs: fix if: needs.fix.outputs.run_agent == 'true' runs-on: ubuntu-latest + timeout-minutes: 40 outputs: has_changes: ${{ steps.snapshot.outputs.has_changes }} steps: @@ -498,6 +500,7 @@ jobs: (needs.fix.outputs.has_changes == 'true' || (needs.agent.result == 'success' && needs.agent.outputs.has_changes == 'true')) runs-on: ubuntu-latest + timeout-minutes: 20 outputs: publish: ${{ steps.result.outputs.publish }} steps: @@ -605,6 +608,7 @@ jobs: !cancelled() && needs.verify.result == 'success' && needs.verify.outputs.publish == 'true' runs-on: ubuntu-latest + timeout-minutes: 10 env: UPDATE_SHA: ${{ needs.fix.outputs.update_sha }} steps: diff --git a/scripts/test_vuln_remediation_workflow.py b/scripts/test_vuln_remediation_workflow.py index c45454b..386119f 100644 --- a/scripts/test_vuln_remediation_workflow.py +++ b/scripts/test_vuln_remediation_workflow.py @@ -98,6 +98,11 @@ def test_secret_check_fails_when_a_key_is_added_to_the_agent_job(self): jobs = load_jobs(text.replace(anchor, anchor + f" LEAK: ${{{{ secrets.{leaked} }}}}\n")) self.assertNotEqual(secrets_of(jobs["agent"]), MODEL_KEYS) + def test_remediation_jobs_have_timeouts(self): + for name in ("fix", "agent", "verify", "pr"): + self.assertIsInstance(JOBS[name].get("timeout-minutes"), int, name) + self.assertGreater(JOBS["agent"]["timeout-minutes"], step("agent", "Run fallback agent")["timeout-minutes"]) + def test_agent_job_does_not_save_caches(self): self.assertIs(step("agent", "Setup Go")["with"]["cache"], False) self.assertIs(step("agent", "Setup Bun")["with"]["no-cache"], True)