diff --git a/.github/actions/llm-credentials/action.yml b/.github/actions/llm-credentials/action.yml new file mode 100644 index 0000000..498bab4 --- /dev/null +++ b/.github/actions/llm-credentials/action.yml @@ -0,0 +1,58 @@ +name: LLM credentials +description: >- + Short-lived Anthropic or OpenAI credentials for pi through workload identity federation, so + the job needs no stored API key. The job needs `permissions: id-token: write`. Credentials are + refreshed every minute until the job ends; later steps run pi with no key in their env. + +inputs: + model: + description: Pi model as provider/id. Its provider, anthropic or openai, picks the credentials. + required: true + # The federation IDs are not secrets: they name the provider-side rules that this org's GitHub + # OIDC tokens match. Pass the kernel org variables; action metadata can't read `vars` itself. + # Only the model's provider's IDs are needed. + anthropic-federation-rule-id: + description: Anthropic federation rule ID (fdrl_...) + anthropic-organization-id: + description: Anthropic organization ID + anthropic-service-account-id: + description: Anthropic service account ID (svac_...) + openai-identity-provider-id: + description: OpenAI workload identity provider ID (idp_...) + openai-service-account-id: + description: OpenAI service account ID + openai-audience: + description: Audience the OpenAI workload identity provider requires + +runs: + using: composite + steps: + - name: Start credential refresh + shell: bash + env: + MODEL: ${{ inputs.model }} + ANTHROPIC_FEDERATION_RULE_ID: ${{ inputs.anthropic-federation-rule-id }} + ANTHROPIC_ORGANIZATION_ID: ${{ inputs.anthropic-organization-id }} + ANTHROPIC_SERVICE_ACCOUNT_ID: ${{ inputs.anthropic-service-account-id }} + ANTHROPIC_IDENTITY_TOKEN_FILE: ${{ runner.temp }}/anthropic-identity-token + OPENAI_IDENTITY_PROVIDER_ID: ${{ inputs.openai-identity-provider-id }} + OPENAI_SERVICE_ACCOUNT_ID: ${{ inputs.openai-service-account-id }} + OPENAI_WIF_AUDIENCE: ${{ inputs.openai-audience }} + run: | + set -euo pipefail + : "${MODEL:?Set the model input to a pi model as provider/id}" + provider=${MODEL%%/*} + if [ "$provider" = anthropic ]; then + : "${ANTHROPIC_FEDERATION_RULE_ID:?Set the anthropic-federation-rule-id input}" + : "${ANTHROPIC_ORGANIZATION_ID:?Set the anthropic-organization-id input}" + : "${ANTHROPIC_SERVICE_ACCOUNT_ID:?Set the anthropic-service-account-id input}" + fi + bash "$GITHUB_ACTION_PATH/llm_credentials.sh" start "$provider" + # The Anthropic SDK in pi does the exchange itself, so later steps need the federation + # settings. OpenAI's token is already in pi's auth.json. + if [ "$provider" = anthropic ]; then + for name in ANTHROPIC_FEDERATION_RULE_ID ANTHROPIC_ORGANIZATION_ID \ + ANTHROPIC_SERVICE_ACCOUNT_ID ANTHROPIC_IDENTITY_TOKEN_FILE; do + echo "$name=${!name}" >> "$GITHUB_ENV" + done + fi diff --git a/.github/actions/llm-credentials/llm_credentials.sh b/.github/actions/llm-credentials/llm_credentials.sh new file mode 100644 index 0000000..e102cc6 --- /dev/null +++ b/.github/actions/llm-credentials/llm_credentials.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Short-lived model credentials for pi in GitHub Actions, through workload identity federation +# instead of stored API keys. The job needs `permissions: id-token: write`. +# +# llm_credentials.sh start fetch once, then refresh every minute until +# the job ends +# llm_credentials.sh refresh fetch once +# +# GitHub's OIDC tokens last about five minutes, and each provider's credential expires no later +# than the token it was exchanged for, so a run longer than that needs the background refresh. +# +# anthropic: writes a fresh GitHub token to $ANTHROPIC_IDENTITY_TOKEN_FILE. The Anthropic SDK in +# pi exchanges it using ANTHROPIC_FEDERATION_RULE_ID, ANTHROPIC_ORGANIZATION_ID and +# ANTHROPIC_SERVICE_ACCOUNT_ID, and rereads the file before each refresh. Each token can be +# exchanged only once. +# openai: exchanges a GitHub token for an OpenAI access token using OPENAI_IDENTITY_PROVIDER_ID, +# OPENAI_SERVICE_ACCOUNT_ID and OPENAI_WIF_AUDIENCE, and writes it to pi's auth.json, which pi +# rereads when the file changes. +set -euo pipefail +umask 077 + +# The functions below return on failure explicitly: `set -e` does not apply inside a function +# called from `refresh || ...`, and a failed fetch must never overwrite a working credential. + +github_token() { + local audience + audience=$(jq -rn --arg a "$1" '$a | @uri') || return 1 + curl -fsS --retry 3 -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=$audience" | jq -re .value +} + +# write : replace the file in one step so a reader never sees a partial write. +write() { + printf '%s' "$2" > "$1.tmp" && mv "$1.tmp" "$1" +} + +refresh_anthropic() { + local token + token=$(github_token https://api.anthropic.com) || return 1 + write "${ANTHROPIC_IDENTITY_TOKEN_FILE:?}" "$token" +} + +refresh_openai() { + local token access dir + token=$(github_token "${OPENAI_WIF_AUDIENCE:?}") || return 1 + access=$(jq -n --arg token "$token" --arg provider "${OPENAI_IDENTITY_PROVIDER_ID:?}" \ + --arg account "${OPENAI_SERVICE_ACCOUNT_ID:?}" \ + '{grant_type: "urn:ietf:params:oauth:grant-type:token-exchange", + subject_token_type: "urn:ietf:params:oauth:token-type:jwt", subject_token: $token, + identity_provider_id: $provider, service_account_id: $account}' | + curl -fsS --retry 3 https://auth.openai.com/oauth/token -H 'Content-Type: application/json' -d @- | + jq -re .access_token) || return 1 + dir=${PI_CODING_AGENT_DIR:-$HOME/.pi/agent} + mkdir -p "$dir" || return 1 + write "$dir/auth.json" "$(jq -n --arg key "$access" '{openai: {type: "api_key", key: $key}}')" +} + +refresh() { + case "$1" in + anthropic | openai) "refresh_$1" ;; + *) echo "no federated credentials for provider '$1'" >&2; return 1 ;; + esac +} + +usage="usage: llm_credentials.sh start|refresh " +case "${1:-}" in + start) + refresh "${2:?$usage}" + nohup bash "${BASH_SOURCE[0]}" loop "$2" >"${RUNNER_TEMP:-/tmp}/llm-credentials.log" 2>&1 & + ;; + refresh) refresh "${2:?$usage}" ;; + loop) + while sleep 60; do + refresh "$2" || echo "$(date -u +%FT%TZ) $2 credential refresh failed" + done + ;; + *) echo "$usage" >&2; exit 1 ;; +esac diff --git a/.github/workflows/pr-risk-tests.yml b/.github/workflows/pr-risk-tests.yml index 78aef4e..18897df 100644 --- a/.github/workflows/pr-risk-tests.yml +++ b/.github/workflows/pr-risk-tests.yml @@ -8,6 +8,8 @@ on: - .github/workflows/pr-risk/** - scripts/pr_risk.py - scripts/test_pr_risk.py + - .github/actions/llm-credentials/** + - scripts/test_llm_credentials.py push: branches: [main] paths: @@ -16,6 +18,8 @@ on: - .github/workflows/pr-risk/** - scripts/pr_risk.py - scripts/test_pr_risk.py + - .github/actions/llm-credentials/** + - scripts/test_llm_credentials.py permissions: contents: read @@ -29,3 +33,4 @@ jobs: python3 -m venv /tmp/pr-risk-tests /tmp/pr-risk-tests/bin/pip install PyYAML==6.0.3 /tmp/pr-risk-tests/bin/python -m unittest discover -s scripts -p test_pr_risk.py + /tmp/pr-risk-tests/bin/python -m unittest discover -s scripts -p test_llm_credentials.py diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index 91ec2f3..9605be4 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -13,6 +13,9 @@ name: PR Risk # Credentials are split by step. The model runs with no tools and no GitHub token, reading # only the PR data gathered with the caller's read-only token. The kernel-internal token is # minted after the model has finished, scoped to this repo and to pull-request writes. +# The model provider's credentials come from workload identity federation: the job's GitHub +# OIDC token is exchanged for short-lived provider credentials, so callers pass no API keys +# but must grant `id-token: write`. on: workflow_call: @@ -39,7 +42,8 @@ on: required: true ADMIN_APP_PRIVATE_KEY: required: true - # Pass the key for the model's provider. Both are optional so either provider works. + # Ignored. Declared so callers that still pass them keep working; remove them from the + # callers, then from here. ANTHROPIC_API_KEY: required: false OPENAI_API_KEY: @@ -53,6 +57,8 @@ jobs: permissions: contents: read pull-requests: read + # For the llm-credentials action to exchange the job's OIDC token for model credentials. + id-token: write concurrency: group: pr-risk-${{ github.repository }}-${{ github.event.pull_request.number }} cancel-in-progress: true @@ -91,11 +97,22 @@ jobs: npm install -g --ignore-scripts @earendil-works/pi-coding-agent pi --version + # `uses` can't take an expression, so this follows main rather than security-workflows-ref. + - name: Model credentials + if: steps.gather.outputs.eligible == 'true' + uses: kernel/security-workflows/.github/actions/llm-credentials@main + with: + model: ${{ inputs.model }} + anthropic-federation-rule-id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }} + anthropic-organization-id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }} + anthropic-service-account-id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }} + openai-identity-provider-id: ${{ vars.OPENAI_IDENTITY_PROVIDER_ID }} + openai-service-account-id: ${{ vars.OPENAI_SERVICE_ACCOUNT_ID }} + openai-audience: ${{ vars.OPENAI_WIF_AUDIENCE }} + - name: Classify if: steps.gather.outputs.eligible == 'true' env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} MODEL: ${{ inputs.model }} PI_SKIP_VERSION_CHECK: '1' PI_TELEMETRY: '0' diff --git a/README.md b/README.md index 1e5831e..f045bd2 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ on: permissions: contents: read pull-requests: read + id-token: write jobs: classify: uses: kernel/security-workflows/.github/workflows/pr-risk.yml@main @@ -95,12 +96,45 @@ jobs: secrets: ADMIN_APP_ID: ${{ secrets.ADMIN_APP_ID }} ADMIN_APP_PRIVATE_KEY: ${{ secrets.ADMIN_APP_PRIVATE_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ``` The model defaults to the `PI_PREFERRED_MODEL` variable and can be overridden with the `model` -input. Pass the API key for that model's provider; both keys are optional secrets. +input. It must be an `anthropic/` or `openai/` model. There is no API key to pass: the +[`llm-credentials`](#llm-credentials-action) action exchanges the job's GitHub OIDC token for +short-lived provider credentials, which is why the caller grants `id-token: write`. + +## Actions + +### `llm-credentials` action + +Gives pi short-lived Anthropic or OpenAI credentials through workload identity federation, so a +job needs no stored API key. It refreshes them every minute until the job ends, so long agent runs +keep working past the five-minute life of a GitHub OIDC token. + +```yaml +permissions: + contents: read + id-token: write +steps: + - uses: kernel/security-workflows/.github/actions/llm-credentials@main + with: + model: ${{ vars.PI_PREFERRED_MODEL }} + anthropic-federation-rule-id: ${{ vars.ANTHROPIC_FEDERATION_RULE_ID }} + anthropic-organization-id: ${{ vars.ANTHROPIC_ORGANIZATION_ID }} + anthropic-service-account-id: ${{ vars.ANTHROPIC_SERVICE_ACCOUNT_ID }} + openai-identity-provider-id: ${{ vars.OPENAI_IDENTITY_PROVIDER_ID }} + openai-service-account-id: ${{ vars.OPENAI_SERVICE_ACCOUNT_ID }} + openai-audience: ${{ vars.OPENAI_WIF_AUDIENCE }} + - env: + PI_PREFERRED_MODEL: ${{ vars.PI_PREFERRED_MODEL }} + run: pi --print --model "$PI_PREFERRED_MODEL" ... # no ANTHROPIC_API_KEY or OPENAI_API_KEY +``` + +The federation IDs come from the org variables listed under [Required variables](#required-variables). +An action can't read `vars` itself, so the caller passes them; only the model's provider's IDs are +used. +Don't also set `ANTHROPIC_API_KEY` or `OPENAI_API_KEY` in the pi step: a key takes precedence over +federation. ## Enrollment @@ -135,7 +169,7 @@ projectIgnorePaths: Consumer repos need these secrets (set at org or repo level): -- `ANTHROPIC_API_KEY` — for the `semgrep.yml` triage agent (Pi) and the `pr-risk.yml` classifier (Claude Code) +- `ANTHROPIC_API_KEY` — for the `semgrep.yml` triage agent (Pi) - `OPENAI_API_KEY` — for the `semgrep.yml` triage agent (Pi), when `PI_PREFERRED_MODEL` is an OpenAI model - `CURSOR_API_KEY` — for the other fix/remediation agents (Cursor) - `ADMIN_APP_ID` + `ADMIN_APP_PRIVATE_KEY` — GitHub App for write access @@ -145,6 +179,8 @@ Consumer repos need these secrets (set at org or repo level): Consumer repos need these variables (set at org or repo level): -- `PI_PREFERRED_MODEL` — model for the `semgrep.yml` triage agent (Pi) -- `CLAUDE_CODE_PREFERRED_MODEL` — model for the `pr-risk.yml` classifier (Claude Code) +- `PI_PREFERRED_MODEL` — model for the `semgrep.yml` triage agent and the `pr-risk.yml` classifier (Pi) +- `ANTHROPIC_FEDERATION_RULE_ID`, `ANTHROPIC_ORGANIZATION_ID`, `ANTHROPIC_SERVICE_ACCOUNT_ID`, + `OPENAI_IDENTITY_PROVIDER_ID`, `OPENAI_SERVICE_ACCOUNT_ID`, `OPENAI_WIF_AUDIENCE` — workload + identity federation for the `llm-credentials` action, set at the org level - `CURSOR_PREFERRED_MODEL` — model for the other Cursor agent invocations diff --git a/scripts/test_llm_credentials.py b/scripts/test_llm_credentials.py new file mode 100644 index 0000000..4701612 --- /dev/null +++ b/scripts/test_llm_credentials.py @@ -0,0 +1,116 @@ +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +import yaml + +ACTION = Path(__file__).resolve().parents[1] / ".github/actions/llm-credentials" +SCRIPT = ACTION / "llm_credentials.sh" + +# Stands in for curl: GitHub's OIDC endpoint returns a token naming its audience, and OpenAI's +# token endpoint returns an access token naming the GitHub token it was given. +FAKE_CURL = r'''#!/usr/bin/env python3 +import json, os, sys, urllib.parse +if os.environ.get("FAKE_CURL_FAIL"): + sys.exit(22) +url = next(a for a in sys.argv[1:] if a.startswith("http")) +if url.startswith("https://auth.openai.com/"): + body = json.loads(sys.stdin.read()) + print(json.dumps({"access_token": "access-for-" + body["subject_token"]})) +else: + audience = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)["audience"][0] + print(json.dumps({"value": "jwt-for-" + audience})) +''' + + +class LlmCredentialsTest(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + (self.root / "bin").mkdir() + (self.root / "bin/curl").write_text(FAKE_CURL) + (self.root / "bin/curl").chmod(0o755) + self.env = { + "PATH": f"{self.root / 'bin'}:{os.environ['PATH']}", + "ACTIONS_ID_TOKEN_REQUEST_URL": "https://github.invalid/token?api-version=2.0", + "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "request-token", + "ANTHROPIC_IDENTITY_TOKEN_FILE": str(self.root / "anthropic-token"), + "OPENAI_IDENTITY_PROVIDER_ID": "idp_test", + "OPENAI_SERVICE_ACCOUNT_ID": "user-test", + "OPENAI_WIF_AUDIENCE": "https://api.openai.com/v1", + "PI_CODING_AGENT_DIR": str(self.root / "pi"), + } + + def run_script(self, *args, **env): + return subprocess.run(["bash", str(SCRIPT), *args], env={**self.env, **env}, + capture_output=True, text=True) + + def test_anthropic_writes_identity_token(self): + self.assertEqual(self.run_script("refresh", "anthropic").returncode, 0) + self.assertEqual((self.root / "anthropic-token").read_text(), "jwt-for-https://api.anthropic.com") + + def test_openai_writes_access_token_to_pi_auth(self): + self.assertEqual(self.run_script("refresh", "openai").returncode, 0) + auth = self.root / "pi/auth.json" + self.assertEqual(json.loads(auth.read_text()), + {"openai": {"type": "api_key", "key": "access-for-jwt-for-https://api.openai.com/v1"}}) + self.assertEqual(auth.stat().st_mode & 0o777, 0o600) + + def test_failed_refresh_keeps_working_credentials(self): + self.run_script("refresh", "openai") + self.run_script("refresh", "anthropic") + before = {p: p.read_text() for p in (self.root / "pi/auth.json", self.root / "anthropic-token")} + for provider in ("openai", "anthropic"): + self.assertNotEqual(self.run_script("refresh", provider, FAKE_CURL_FAIL="1").returncode, 0) + self.assertEqual({p: p.read_text() for p in before}, before) + + def run_action(self, model, **overrides): + step = yaml.safe_load((ACTION / "action.yml").read_text())["runs"]["steps"][0] + fake = self.root / "fake-action" + fake.mkdir(exist_ok=True) + (fake / "llm_credentials.sh").write_text('echo "$@" > "$FAKE_CREDENTIALS_LOG"\n') + env = {name: "id-" + name for name in step["env"]} + env.update(PATH=os.environ["PATH"], MODEL=model, GITHUB_ACTION_PATH=str(fake), + GITHUB_ENV=str(self.root / "github-env"), FAKE_CREDENTIALS_LOG=str(self.root / "calls")) + env.update(overrides) + result = subprocess.run(["bash", "-c", step["run"]], env=env, capture_output=True, text=True) + read = lambda name: (self.root / name).read_text() if (self.root / name).exists() else "" + return result, read("calls"), read("github-env") + + def test_action_starts_the_model_provider(self): + _, calls, github_env = self.run_action("anthropic/claude-x") + self.assertEqual(calls, "start anthropic\n") + # Later steps run pi, whose Anthropic SDK needs the federation settings. + self.assertEqual(github_env.splitlines(), [ + f"{name}=id-{name}" for name in ("ANTHROPIC_FEDERATION_RULE_ID", "ANTHROPIC_ORGANIZATION_ID", + "ANTHROPIC_SERVICE_ACCOUNT_ID", "ANTHROPIC_IDENTITY_TOKEN_FILE")]) + + (self.root / "github-env").unlink() + _, calls, github_env = self.run_action("openai/gpt-x") + self.assertEqual(calls, "start openai\n") + self.assertEqual(github_env, "") + + def test_action_requires_a_model(self): + result, calls, _ = self.run_action("") + self.assertNotEqual(result.returncode, 0) + self.assertIn("model input", result.stderr) + self.assertEqual(calls, "") + + def test_action_requires_anthropic_ids(self): + result, calls, _ = self.run_action("anthropic/claude-x", ANTHROPIC_FEDERATION_RULE_ID="") + self.assertNotEqual(result.returncode, 0) + self.assertIn("anthropic-federation-rule-id", result.stderr) + self.assertEqual(calls, "") + + def test_unknown_provider_fails(self): + result = self.run_script("start", "google") + self.assertNotEqual(result.returncode, 0) + self.assertIn("google", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_pr_risk.py b/scripts/test_pr_risk.py index 5147715..b5daf99 100644 --- a/scripts/test_pr_risk.py +++ b/scripts/test_pr_risk.py @@ -357,11 +357,16 @@ def test_verdict_from_events_feeds_comment(self): def test_workflow_isolates_the_model(self): workflow = yaml.safe_load(WORKFLOW.read_text()) job = workflow["jobs"]["classify"] - self.assertEqual(job["permissions"], {"contents": "read", "pull-requests": "read"}) + self.assertEqual(job["permissions"], {"contents": "read", "pull-requests": "read", "id-token": "write"}) + self.assertNotIn("secrets.", json.dumps(job["env"])) steps = {s["name"]: s for s in job["steps"]} classify = steps["Classify"] - self.assertEqual(set(classify["env"]), {"ANTHROPIC_API_KEY", "OPENAI_API_KEY", "MODEL", - "PI_SKIP_VERSION_CHECK", "PI_TELEMETRY"}) + self.assertEqual(set(classify["env"]), {"MODEL", "PI_SKIP_VERSION_CHECK", "PI_TELEMETRY"}) + credentials = steps["Model credentials"] + self.assertEqual(credentials["uses"], "kernel/security-workflows/.github/actions/llm-credentials@main") + self.assertEqual(credentials["with"]["model"], "${{ inputs.model }}") + self.assertNotIn("secrets.", json.dumps(credentials["with"])) + self.assertEqual(credentials["if"], classify["if"]) self.assertEqual(classify["env"]["MODEL"], "${{ inputs.model }}") for flag in ("--no-tools", "--no-mcp", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-context-files", "--no-approve", "--no-session"): @@ -369,6 +374,7 @@ def test_workflow_isolates_the_model(self): self.assertNotIn("--approve ", classify["run"]) self.assertNotIn("GH_TOKEN", json.dumps({k: steps[k] for k in ("Install Pi", "Classify")})) names = list(steps) + self.assertLess(names.index("Model credentials"), names.index("Classify")) self.assertLess(names.index("Classify"), names.index("Generate app token")) self.assertEqual(steps["Generate app token"]["with"]["permission-pull-requests"], "write") self.assertEqual(steps["Gather"]["env"]["GH_TOKEN"], "${{ github.token }}") @@ -395,7 +401,7 @@ def test_classify_step_runs_pi(self): (root / "bin/pi").chmod(0o755) (root / "events.jsonl").write_text(pi_events(json.dumps(VERDICT))) env = {"PATH": f"{root / 'bin'}:{os.environ['PATH']}", "WORKDIR": str(work), - "MODEL": "openai/some-model", "OPENAI_API_KEY": "sk-test", + "MODEL": "openai/some-model", "FAKE_PI_LOG": str(root / "pi.json"), "FAKE_PI_EVENTS": str(root / "events.jsonl")} subprocess.run(["bash", "-c", script], env=env, check=True) call = json.loads((root / "pi.json").read_text())