diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ad2687b254..6592a51e85 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -186,6 +186,8 @@ jobs: env: DATABASE_URL: postgres://postgres:password@localhost:5432/test REDIS_URL: redis://localhost:6379 + - name: Test Judgement CI isolation + run: node --test scripts/judgement/__tests__/*.test.mjs - name: Run release script unit tests run: pnpm test:release-scripts diff --git a/.github/workflows/judgement.yml b/.github/workflows/judgement.yml new file mode 100644 index 0000000000..0d74b2b581 --- /dev/null +++ b/.github/workflows/judgement.yml @@ -0,0 +1,152 @@ +name: Judgement + +on: + pull_request_target: + branches: [main, develop] + types: [opened, synchronize, reopened, ready_for_review, edited] + +permissions: {} + +concurrency: + group: judgement-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + pending: + name: Mark Judgement pending + runs-on: ubuntu-24.04 + permissions: + statuses: write + steps: + - name: Mark the PR check pending + env: + GH_TOKEN: ${{ github.token }} + PR_HEAD: ${{ github.event.pull_request.head.sha }} + run: | + gh api --silent "repos/$GITHUB_REPOSITORY/statuses/$PR_HEAD" \ + -f state=pending -f context=Judgement \ + -f description='Checking repository rules' \ + -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + + collect: + name: Collect Git evidence + needs: pending + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + outputs: + base: ${{ steps.export.outputs.base }} + env: + PR_HEAD: ${{ github.event.pull_request.head.sha }} + PR_BASE: ${{ github.event.pull_request.base.sha }} + steps: + # This job has no inference secret or repository write permission. + - name: Checkout trusted base + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Export snapshot objects + id: export + run: | + set -euo pipefail + [[ "$PR_HEAD" =~ ^[0-9a-f]{40}$ && "$PR_BASE" =~ ^[0-9a-f]{40}$ ]] + git fetch --no-tags origin "$PR_HEAD" + base="$(git merge-base "$PR_BASE" "$PR_HEAD")" + git cat-file -e "$base:.judgement/rules.json" + mkdir -p "$RUNNER_TEMP/judgement-evidence" + # Transfer objects only: no worktree, Git config, hooks, or full history. + { + printf '%s\n' "$base" "$PR_HEAD" + git rev-list --objects --no-object-names "$base^{tree}" "$PR_HEAD^{tree}" + } | sort -u | git pack-objects --stdout > "$RUNNER_TEMP/judgement-evidence/objects.pack" + echo "base=$base" >> "$GITHUB_OUTPUT" + - name: Upload snapshot objects + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: judgement-evidence + path: ${{ runner.temp }}/judgement-evidence/objects.pack + if-no-files-found: error + retention-days: 1 + compression-level: 0 + + judgement: + name: Judge Git evidence + needs: collect + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: {} + env: + PR_HEAD: ${{ github.event.pull_request.head.sha }} + JUDGEMENT_BASE: ${{ needs.collect.outputs.base }} + steps: + # No repository checkout in this job. The checker is installed separately. + - name: Set up Node + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '24.13.1' + - name: Install trusted checker + run: | + mkdir -p "$RUNNER_TEMP/judgement-tool" + cd "$RUNNER_TEMP/judgement-tool" + npm install --ignore-scripts --no-audit --no-fund \ + '@roo-code/judgement@0.4.0' + - name: Download snapshot objects + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: judgement-evidence + path: ${{ runner.temp }}/judgement-evidence + - name: Import snapshot objects without checkout + id: import + run: | + set -euo pipefail + [[ "$PR_HEAD" =~ ^[0-9a-f]{40}$ && "$JUDGEMENT_BASE" =~ ^[0-9a-f]{40}$ ]] + git init --quiet --template= "$RUNNER_TEMP/judgement-input" + cd "$RUNNER_TEMP/judgement-input" + git index-pack --stdin < "$RUNNER_TEMP/judgement-evidence/objects.pack" + git cat-file -e "$JUDGEMENT_BASE^{commit}" + git cat-file -e "$PR_HEAD^{commit}" + git cat-file -e "$JUDGEMENT_BASE:.judgement/rules.json" + git update-ref refs/heads/evidence "$JUDGEMENT_BASE" + git symbolic-ref HEAD refs/heads/evidence + - name: Run strict check + env: + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} + JUDGEMENT_MODEL: ${{ vars.JUDGEMENT_MODEL || 'jev-1.13.0' }} + run: | + set -euo pipefail + if [ -z "$TYPESAFE_API_KEY" ]; then + echo 'Configure the TYPESAFE_API_KEY repository secret for Judgement.' >&2 + exit 2 + fi + node "$RUNNER_TEMP/judgement-tool/node_modules/@roo-code/judgement/src/cli.js" \ + check --cwd "$RUNNER_TEMP/judgement-input" \ + --base "$JUDGEMENT_BASE" --head "$PR_HEAD" \ + --model "$JUDGEMENT_MODEL" --timeout-ms 120000 --no-cache + + report: + name: Publish Judgement result + needs: [pending, collect, judgement] + if: ${{ always() && !cancelled() && needs.pending.result == 'success' }} + runs-on: ubuntu-24.04 + permissions: + statuses: write + steps: + # This job receives only the job conclusion, never repository contents or secrets. + - name: Publish strict result + env: + GH_TOKEN: ${{ github.token }} + PR_HEAD: ${{ github.event.pull_request.head.sha }} + CHECK_OUTCOME: ${{ needs.judgement.result }} + run: | + state=failure + description='Judgement failed or did not complete; inspect the run' + if [ "$CHECK_OUTCOME" = success ]; then + state=success + description='Repository rule check completed without findings' + fi + gh api --silent "repos/$GITHUB_REPOSITORY/statuses/$PR_HEAD" \ + -f state="$state" -f context=Judgement -f description="$description" \ + -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" diff --git a/.judgement/README.md b/.judgement/README.md index dd16f300aa..b804bdf5bd 100644 --- a/.judgement/README.md +++ b/.judgement/README.md @@ -94,3 +94,59 @@ Rules 7 and 8 use a violation cutoff of 0.75. Validate the capitalization and pr rules together with sentence starts, quoted UI labels, headings, migration and security requirements, and edits near unchanged violations. Difficult examples remain in the validation fixtures even when their scores fall below the cutoff. + +## Turn feedback into examples + +Capture reads the staged version and previews a fixture without inference: + +```sh +pnpm judgement capture --rule criterion_7 --path docs/example.md --name 'ordinary noun' --expected violation +``` + +Use the actual changed path and the intended label (`pass` for a false positive, +`violation` for a missed violation). Inspect the preview and redact private data. +Repeat with `--output .judgement/examples/new-case.json` to save a new file; +existing files are never overwritten or staged. Test it with +`pnpm judgement test --rule criterion_7 --examples .judgement/examples/new-case.json`, +then merge the reviewed example into that rule's fixture file. Supporting context +is included when the rule requests it; `--context ` adds an unchanged file. +Committed changes can be captured with `--base --head `. + +To compare model, rule, or cutoff changes, save JSON reports before and after on +the same examples, then run: + +```sh +pnpm judgement compare --before .judgement/results/before.json --after .judgement/results/after.json +``` + +Comparison reports improvements and regressions per example, including inference +failures. Both reports must include fixture hashes from the current library. +Use `--before-threshold` and `--after-threshold` to select recorded calibration +cutoffs. A regression exits with code 1. Comparison makes no inference calls. + +## Check output and CI + +Findings include the observed probability, rule cutoff, and a bounded preview of +changed lines. The preview identifies the packet's edits; it does not claim the +model located the violation on a particular line. Longer checks print progress to +stderr; `--no-progress` disables it. JSON reports remain on stdout. + +The standalone library reuses cached judgments for matching packets. Roomote's +custom inference adapter keeps caching disabled because backend settings can +change without a stable cache fingerprint. + +The `Judgement` GitHub Actions workflow runs a strict check of the full PR diff. +A job with read-only permissions and no inference secret collects Git objects for +the base and head snapshots. A separate job imports those objects into an empty +repository and runs the pinned checker without checking out PR files. Git hooks, +configuration, and repository history are not transferred. Status publication +runs separately with repository status permission. Violations, +missing inference credentials, timeouts, and incomplete checks fail the status. +The workflow uses a dedicated `TYPESAFE_API_KEY` repository secret and the +`JUDGEMENT_MODEL` repository variable (default `jev-1.13.0`). Keep that model aligned +with calibration; local commands use Roomote's configured model. + +After this workflow lands, configure the secret, verify a successful run, and add +`Judgement` as a required status check for protected branches. Until then, the +workflow alone does not enforce a merge gate. Keep the workflow's pinned checker +version aligned with the application dependencies. diff --git a/apps/worker/package.json b/apps/worker/package.json index 771e1e9bf2..6df3824c67 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -20,7 +20,7 @@ "dependencies": { "@hono/node-server": "2.0.10", "@modelcontextprotocol/sdk": "^1.29.0", - "@roo-code/judgement": "0.3.1", + "@roo-code/judgement": "0.4.0", "@roomote/auth": "workspace:^", "@roomote/cloud-agents": "workspace:^", "@roomote/communication": "workspace:^", diff --git a/package.json b/package.json index a5793ea918..a5d4a7879a 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "devDependencies": { "@changesets/cli": "^2.31.0", "@dotenvx/dotenvx": "^1.52.0", + "@roo-code/judgement": "0.4.0", "@roomote/config-typescript": "workspace:^", "@types/node": "^24.10.13", "@typescript/native-preview": "7.0.0-dev.20260614.1", diff --git a/packages/cloud-agents/package.json b/packages/cloud-agents/package.json index 9e32fb551d..3052cde230 100644 --- a/packages/cloud-agents/package.json +++ b/packages/cloud-agents/package.json @@ -100,7 +100,7 @@ "@aws-sdk/client-s3": "3.1000.0", "@modelcontextprotocol/sdk": "^1.29.0", "@opencode-ai/sdk": "1.18.30", - "@roo-code/judgement": "0.3.1", + "@roo-code/judgement": "0.4.0", "@roomote/ado": "workspace:^", "@roomote/auth": "workspace:^", "@roomote/bitbucket": "workspace:^", diff --git a/packages/cloud-agents/src/server/__tests__/judgement-local.test.ts b/packages/cloud-agents/src/server/__tests__/judgement-local.test.ts index 817bfcf9d1..6edb58fe0e 100644 --- a/packages/cloud-agents/src/server/__tests__/judgement-local.test.ts +++ b/packages/cloud-agents/src/server/__tests__/judgement-local.test.ts @@ -116,3 +116,58 @@ it('does not expose backend errors in example suite reports', async () => { expect(report.reports[0]?.results[0]?.operationalFailure).toBe(true); expect(JSON.stringify(report)).not.toContain('secret provider details'); }); + +it('routes shared CLI checks through Roomote inference and keeps JSON readable', async () => { + const { runCli } = await import('@roo-code/judgement'); + const stdout: string[] = []; + const code = await runCli(['check', '--format', 'json', '--no-progress'], { + cwd, + check: checkLocalRepositoryJudgement, + stdout: (text) => { + stdout.push(text); + }, + }); + expect(code).toBe(1); + const report = JSON.parse(stdout.join('')); + expect(report.status).toBe('violation'); + expect(report.rules[0].findings[0].changes).toEqual([ + expect.objectContaining({ path: 'example.md', side: 'after', line: 1 }), + ]); + expect(evaluate).toHaveBeenCalled(); +}); + +it('captures staged examples through the shared CLI without loading inference', async () => { + const { runCli } = await import('@roo-code/judgement'); + const { evaluateLocalRepositoryJudgement } = + await import('../judgement-local'); + await writeFile(join(cwd, 'example.md'), 'unstaged content'); + const stdout: string[] = []; + const code = await runCli( + [ + 'capture', + '--rule', + 'criterion_1', + '--path', + 'example.md', + '--name', + 'ordinary noun', + '--expected', + 'violation', + ], + { + cwd, + evaluate: evaluateLocalRepositoryJudgement, + check: checkLocalRepositoryJudgement, + stdout: (text) => { + stdout.push(text); + }, + }, + ); + expect(code).toBe(0); + const fixture = JSON.parse(stdout.join('')); + expect(fixture.examples[0]).toMatchObject({ + after: 'new Session checks', + expected: 'violation', + }); + expect(evaluate).not.toHaveBeenCalled(); +}); diff --git a/packages/cloud-agents/src/server/judgement-local-cli.ts b/packages/cloud-agents/src/server/judgement-local-cli.ts index 6739b35926..68adbb9e93 100644 --- a/packages/cloud-agents/src/server/judgement-local-cli.ts +++ b/packages/cloud-agents/src/server/judgement-local-cli.ts @@ -1,167 +1,33 @@ -import { parseArgs } from 'node:util'; -import { - exitCode, - formatReport, - testRules, - calibrateRules, - formatExampleSuite, - exampleSuiteExitCode, -} from '@roo-code/judgement'; +import { runCli, ConfigurationError } from '@roo-code/judgement'; import { checkLocalRepositoryJudgement, evaluateLocalRepositoryJudgement, } from './judgement-local'; -async function main() { - const { values, positionals } = parseArgs({ - allowPositionals: true, - options: { - cwd: { type: 'string' }, - all: { type: 'boolean' }, - rule: { type: 'string' }, - examples: { type: 'string' }, - 'examples-dir': { type: 'string' }, - repeats: { type: 'string' }, - thresholds: { type: 'string' }, - concurrency: { type: 'string' }, - staged: { type: 'boolean' }, - verbose: { type: 'boolean', short: 'v' }, - hook: { type: 'boolean' }, - advisory: { type: 'boolean' }, - base: { type: 'string' }, - head: { type: 'string' }, - 'dry-run': { type: 'boolean' }, - 'no-cache': { type: 'boolean' }, - format: { type: 'string' }, - 'timeout-ms': { type: 'string' }, - help: { type: 'boolean', short: 'h' }, - }, - }); - if (values.help) { - console.log( - 'pnpm judgement [--verbose] [--dry-run] [--format json] [--hook] [--base --head ] [--timeout-ms ] [--advisory]\npnpm judgement test [--rule ] [--dry-run] [--format json]\npnpm judgement calibrate (--rule | --all) [--thresholds 0.8,0.85,0.9] [--format json]\nUses Roomote Settings > Models and server-side provider credentials. Cache is disabled because deployment inference settings can change.', - ); - return 0; - } - const command = positionals[0] ?? 'check'; - if ( - positionals.length > 1 || - !['check', 'test', 'calibrate'].includes(command) - ) { - throw new Error('Invalid command.'); - } - if (command !== 'check') { - if ( - values.staged || - values.base || - values.head || - values.hook || - values.advisory || - (values.all && values.rule) || - (values.examples && values['examples-dir']) || - (command === 'test' && values.thresholds !== undefined) || - (command === 'calibrate' && !values.rule && !values.all) || - values.thresholds?.split(',').some((value) => !value.trim()) || - (values.format && !['text', 'json'].includes(values.format)) - ) { - throw new Error('Invalid example command arguments.'); - } - const controller = new AbortController(); - const interrupt = () => controller.abort(); - process.once('SIGINT', interrupt); - try { - const options = { - cwd: values.cwd, - ruleId: values.rule, - examplesPath: values.examples, - examplesDirectory: values['examples-dir'], - repeats: - values.repeats === undefined ? undefined : Number(values.repeats), - concurrency: - values.concurrency === undefined - ? undefined - : Number(values.concurrency), - deadlineMs: - values['timeout-ms'] === undefined - ? undefined - : Number(values['timeout-ms']), - dryRun: values['dry-run'], - signal: controller.signal, - evaluate: evaluateLocalRepositoryJudgement, - onDiagnostic: values.verbose - ? (message: string) => console.error(`judgement: ${message}`) - : undefined, - }; - const report = - command === 'test' - ? await testRules(options) - : await calibrateRules({ - ...options, - thresholds: values.thresholds?.split(',').map(Number), - }); - console.log( - values.format === 'json' - ? JSON.stringify(report) - : formatExampleSuite(report), - ); - return exampleSuiteExitCode(report); - } finally { - process.removeListener('SIGINT', interrupt); - } - } - if ( - values.all || - values.rule || - values.examples || - values['examples-dir'] || - values.repeats || - values.thresholds || - values.concurrency - ) { - throw new Error('Example options require test or calibrate.'); - } - if ( - (values.staged && values.base) || - (values.head && !values.base) || - (values.hook && values.base) || - (values.format && !['text', 'json'].includes(values.format)) - ) { - console.error('Invalid arguments. Use pnpm judgement --help.'); - return 2; - } - const report = await checkLocalRepositoryJudgement({ - cwd: values.cwd, - base: values.base, - head: values.head, - hook: values.hook, - dryRun: values['dry-run'], - deadlineMs: - values['timeout-ms'] === undefined - ? undefined - : Number(values['timeout-ms']), - onDiagnostic: values.verbose - ? (message) => console.error(`judgement: ${message}`) - : undefined, - }); - console.log( - values.format === 'json' ? JSON.stringify(report) : formatReport(report), - ); - return values['dry-run'] && report.status !== 'invalid' - ? 0 - : exitCode(report, values); -} - -// The settings resolver opens shared database pools; this one-shot CLI must -// exit after reporting rather than waiting for the server pools to idle out. -void main() +// The host supplies inference; command parsing, fixtures, and reports belong to Judgement. +void runCli(process.argv.slice(2), { + check: checkLocalRepositoryJudgement, + evaluate: evaluateLocalRepositoryJudgement, +}) + // The deployment settings resolver opens database pools; this CLI is one-shot. .then((code) => process.exit(code)) .catch((error: unknown) => { if (error instanceof Error && error.name === 'AbortError') { console.error('judgement: interrupted.'); process.exit(130); } + if (error instanceof ConfigurationError) { + console.error(`judgement: ${error.message}`); + process.exit(2); + } + if (error instanceof Error && 'code' in error && error.code === 'EEXIST') { + console.error( + 'judgement: output already exists. Choose a new file; capture never overwrites.', + ); + process.exit(2); + } console.error( - 'judgement: failed to start. Check arguments, .env.local, and local Roomote configuration.', + 'judgement: failed to start. Check arguments, .env.local, and local Roomote configuration. Use pnpm judgement --help for command syntax.', ); process.exit(2); }); diff --git a/packages/cloud-agents/src/server/judgement-local.ts b/packages/cloud-agents/src/server/judgement-local.ts index 831160378d..b9d2bb0d1b 100644 --- a/packages/cloud-agents/src/server/judgement-local.ts +++ b/packages/cloud-agents/src/server/judgement-local.ts @@ -29,7 +29,9 @@ export async function evaluateLocalRepositoryJudgement( return answer; } -export async function checkLocalRepositoryJudgement(options: CheckOptions) { +export async function checkLocalRepositoryJudgement( + options: CheckOptions = {}, +) { const failures = new Set(); const report = await check({ ...options, diff --git a/packages/cloud-agents/src/server/judgement-presets.generated.json b/packages/cloud-agents/src/server/judgement-presets.generated.json index 99fab862da..73fc6a8a0c 100644 --- a/packages/cloud-agents/src/server/judgement-presets.generated.json +++ b/packages/cloud-agents/src/server/judgement-presets.generated.json @@ -2559,7 +2559,7 @@ ".judgement/examples/criterion_8.json": "112b13c4f30ad1e79f5d082fe5ff6ee3911839f0632f7c70997db90fddd3265d", "packages/cloud-agents/scripts/prepare-judgement-presets.mjs": "aef362f4686e232a472ff9e2aaa816882f5b60a2c2cfbcea9840efff040bec4e" }, - "library": "0.3.1", + "library": "0.4.0", "questionSets": [ { "result": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 96b338d0d0..b4791dbd42 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -91,6 +91,9 @@ importers: '@dotenvx/dotenvx': specifier: ^1.52.0 version: 1.52.0 + '@roo-code/judgement': + specifier: 0.4.0 + version: 0.4.0 '@roomote/config-typescript': specifier: workspace:^ version: link:packages/config-typescript @@ -942,8 +945,8 @@ importers: specifier: ^1.29.0 version: 1.29.0(@cfworker/json-schema@4.1.1)(zod@3.25.76) '@roo-code/judgement': - specifier: 0.3.1 - version: 0.3.1 + specifier: 0.4.0 + version: 0.4.0 '@roomote/auth': specifier: workspace:^ version: link:../../packages/auth @@ -1147,8 +1150,8 @@ importers: specifier: 1.18.30 version: 1.18.30 '@roo-code/judgement': - specifier: 0.3.1 - version: 0.3.1 + specifier: 0.4.0 + version: 0.4.0 '@roomote/ado': specifier: workspace:^ version: link:../ado @@ -5101,8 +5104,8 @@ packages: cpu: [x64] os: [win32] - '@roo-code/judgement@0.3.1': - resolution: {integrity: sha512-h4r+drsLIwILr8OdnywqgSTAlONF32t6D4PoAtHW2TjIh7/Y8jvOIBoAlLwC4H5mtTdaaQhEBsyUFAsEvQncxQ==} + '@roo-code/judgement@0.4.0': + resolution: {integrity: sha512-MPleLuyuZI4OZjijiiCwroC8XS7ZdoxPbnbiLjg23ryirTM9WLZoqAuySVGTOPmQraOTTRiXKFuuBCDik5LdDw==} engines: {node: '>=22.20.0'} hasBin: true @@ -15587,7 +15590,7 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.61.1': optional: true - '@roo-code/judgement@0.3.1': + '@roo-code/judgement@0.4.0': dependencies: picomatch: 4.0.7 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 1751cedbc6..180e365f83 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,7 +6,7 @@ minimumReleaseAge: 10080 # lockfile-affecting changes while it finishes maturing. minimumReleaseAgeExclude: # First-party release reviewed with the Judgement integration. - - "@roo-code/judgement@0.3.1" + - "@roo-code/judgement@0.4.0" - jose # Allow the patched releases while they finish maturing. - brace-expansion diff --git a/scripts/judgement/__tests__/workflow.test.mjs b/scripts/judgement/__tests__/workflow.test.mjs new file mode 100644 index 0000000000..028ef058b6 --- /dev/null +++ b/scripts/judgement/__tests__/workflow.test.mjs @@ -0,0 +1,210 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { + existsSync, + mkdtempSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; +import YAML from 'yaml'; +import { check } from '@roo-code/judgement'; + +const root = join(dirname(fileURLToPath(import.meta.url)), '../../..'); +const workflow = YAML.parse( + readFileSync(join(root, '.github/workflows/judgement.yml'), 'utf8'), +); +const script = (job, id) => + workflow.jobs[job].steps.find((step) => step.id === id).run; + +function fixture(t) { + const dir = mkdtempSync(join(tmpdir(), 'judgement-ci-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + return dir; +} + +test('CI separates PR object fetching, inference secrets, and status writes', () => { + assert.deepEqual(workflow.permissions, {}); + assert.deepEqual(workflow.jobs.collect.permissions, { contents: 'read' }); + assert.doesNotMatch(JSON.stringify(workflow.jobs.collect), /secrets\./); + assert.deepEqual(workflow.jobs.judgement.permissions, {}); + assert.ok( + workflow.jobs.judgement.steps.every( + (step) => !step.uses?.startsWith('actions/checkout@'), + ), + ); + for (const job of ['pending', 'report']) { + assert.deepEqual(workflow.jobs[job].permissions, { statuses: 'write' }); + assert.doesNotMatch( + JSON.stringify(workflow.jobs[job]), + /secrets\.|actions\/checkout|download-artifact/, + ); + } + assert.match(workflow.jobs.report.if, /always\(\)/); + assert.deepEqual(workflow.jobs.report.needs, [ + 'pending', + 'collect', + 'judgement', + ]); +}); + +test('snapshot transfer preserves judgments and trusted policy without checking out PR files or copying Git configuration', async (t) => { + const dir = fixture(t); + const source = join(dir, 'source'); + const runner = join(dir, 'runner'); + mkdirSync(source); + mkdirSync(runner); + const git = (...args) => + execFileSync('git', args, { cwd: source, encoding: 'utf8' }).trim(); + git('init', '-q'); + git('config', 'user.name', 'Test'); + git('config', 'user.email', 'test@example.invalid'); + writeFileSync(join(source, 'ancestor'), 'not needed by either snapshot'); + git('add', '.'); + git('commit', '-qm', 'ancestor'); + const ancestor = git('rev-parse', 'HEAD'); + git('rm', 'ancestor'); + mkdirSync(join(source, '.judgement')); + writeFileSync( + join(source, '.judgement/rules.json'), + JSON.stringify({ + criteria: [ + { + id: 'wording', + rule: 'Use lowercase session.', + files: ['**/*.md'], + threshold: 0.75, + }, + ], + }), + ); + writeFileSync(join(source, 'example.md'), 'session\n'); + git('add', '.'); + git('commit', '-qm', 'base'); + const base = git('rev-parse', 'HEAD'); + writeFileSync(join(source, 'example.md'), 'Session\n'); + writeFileSync( + join(source, '.judgement/rules.json'), + JSON.stringify({ criteria: [{ rule: 'Allow every change.' }] }), + ); + writeFileSync( + join(source, 'package.json'), + JSON.stringify({ scripts: { postinstall: 'exit 99' } }), + ); + symlinkSync('/outside-repository', join(source, '.npmrc')); + git('add', '.'); + git('commit', '-qm', 'proposed'); + const head = git('rev-parse', 'HEAD'); + git('remote', 'add', 'origin', source); + git('config', 'core.hooksPath', '/untrusted-hooks'); + const env = { + ...process.env, + RUNNER_TEMP: runner, + PR_HEAD: head, + PR_BASE: base, + GITHUB_OUTPUT: join(dir, 'outputs'), + }; + execFileSync('bash', ['-c', script('collect', 'export')], { + cwd: source, + env, + stdio: 'pipe', + }); + const actualBase = readFileSync(env.GITHUB_OUTPUT, 'utf8') + .trim() + .split('=')[1]; + assert.equal(actualBase, base); + execFileSync('bash', ['-c', script('judgement', 'import')], { + cwd: dir, + env: { ...env, JUDGEMENT_BASE: actualBase }, + stdio: 'pipe', + }); + const input = join(runner, 'judgement-input'); + assert.deepEqual(readdirSync(input), ['.git']); + assert.equal(existsSync(join(input, '.git/hooks')), false); + assert.doesNotMatch( + readFileSync(join(input, '.git/config'), 'utf8'), + /hooksPath|origin/, + ); + assert.throws(() => + execFileSync('git', ['cat-file', '-e', ancestor], { + cwd: input, + stdio: 'pipe', + }), + ); + const requests = []; + const original = await check({ + cwd: source, + base, + head, + cache: false, + evaluate: async (request) => { + requests.push(request); + return { violationProbability: 0.99 }; + }, + }); + const copiedRequests = []; + const copied = await check({ + cwd: input, + base, + head, + cache: false, + evaluate: async (request) => { + copiedRequests.push(request); + return { violationProbability: 0.99 }; + }, + }); + assert.equal(original.status, 'violation'); + assert.equal(copied.status, original.status); + assert.equal(copied.policySource, 'base'); + assert.deepEqual(copiedRequests, requests); + assert.deepEqual(readdirSync(input), ['.git']); +}); + +test('corrupt snapshot objects fail before inference', (t) => { + const dir = fixture(t); + mkdirSync(join(dir, 'judgement-evidence')); + writeFileSync(join(dir, 'judgement-evidence/objects.pack'), 'not a Git pack'); + assert.throws(() => + execFileSync('bash', ['-c', script('judgement', 'import')], { + cwd: dir, + env: { + ...process.env, + RUNNER_TEMP: dir, + PR_HEAD: 'a'.repeat(40), + JUDGEMENT_BASE: 'b'.repeat(40), + }, + stdio: 'pipe', + }), + ); +}); + +test('skipped and failed inference jobs publish failure, while successful checks publish success', (t) => { + const dir = fixture(t); + const gh = join(dir, 'gh'); + writeFileSync(gh, '#!/bin/sh\nprintf "%s\\n" "$@"\n', { mode: 0o755 }); + const publish = workflow.jobs.report.steps[0].run; + for (const outcome of ['success', 'failure', 'skipped', 'cancelled']) { + const output = execFileSync('bash', ['-c', publish], { + cwd: dir, + encoding: 'utf8', + env: { + ...process.env, + PATH: `${dir}:${process.env.PATH}`, + CHECK_OUTCOME: outcome, + PR_HEAD: 'a'.repeat(40), + GITHUB_REPOSITORY: 'example/repo', + }, + }); + assert.match( + output, + new RegExp(`state=${outcome === 'success' ? 'success' : 'failure'}\\n`), + ); + } +});