Integrate Judgement feedback tools and strict PR checks

.github/workflows/CI.yml

@@ -186,6 +186,8 @@ jobs:
         env:
           DATABASE_URL: postgres://postgres:password@localhost:5432/test
           REDIS_URL: redis://localhost:6379
+      - name: Test Judgement CI isolation
+        run: node --test scripts/judgement/__tests__/*.test.mjs
       - name: Run release script unit tests
         run: pnpm test:release-scripts
 

.github/workflows/judgement.yml

@@ -0,0 +1,152 @@
+name: Judgement
+
+on:
+  pull_request_target:
+    branches: [main, develop]
+    types: [opened, synchronize, reopened, ready_for_review, edited]
+
+permissions: {}
+
+concurrency:
+  group: judgement-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  pending:
+    name: Mark Judgement pending
+    runs-on: ubuntu-24.04
+    permissions:
+      statuses: write
+    steps:
+      - name: Mark the PR check pending
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_HEAD: ${{ github.event.pull_request.head.sha }}
+        run: |
+          gh api --silent "repos/$GITHUB_REPOSITORY/statuses/$PR_HEAD" \
+            -f state=pending -f context=Judgement \
+            -f description='Checking repository rules' \
+            -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+
+  collect:
+    name: Collect Git evidence
+    needs: pending
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    outputs:
+      base: ${{ steps.export.outputs.base }}
+    env:
+      PR_HEAD: ${{ github.event.pull_request.head.sha }}
+      PR_BASE: ${{ github.event.pull_request.base.sha }}
+    steps:
+      # This job has no inference secret or repository write permission.
+      - name: Checkout trusted base
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Export snapshot objects
+        id: export
+        run: |
+          set -euo pipefail
+          [[ "$PR_HEAD" =~ ^[0-9a-f]{40}$ && "$PR_BASE" =~ ^[0-9a-f]{40}$ ]]
+          git fetch --no-tags origin "$PR_HEAD"
+          base="$(git merge-base "$PR_BASE" "$PR_HEAD")"
+          git cat-file -e "$base:.judgement/rules.json"
+          mkdir -p "$RUNNER_TEMP/judgement-evidence"
+          # Transfer objects only: no worktree, Git config, hooks, or full history.
+          {
+            printf '%s\n' "$base" "$PR_HEAD"
+            git rev-list --objects --no-object-names "$base^{tree}" "$PR_HEAD^{tree}"
+          } | sort -u | git pack-objects --stdout > "$RUNNER_TEMP/judgement-evidence/objects.pack"
+          echo "base=$base" >> "$GITHUB_OUTPUT"
+      - name: Upload snapshot objects
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: judgement-evidence
+          path: ${{ runner.temp }}/judgement-evidence/objects.pack
+          if-no-files-found: error
+          retention-days: 1
+          compression-level: 0
+
+  judgement:
+    name: Judge Git evidence
+    needs: collect
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    permissions: {}
+    env:
+      PR_HEAD: ${{ github.event.pull_request.head.sha }}
+      JUDGEMENT_BASE: ${{ needs.collect.outputs.base }}
+    steps:
+      # No repository checkout in this job. The checker is installed separately.
+      - name: Set up Node
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: '24.13.1'
+      - name: Install trusted checker
+        run: |
+          mkdir -p "$RUNNER_TEMP/judgement-tool"
+          cd "$RUNNER_TEMP/judgement-tool"
+          npm install --ignore-scripts --no-audit --no-fund \
+            '@roo-code/judgement@0.4.0'
+      - name: Download snapshot objects
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: judgement-evidence
+          path: ${{ runner.temp }}/judgement-evidence
+      - name: Import snapshot objects without checkout
+        id: import
+        run: |
+          set -euo pipefail
+          [[ "$PR_HEAD" =~ ^[0-9a-f]{40}$ && "$JUDGEMENT_BASE" =~ ^[0-9a-f]{40}$ ]]
+          git init --quiet --template= "$RUNNER_TEMP/judgement-input"
+          cd "$RUNNER_TEMP/judgement-input"
+          git index-pack --stdin < "$RUNNER_TEMP/judgement-evidence/objects.pack"
+          git cat-file -e "$JUDGEMENT_BASE^{commit}"
+          git cat-file -e "$PR_HEAD^{commit}"
+          git cat-file -e "$JUDGEMENT_BASE:.judgement/rules.json"
+          git update-ref refs/heads/evidence "$JUDGEMENT_BASE"
+          git symbolic-ref HEAD refs/heads/evidence
+      - name: Run strict check
+        env:
+          TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }}
+          JUDGEMENT_MODEL: ${{ vars.JUDGEMENT_MODEL || 'jev-1.13.0' }}
+        run: |
+          set -euo pipefail
+          if [ -z "$TYPESAFE_API_KEY" ]; then
+            echo 'Configure the TYPESAFE_API_KEY repository secret for Judgement.' >&2
+            exit 2
+          fi
+          node "$RUNNER_TEMP/judgement-tool/node_modules/@roo-code/judgement/src/cli.js" \
+            check --cwd "$RUNNER_TEMP/judgement-input" \
+            --base "$JUDGEMENT_BASE" --head "$PR_HEAD" \
+            --model "$JUDGEMENT_MODEL" --timeout-ms 120000 --no-cache
+
+  report:
+    name: Publish Judgement result
+    needs: [pending, collect, judgement]
+    if: ${{ always() && !cancelled() && needs.pending.result == 'success' }}
+    runs-on: ubuntu-24.04
+    permissions:
+      statuses: write
+    steps:
+      # This job receives only the job conclusion, never repository contents or secrets.
+      - name: Publish strict result
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_HEAD: ${{ github.event.pull_request.head.sha }}
+          CHECK_OUTCOME: ${{ needs.judgement.result }}
+        run: |
+          state=failure
+          description='Judgement failed or did not complete; inspect the run'
+          if [ "$CHECK_OUTCOME" = success ]; then
+            state=success
+            description='Repository rule check completed without findings'
+          fi
+          gh api --silent "repos/$GITHUB_REPOSITORY/statuses/$PR_HEAD" \
+            -f state="$state" -f context=Judgement -f description="$description" \
+            -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"

.judgement/README.md

@@ -94,3 +94,59 @@ Rules 7 and 8 use a violation cutoff of 0.75. Validate the capitalization and pr
 rules together with sentence starts, quoted UI labels, headings, migration and
 security requirements, and edits near unchanged violations. Difficult examples
 remain in the validation fixtures even when their scores fall below the cutoff.
+
+## Turn feedback into examples
+
+Capture reads the staged version and previews a fixture without inference:
+
+```sh
+pnpm judgement capture --rule criterion_7 --path docs/example.md --name 'ordinary noun' --expected violation
+```
+
+Use the actual changed path and the intended label (`pass` for a false positive,
+`violation` for a missed violation). Inspect the preview and redact private data.
+Repeat with `--output .judgement/examples/new-case.json` to save a new file;
+existing files are never overwritten or staged. Test it with
+`pnpm judgement test --rule criterion_7 --examples .judgement/examples/new-case.json`,
+then merge the reviewed example into that rule's fixture file. Supporting context
+is included when the rule requests it; `--context <path>` adds an unchanged file.
+Committed changes can be captured with `--base <commit> --head <commit>`.
+
+To compare model, rule, or cutoff changes, save JSON reports before and after on
+the same examples, then run:
+
+```sh
+pnpm judgement compare --before .judgement/results/before.json --after .judgement/results/after.json
+```
+
+Comparison reports improvements and regressions per example, including inference
+failures. Both reports must include fixture hashes from the current library.
+Use `--before-threshold` and `--after-threshold` to select recorded calibration
+cutoffs. A regression exits with code 1. Comparison makes no inference calls.
+
+## Check output and CI
+
+Findings include the observed probability, rule cutoff, and a bounded preview of
+changed lines. The preview identifies the packet's edits; it does not claim the
+model located the violation on a particular line. Longer checks print progress to
+stderr; `--no-progress` disables it. JSON reports remain on stdout.
+
+The standalone library reuses cached judgments for matching packets. Roomote's
+custom inference adapter keeps caching disabled because backend settings can
+change without a stable cache fingerprint.
+
+The `Judgement` GitHub Actions workflow runs a strict check of the full PR diff.
+A job with read-only permissions and no inference secret collects Git objects for
+the base and head snapshots. A separate job imports those objects into an empty
+repository and runs the pinned checker without checking out PR files. Git hooks,
+configuration, and repository history are not transferred. Status publication
+runs separately with repository status permission. Violations,
+missing inference credentials, timeouts, and incomplete checks fail the status.
+The workflow uses a dedicated `TYPESAFE_API_KEY` repository secret and the
+`JUDGEMENT_MODEL` repository variable (default `jev-1.13.0`). Keep that model aligned
+with calibration; local commands use Roomote's configured model.
+
+After this workflow lands, configure the secret, verify a successful run, and add
+`Judgement` as a required status check for protected branches. Until then, the
+workflow alone does not enforce a merge gate. Keep the workflow's pinned checker
+version aligned with the application dependencies.

apps/worker/package.json

@@ -20,7 +20,7 @@
   "dependencies": {
     "@hono/node-server": "2.0.10",
     "@modelcontextprotocol/sdk": "^1.29.0",
-    "@roo-code/judgement": "0.3.1",
+    "@roo-code/judgement": "0.4.0",
     "@roomote/auth": "workspace:^",
     "@roomote/cloud-agents": "workspace:^",
     "@roomote/communication": "workspace:^",

package.json

@@ -47,6 +47,7 @@
   "devDependencies": {
     "@changesets/cli": "^2.31.0",
     "@dotenvx/dotenvx": "^1.52.0",
+    "@roo-code/judgement": "0.4.0",
     "@roomote/config-typescript": "workspace:^",
     "@types/node": "^24.10.13",
     "@typescript/native-preview": "7.0.0-dev.20260614.1",

packages/cloud-agents/package.json

@@ -100,7 +100,7 @@
     "@aws-sdk/client-s3": "3.1000.0",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "@opencode-ai/sdk": "1.18.30",
-    "@roo-code/judgement": "0.3.1",
+    "@roo-code/judgement": "0.4.0",
     "@roomote/ado": "workspace:^",
     "@roomote/auth": "workspace:^",
     "@roomote/bitbucket": "workspace:^",