feat: add the skill eval runner

.github/workflows/ci.yml

@@ -75,6 +75,13 @@ jobs:
       # shellcheck is preinstalled on ubuntu-latest runners.
       - run: make test-scripts
 
+  evals:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+      # Offline unit tests of the eval runner: no agents, no tokens.
+      - run: make test-evals
+
   docs:
     runs-on: ubuntu-latest
     env:

.gitignore

@@ -34,3 +34,9 @@ REPO_READY_AUDIT.md
 # be edited without a PR. A stray copy at the root stays untracked.
 /ROADMAP.md
 NOTICE.txt.tmp
+
+# Skill evals: the tester's config and raw results stay local; only
+# evals/releases/<version>/SUMMARY.md is committed.
+/evals/eval.toml
+/evals/results/
+/evals/__pycache__/

CLAUDE.md

@@ -63,6 +63,8 @@ self-hosted Control Plane).
   drift
 - `make docs-check` — report reference drift without rewriting (CI)
 - `make test-scripts` — shellcheck + the shell unit-test suites
+- `make test-evals` — unit tests for the skill eval runner's grading,
+  scoring and publishing (`evals/`); runs no agents, needs no tenant
 - `make generate` — regenerate API clients from the OpenAPI specs
   (do not regenerate casually — see CONTRIBUTING.md "OpenAPI specs
   and generated clients")
@@ -92,6 +94,19 @@ self-hosted Control Plane).
 - `make build-all` — cross-platform build matrix; `make build-one
   GOOS=... GOARCH=...` is one target of it, which CI runs one per job
 
+## Before tagging a release
+
+Score the skills at the commit the tag will point to, then hand the
+scorecard to whoever is tagging. It reports; the decision to release
+is theirs, never the script's.
+
+    python3 evals/run.py --config evals/eval.toml \
+        --release <version> --ref <sha>
+
+Setup, the config and what each scorecard column means are in
+`evals/README.md`. The run writes `evals/releases/<version>/SUMMARY.md`;
+commit it in the next PR.
+
 ## Verifying a change — the compiler is the authority
 
 Every trap below has produced a confidently wrong conclusion here.

Makefile

@@ -16,7 +16,7 @@ CONTROLPLANE_VERSION    ?= dev
 
 LDFLAGS := -ldflags "-X $(PKG).Version=$(VERSION) -X $(PKG).Commit=$(COMMIT) -X $(PKG).BuildDate=$(DATE) -X $(PKG_STARFLEET).Version=$(STARFLEET_VERSION) -X $(PKG_CONTROLPLANE).Version=$(CONTROLPLANE_VERSION)"
 
-.PHONY: build test test-scripts test-integration test-integration-lifecycle test-inspect-live lint lint-docs clean generate docs docs-check build-all build-one notice vendor-spec vendor-spec-check
+.PHONY: build test test-evals test-scripts test-integration test-integration-lifecycle test-inspect-live lint lint-docs clean generate docs docs-check build-all build-one notice vendor-spec vendor-spec-check
 
 build:
 	go build $(LDFLAGS) -o $(BINARY) ./cmd/pgedge
@@ -30,6 +30,10 @@ test:
 	go test -race -coverprofile=coverage.raw.out ./...
 	scripts/coverage-gate.sh
 
+# The skill eval runner's grading and scoring, offline: no agents run.
+test-evals:
+	python3 -m unittest discover -s evals
+
 # Shell-script tests: shellcheck + the shell unit-test suites.
 test-scripts:
 	shellcheck install.sh scripts/coverage-gate.sh scripts/lint-docs.sh \

evals/eval.example.toml

@@ -0,0 +1,30 @@
+# Copy to eval.toml (ignored by git) and edit.
+
+# A pgedge config and the profile in it to test against. The eval
+# creates and deletes databases in that profile's tenant, so use one
+# where that is safe. Runs get a HOME of their own and cannot reach the
+# OS keychain, so log in with --insecure-storage, into a config kept
+# for the eval:
+#
+#   pgedge starfleet auth login --profile eval \
+#       --config ~/.pgedge/cli/eval.yaml --insecure-storage
+pgedge_config = "~/.pgedge/cli/eval.yaml"
+profile = "eval"
+
+# The values below are the release suite. Keep them for a --release
+# run, so one release compares with the next.
+runs = 3
+tasks = [6, 7, 8, 9, 10]
+arms = ["with", "without", "api"]
+
+# One table per harness. Each model gets its own lane and fixture.
+[harness.claude]
+models = ["sonnet"]
+
+# Only this one: the runner refuses any other Cursor model.
+[harness.cursor]
+models = ["grok-4.7-medium"]
+
+# The ChatGPT subscription only: the runner refuses an API-key login.
+[harness.codex]
+models = ["gpt-6.1-sol", "gpt-6-luna"]