diff --git a/.well-known/agents-shipgate.json b/.well-known/agents-shipgate.json index 57ff83d5..8ba40386 100644 --- a/.well-known/agents-shipgate.json +++ b/.well-known/agents-shipgate.json @@ -180,7 +180,7 @@ "agent_boundary_result_schema_path": "docs/agent-boundary-result-schema.v1.json", "report_schema_version": "0.34", "packet_schema_version": "0.12", - "verifier_schema_version": "0.6", + "verifier_schema_version": "0.7", "verify_run_schema_version": "shipgate.verify_run/v3", "verification_plan_schema_version": "shipgate.verification_plan/v1", "verification_unit_result_schema_version": "shipgate.verification_unit_result/v1", @@ -279,7 +279,7 @@ "host_grants_inventory_schema_version": "0.2", "host_grants_baseline_schema_version": "0.2", "host_grants_drift_schema_version": "0.2", - "trigger_catalog_schema_version": "0.2", + "trigger_catalog_schema_version": "0.3", "capability_standard_version": "0.5", "governance_benchmark_catalog_schema_version": "0.2", "governance_benchmark_result_schema_version": "0.2", @@ -441,7 +441,7 @@ "agent_result": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/agent-result-schema.v2.json", "agent_boundary_result": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/agent-boundary-result-schema.v1.json", "codex_boundary_result": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/codex-boundary-result-schema.v2.json", - "verifier": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/verifier-schema.v0.6.json", + "verifier": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/verifier-schema.v0.7.json", "verify_run": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/verify-run-schema.v3.json", "verification_plan": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/verification-plan-schema.v1.json", "verification_unit_result": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/verification-unit-result-schema.v1.json", diff --git a/AGENTS.md b/AGENTS.md index bd326965..99b68025 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -315,7 +315,7 @@ agents-shipgate trigger --base origin/main --head HEAD --json agents-shipgate trigger --list-rules --json ``` -The command emits a stable JSON verdict: `should_run` (alias of `run_shipgate`), `force_run`, `dry_run_recommended`, `skip_reason`, `matched_rules`, `changed_files`, and `diff_tokens`. The developer entry point `python -m agents_shipgate.triggers shipgate.yaml prompts/refund.md` is preserved. +The command emits a stable JSON verdict: `should_run` (alias of `run_shipgate`), `force_run`, `dry_run_recommended`, `skip_reason`, `matched_rules`, `changed_files`, `diff_tokens`, plus `input_status` and `evaluation_status` (catalog schema `0.3`). When the diff could not be read in full, a *skip* verdict is withheld: `evaluation_status` is `not_evaluated` and `should_run`/`run_shipgate`/`skip`/`skip_reason` are `null`, so an unread diff is never reported as `no_match`. A *run* verdict is still published, because rule matching is monotone — evidence that already matched cannot be un-matched by the bytes that are missing — and it arrives as `evaluation_status: evaluated` with `should_run: true`. That evidence may be a rule matched on the change set or `force_run` from a manifest that is present regardless of the diff, so read `matched_rules` before attributing it. Branch on `evaluation_status`, not on `should_run` alone. The developer entry point `python -m agents_shipgate.triggers shipgate.yaml prompts/refund.md` is preserved. **Stop conditions.** Stop and do not run `init` only when **all** of these hold: diff --git a/CHANGELOG.md b/CHANGELOG.md index 457db7ab..e4ca7d4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,44 @@ ## Unreleased +- **An unreadable PR diff is no longer reported as "nothing here is + agent-related."** `verify --preview` collapsed every diff-acquisition failure + into one message, then evaluated the trigger catalog against the empty inputs + that failure left behind — publishing `skip_reason: "no_match"` with the + rationale *"nothing in this PR signals a tool-surface change"* about a PR it + had never read. The top-level control result stayed fail-closed + (`merge_verdict: "unknown"`), but the explanation invited exactly the wrong + conclusion, and on an unconfigured workspace the failure was not reported at + all: both diff-failure branches were gated on a manifest being present, so a + shallow or blobless clone of an un-adopted repository — the normal shape of + first contact — fell through to *"Shipgate is not configured in this + workspace"* with the Git error visible nowhere but `base_notes`. Three + things changed. Diff acquisition is now classified rather than flattened: + `not_attempted`, `refs_missing`, `merge_base_missing`, + `unrelated_histories`, `objects_missing`, `metadata_limit_exceeded`, + `body_limit_exceeded`, `git_timeout`, and `git_failed` are read off Git's own + diagnostic — including the two causes Git reports identically as "no merge + base", a shallow checkout that deepening repairs versus two roots that no + fetch can ever join — and travel on the new + `verifier.json` `diff_status` block together with a bounded, path-redacted + excerpt and the precise repair — deepen history, hydrate partial-clone + objects (verification sets `GIT_NO_LAZY_FETCH=1`, so Git will not fetch them + implicitly), or take it to a human when fetching cannot help. Metadata and + body are collected separately, so a diff whose body cannot be read no longer + discards the changed paths that were read successfully; a blobless clone + answers `--name-status` in full, and those paths are exactly what says a PR + touches an agent surface. And the trigger evaluator gained the state it was + missing: `input_status` and `evaluation_status`, with `should_run`, + `run_shipgate`, `skip`, and `skip_reason` all `null` when the inputs were not + fully read. The asymmetry is deliberate — rule matching is monotone in the + evidence, so a *run* verdict reached from partial evidence stays sound and is + still published, while any *skip* verdict is withheld. Trigger catalog schema + `0.2 → 0.3` (nullable verdict fields, the two new fields, and the new + `next_action.kind: "input_required"`); verifier schema `0.6 → 0.7` + (`diff_status`; v0.6 remains a frozen reference and is still readable). + `contract_version`, `report_schema_version`, and every other schema counter + are unchanged. + - **Google ADK repositories that share one tool between agents can be scanned again.** Binding the same `FunctionTool` to a coordinator and its sub-agents is the canonical ADK multi-agent shape — it is what `google/adk-samples` diff --git a/README.md b/README.md index cd59037c..f77d3e52 100644 --- a/README.md +++ b/README.md @@ -639,7 +639,7 @@ artifacts — in read order: - **`agents-shipgate-reports/verification-receipt.json`** — the **first artifact a coding agent validates**: a terminal content-addressed closure over the exact request (including `verification-input.diff`), worker result, decision, and artifact set. It is written last; use `agents-shipgate verification reproduce` to validate every referenced hash. - **`agents-shipgate-reports/agent-handoff.json`** — the compact `shipgate.agent_handoff/v6` object. Lead with `control.state`, then `gate.merge_verdict`; it projects the same request, decision, and authorization evaluation and does not introduce a second verdict. -- **`agents-shipgate-reports/verifier.json`** — the **authoritative PR/control evidence substrate** (`verifier_schema_version: "0.6"`). A coding agent switches on `control.state`, then reads `authorization`, `merge_verdict` (`mergeable | human_review_required | insufficient_evidence | blocked | unknown`), `can_merge_without_human`, `control.next_action`, and `fix_task` when producing reviewer evidence for an agent-capability PR. Only an accepted signed authorization evaluation may expose an exact reviewed command; the release verdict remains unchanged. Local control comes from `shipgate check --format agent-boundary-json` and `shipgate.agent_boundary_result/v1`. See [`docs/agent-contract-current.md`](docs/agent-contract-current.md) for the field contract. +- **`agents-shipgate-reports/verifier.json`** — the **authoritative PR/control evidence substrate** (`verifier_schema_version: "0.7"`). A coding agent switches on `control.state`, then reads `authorization`, `merge_verdict` (`mergeable | human_review_required | insufficient_evidence | blocked | unknown`), `can_merge_without_human`, `control.next_action`, and `fix_task` when producing reviewer evidence for an agent-capability PR. Only an accepted signed authorization evaluation may expose an exact reviewed command; the release verdict remains unchanged. Local control comes from `shipgate check --format agent-boundary-json` and `shipgate.agent_boundary_result/v1`. See [`docs/agent-contract-current.md`](docs/agent-contract-current.md) for the field contract. - **`agents-shipgate-reports/verify-run.json`** — the `shipgate.verify_run/v3` projection embedding the exact verification plan, executor, unit-result IDs, decision ID, outcome, and artifact paths. Its deprecated `run_id` is an exact alias of `request_id`. - **`agents-shipgate-reports/attestation.json`** + **`agents-shipgate-reports/org-evidence-bundle.json`** — optional organization-governance projections over the same verifier/report artifacts. They are ledger inputs for platform teams, not release gates; `report.json.release_decision.decision` remains the decision engine. - **`agents-shipgate-reports/host-grants.json`** + **`agents-shipgate-reports/org-status.json`** — optional fleet-governance artifacts from `audit --host --out` and `org status --json`, useful for host-grant drift, policy-pack pin state, and exception hygiene. diff --git a/STABILITY.md b/STABILITY.md index eb2ede89..deec681e 100644 --- a/STABILITY.md +++ b/STABILITY.md @@ -13,6 +13,42 @@ for reproducible CI. --- + + +## Migration Note: unreleased — diff input health + +Verifier schema `0.6 → 0.7` and trigger catalog `0.2 → 0.3`. `contract_version` +stays at `19`; no CLI surface changed. + +`verifier.json` gains a top-level `diff_status` block that reports whether the +compared change set was actually read: `completeness` (`complete` / `partial` / +`unavailable`), a `reason` token (`not_attempted`, `refs_missing`, +`merge_base_missing`, `unrelated_histories`, `objects_missing`, +`metadata_limit_exceeded`, `body_limit_exceeded`, `git_timeout`, +`git_failed`), a bounded path-redacted `detail`, the +`remediation`, and `fetch_repairable`. Verifier v0.6 remains a frozen reference +and its artifacts still parse. + +The trigger evaluator gains `input_status` and `evaluation_status`, and +`should_run`, `run_shipgate`, `skip`, and `skip_reason` become nullable. +**Consumers that switch on `should_run` must handle `null`**: it means the diff +was not read in full, so no verdict exists. Treating `null` as falsy is safe — +it routes to "do not claim this PR is irrelevant" — but reporting it as "skip" +is not. `next_action.kind` gains `"input_required"`; treat unrecognized kinds as +"no command is authorized". + +Before this change, a shallow clone with no reachable merge base and a partial +clone with unfetched blobs both surfaced as one message, and the trigger then +evaluated the empty inputs those failures left behind and reported +`skip_reason: "no_match"` — "nothing in this PR signals a tool-surface change" — +about a PR the verifier never read. On a workspace without `shipgate.yaml` the +failure was not surfaced at all: preview routed to "Shipgate is not configured +in this workspace". Both are fixed, and a diff whose body cannot be read now +keeps the changed paths that were collected successfully instead of discarding +them. + +--- + ## Migration Note: 0.16.0b7 @@ -595,6 +631,9 @@ Stable JSON fields: control contract vocabulary. - `verifier_schema_version` — schema version for `agents-shipgate-reports/verifier.json`. +- `trigger_catalog_schema_version` — schema version of the published trigger + catalog (`docs/triggers.json`) and, with it, of the run/skip verdict the + evaluator emits. - `verify_run_schema_version` — schema version for `agents-shipgate-reports/verify-run.json`. - `human_authorization_request_schema_version`, @@ -1300,17 +1339,38 @@ release decision. That action may be `detect`/`initialize` for relevant unconfigured repos, or `verify` for configured repos. Use it as the first touch on a repo or PR before committing to a full scan. -`verifier.json` is governed by [`docs/verifier-schema.v0.6.json`](docs/verifier-schema.v0.6.json). -Verifier v0.1 through v0.5 remain frozen references. It remains an orchestration artifact: `release_decision.decision` in +`verifier.json` is governed by [`docs/verifier-schema.v0.7.json`](docs/verifier-schema.v0.7.json). +Verifier v0.1 through v0.6 remain frozen references. It remains an orchestration artifact: `release_decision.decision` in `report.json` is still the only release gate. Release and merge fields remain mirrors or deterministic projections of report data; the v0.6 authorization -evaluation is an operational overlay that cannot change them. Stable additive +evaluation and the v0.7 `diff_status` block are operational overlays that +cannot change them. Stable additive fields a consumer may read: - `control` — the schema-enforced `complete | agent_action_required | human_review_required` operational projection. The same serialized object is emitted by verifier, handoff, and verify-run. - `execution` — `"not_run" | "succeeded" | "skipped" | "failed"`. +- `diff_status` (v0.7+) — how completely the compared change set was read, and + why not when it was not. `completeness` is `"complete" | "partial" | + "unavailable"`; `reason` is `null` exactly when `completeness` is + `"complete"`, and otherwise one of `not_attempted`, `refs_missing`, + `merge_base_missing`, `unrelated_histories`, `objects_missing`, + `metadata_limit_exceeded`, `body_limit_exceeded`, `git_timeout`, + `git_failed`. `merge_base_missing` and `unrelated_histories` are + deliberately distinct: the first is a shallow checkout that truncated a + merge base which does exist, and deepening restores it; the second is two + roots with no common ancestor, which no fetch can create — `fetch_repairable` + is the field to branch on. `detail` is a bounded, path-redacted excerpt of + Git's own diagnostic; `remediation` names the repair; `fetch_repairable` + says whether making refs or objects available locally can fix it. + **`"complete"` is the only value that licenses reading a negative `trigger` + result.** Anything else means the evidence the verdict would rest on was + missing — it is never evidence that a PR is unrelated to agent capabilities. + `null` means the artifact predates v0.7 and carries no input-health + evidence, which a consumer must treat as unknown, never as complete. New + `reason` values may be added additively; treat an unrecognized reason as + "the diff was not read in full". - `static_analysis_only`, `runtime_behavior_verified`, and `static_verdict_disclaimer` — locked to `true`, `false`, and the canonical static-only disclaimer. When an embedded release decision is present, the @@ -1357,7 +1417,17 @@ fields a consumer may read: context, not as the controller's primary verdict. - `mode` — `"advisory"` / `"strict"` / `"skipped"` / `"preview"`. -`verifier.json` also carries `trigger` (the run/skip evaluation), `base_status`, +`verifier.json` also carries `trigger` — the run/skip evaluation, catalog +schema `0.3`. Read `trigger.evaluation_status` before `trigger.should_run`: +when it is `"not_evaluated"`, `should_run`, `run_shipgate`, `skip`, and +`skip_reason` are all `null` because the diff was not read in full (see +`diff_status`), and `next_action.kind` is `"input_required"`. `skip_reason` is +one of `stop_conditions`, `skip_rule`, `dry_run_only`, `no_match` — and +`no_match` is never emitted for inputs that were not fully read. A `run` +verdict *is* still published from partial evidence: rule matching is monotone, +so more evidence can only add matches. `matched_rules` says what carried it — +a `force_run` match rests on the manifest being present, not on anything the +diff showed. It also carries `base_status`, `head_status`, `base_ref`, `head_ref`, `changed_files`, `base_notes`, the full embedded `release_decision`, and an `artifacts` map (`{verifier_json, pr_comment, report_json, report_markdown, report_sarif, diff --git a/adoption-kits/claude-code-skill/.agents-shipgate-kit-metadata.json b/adoption-kits/claude-code-skill/.agents-shipgate-kit-metadata.json index 6615d248..bfc73f6a 100644 --- a/adoption-kits/claude-code-skill/.agents-shipgate-kit-metadata.json +++ b/adoption-kits/claude-code-skill/.agents-shipgate-kit-metadata.json @@ -36,7 +36,8 @@ "e45f9d385f0e7744a5731694f337952682e1849e97be2d0a488ca3cff9db5792", "98ba22d7518ae4635ed109fd187323da0541281061dd4f259ac7fdb950c7b185", "02e780f5a1506d948e4c1d77f6ee4c6b4193227a4fd2ced081847d1fb2e5fbd0", - "bc5cd31a5c4d4f6a1ebf6a04db3f80480e7cc5f9ab2b7a6f7e3f62e8ddfc3937" + "bc5cd31a5c4d4f6a1ebf6a04db3f80480e7cc5f9ab2b7a6f7e3f62e8ddfc3937", + "58ea3b6bba89078ec54d6b5493ffebf9250d9619fbacef5090285b009e58cdcd" ], "prompts/add-shipgate-to-repo.md": [ "ea3c37cfbbd42c40d164abfe21d468a3a5550d5384125f94a53c947dea6b4b2a", @@ -83,7 +84,8 @@ "8d1540095101cd7ff3aec4ba998ced5c135cdbdb71637ad0c4e5d42fc6ec9ab7", "a8ee5f93cab1017c623075c39c1c5bdc639855c37e588e1c9190ab963bb50446", "8f408aed05cb85e06c9f8bb13ee189131eeccfa66fa2c1119e802c43ae97f19c", - "686ab73c76936dee6290716d197c97bf77893534654157dc01274e7aeb32fde7" + "686ab73c76936dee6290716d197c97bf77893534654157dc01274e7aeb32fde7", + "370a81cf1c35212584702ca89c5476f3cd6c19aaaf8b4bb9f57c18476f0d13ef" ] }, "bootstrap_legacy_sha256": { diff --git a/adoption-kits/claude-code-skill/SKILL.md b/adoption-kits/claude-code-skill/SKILL.md index f60c44c0..93c8815e 100644 --- a/adoption-kits/claude-code-skill/SKILL.md +++ b/adoption-kits/claude-code-skill/SKILL.md @@ -74,7 +74,7 @@ For non-GitHub CI (GitLab, CircleCI, Jenkins, Azure Pipelines, Buildkite, Bitbuc - **CLI surface** follows the current 0.x contract line — see https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/STABILITY.md. - **Installed CLI contract**: when available, run `agents-shipgate contract --json` to verify local schema versions, capability/research surfaces, `release_decision.decision`, and manual-review signal fields. Older installs should use [`docs/agent-contract-current.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md) or upgrade before automating against the local contract command. -- **Verifier JSON**: `verifier_schema_version: "0.6"`. Switch on `control.state`, then read `merge_verdict`, `can_merge_without_human`, `control.next_action`, `fix_task`, `capability_review.top_changes`, `trust_root_touched`, and `policy_weakened` before summarizing an AI-generated PR. `merge_verdict` is a deterministic projection; the gate remains `report.json.release_decision.decision`. +- **Verifier JSON**: `verifier_schema_version: "0.7"`. Switch on `control.state`, then read `merge_verdict`, `can_merge_without_human`, `control.next_action`, `fix_task`, `capability_review.top_changes`, `trust_root_touched`, and `policy_weakened` before summarizing an AI-generated PR. `merge_verdict` is a deterministic projection; the gate remains `report.json.release_decision.decision`. Check `diff_status.completeness` before you believe any negative result: only `"complete"` means the PR diff was actually read. Anything else (`reason` is one of `not_attempted`, `refs_missing`, `merge_base_missing`, `unrelated_histories`, `objects_missing`, `metadata_limit_exceeded`, `body_limit_exceeded`, `git_timeout`, `git_failed`) means evidence was missing: follow `remediation`, and never report the PR as unrelated to agent capabilities. Then read `trigger.evaluation_status` for what that cost. `"not_evaluated"` (with `trigger.should_run` `null`) means no verdict exists. `"evaluated"` on an incomplete diff is not a contradiction — evidence that did not depend on the missing bytes already proved Shipgate should run — so honor `should_run: true` instead of overriding it, and still recover the diff before trusting a merge verdict. That evidence is either a rule matched on the change set or, in an already-adopted repository, `force_run: true` from the manifest alone; check `matched_rules` before attributing the verdict to anything the diff showed. - **Verification receipt**: `verification-receipt.json` uses `schema_version: "shipgate.verification_receipt/v1"` and is written last. Validate it before trusting any projected verdict; it content-addresses the request, executor, unit result, decision, and complete artifact set. - **Verify run JSON**: `verify-run.json` uses `schema_version: "shipgate.verify_run/v3"`, embeds the content-addressed plan and executor, and binds unit-result and decision IDs. `run_id` is an exact compatibility alias of `request_id`; do not treat the run projection as a second gate. - **Report JSON**: `report_schema_version: "0.34"`. Read `release_decision.decision` first. A `passed` decision requires a complete root-reachable static binding graph plus complete, conflict-free identity, effect, and authority evidence for every reachable action; it does not prove runtime behavior. Preserve `release_decision.static_analysis_only=true`, `runtime_behavior_verified=false`, and `static_verdict_disclaimer` in summaries. Read `release_decision.evidence_coverage.binding_coverage`, `semantic_coverage`, `identity_coverage`, and `policy_gap_count`, then work every `evidence_gaps[].next_action` in order. Binding, semantic, and policy-applicability gaps are not Findings and cannot be suppressed, baselined, severity-overridden, cleared by `--no-heuristics`, or satisfied by `human_ack`; binding, effect, and authority declarations are human assertions and must never be auto-written. Use `tool_catalog[]` for diagnostics and `tool_inventory[]` for the proven reachable surface. The current schema is [`docs/report-schema.v0.34.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/report-schema.v0.34.json); v0.33 is a frozen compatibility reference. See the [current agent contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md), [verification identity contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/verification-reproducibility.md), and [evidence-backed passed contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/passed-verdict-contract.md). diff --git a/adoption-kits/claude-code-skill/prompts/decide-shipgate-relevance.md b/adoption-kits/claude-code-skill/prompts/decide-shipgate-relevance.md index 5087607f..43e0dac1 100644 --- a/adoption-kits/claude-code-skill/prompts/decide-shipgate-relevance.md +++ b/adoption-kits/claude-code-skill/prompts/decide-shipgate-relevance.md @@ -22,7 +22,7 @@ the rules to the changed file list. - **Local repo** (already adopted Shipgate): read `docs/triggers.json` directly. - **Remote** (target repo without Shipgate): fetch `https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/triggers.json`. - - The catalog has `schema_version: "0.2"`; match `surface_class` instead of maintaining a parallel path list. + - The catalog has `schema_version: "0.3"`; match `surface_class` instead of maintaining a parallel path list. 3. **Apply the rules.** Two equivalent options: @@ -52,7 +52,18 @@ the rules to the changed file list. manually). If your repo already has a manifest, also pass `--manifest-present` so the `force_run` rule can fire. The output shape is `{run_shipgate, dry_run_recommended, - matched_rules, stop_conditions_fired, rationale, schema_version}`. + matched_rules, stop_conditions_fired, rationale, schema_version, + input_status, evaluation_status}`. + When `evaluation_status` is `not_evaluated`, `run_shipgate` is `null`: + the diff could not be read, so there is no verdict. Report the missing + input and stop — never treat it as "this PR is not agent-related". + An `input_status` other than `complete` alongside + `evaluation_status: evaluated` is not a contradiction: evidence that did + not depend on the missing bytes already proved Shipgate should run — a + rule matched on the change set, or `force_run` from a manifest that is + present regardless of the diff. Honor that verdict, read `matched_rules` + before saying what established it, and say the diff still needs + recovering. 4. **Emit the decision.** Always reply in this exact JSON shape so downstream automation can parse you: diff --git a/docs/INDEX.md b/docs/INDEX.md index 1dad0257..73a5e110 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -48,7 +48,8 @@ A single entry point for human readers and AI agents walking the `docs/` tree. - [`report-schema.v0.27.json`](report-schema.v0.27.json) — frozen v0.27 reference schema; pre-v0.28 reports validate against this - [`report-schema.v0.26.json`](report-schema.v0.26.json) — frozen v0.26 reference schema; pre-v0.27 reports validate against this - [`report-schema.v0.25.json`](report-schema.v0.25.json) — frozen v0.25 reference schema; pre-v0.26 reports validate against this -- [`verifier-schema.v0.6.json`](verifier-schema.v0.6.json) — current JSON Schema for `verifier.json`, including the fail-closed signed authorization evaluation +- [`verifier-schema.v0.7.json`](verifier-schema.v0.7.json) — current JSON Schema for `verifier.json`, including the fail-closed signed authorization evaluation and the `diff_status` input-health block +- [`verifier-schema.v0.6.json`](verifier-schema.v0.6.json) — frozen v0.6 reference schema - [`verifier-schema.v0.5.json`](verifier-schema.v0.5.json) — frozen v0.5 verifier reference - [`verifier-schema.v0.4.json`](verifier-schema.v0.4.json) — frozen v0.4 verifier reference - [`verifier-schema.v0.3.json`](verifier-schema.v0.3.json) — frozen v0.3 verifier reference diff --git a/docs/agent-contract-current.md b/docs/agent-contract-current.md index ecc0574b..86d6cd60 100644 --- a/docs/agent-contract-current.md +++ b/docs/agent-contract-current.md @@ -77,7 +77,7 @@ Downstream repos generated with - Current report schema: `0.34` — [`docs/report-schema.v0.34.json`](report-schema.v0.34.json) - Current packet schema: `0.12` — [`docs/packet-schema.v0.12.json`](packet-schema.v0.12.json) - Current shared agent result schema: `agent_result_v2` — [`docs/agent-result-schema.v2.json`](agent-result-schema.v2.json) -- Current verifier schema: `0.6` — [`docs/verifier-schema.v0.6.json`](verifier-schema.v0.6.json) +- Current verifier schema: `0.7` — [`docs/verifier-schema.v0.7.json`](verifier-schema.v0.7.json) - Current verify-run schema: `shipgate.verify_run/v3` — [`docs/verify-run-schema.v3.json`](verify-run-schema.v3.json) - Current verification identity schemas: [`plan v1`](verification-plan-schema.v1.json), [`unit result v1`](verification-unit-result-schema.v1.json), [`artifact manifest v1`](verification-artifact-manifest-schema.v1.json), and [`terminal receipt v1`](verification-receipt-schema.v1.json) - Current human-authorization schemas: request, signed grant, verifier evaluation, and external trust policy v1 — [`docs/human-authorization-schema.v1.json`](human-authorization-schema.v1.json) @@ -93,7 +93,7 @@ Downstream repos generated with - Current registry schema: `0.4` — [`docs/registry-schema.v0.4.json`](registry-schema.v0.4.json) - Current org evidence bundle schema: `shipgate.org_evidence_bundle/v2` — [`docs/org-evidence-bundle-schema.v2.json`](org-evidence-bundle-schema.v2.json) - Current host-grants inventory, baseline, and drift schemas: `0.2` — [`inventory`](host-grants-inventory-schema.v0.2.json), [`baseline`](host-grants-baseline-schema.v0.2.json), [`drift`](host-grants-drift-schema.v0.2.json) -- Current trigger catalog schema: `0.2` — [`docs/triggers.json`](triggers.json) +- Current trigger catalog schema: `0.3` — [`docs/triggers.json`](triggers.json) - Current governance benchmark catalog schema: `0.2` — [`docs/governance-benchmark-catalog-schema.v0.2.json`](governance-benchmark-catalog-schema.v0.2.json) - Current governance benchmark result schema: `0.2` — [`docs/governance-benchmark-result-schema.v0.2.json`](governance-benchmark-result-schema.v0.2.json) - Frozen-reference report schemas: frozen [`v0.33`](report-schema.v0.33.json), frozen [`v0.32`](report-schema.v0.32.json), frozen [`v0.31`](report-schema.v0.31.json), frozen [`v0.30`](report-schema.v0.30.json), and older versions listed in [`docs/INDEX.md`](INDEX.md#reference) @@ -331,8 +331,8 @@ from existing artifacts with: agents-shipgate agent handoff --from agents-shipgate-reports/verifier.json --json ``` -In `agents-shipgate-reports/verifier.json`, read the v0.6 fields below (full -schema [`docs/verifier-schema.v0.6.json`](verifier-schema.v0.6.json)). **Lead +In `agents-shipgate-reports/verifier.json`, read the v0.7 fields below (full +schema [`docs/verifier-schema.v0.7.json`](verifier-schema.v0.7.json)). **Lead with `control.state`.** Every release and merge field below is a mirror or deterministic projection of `report.json`; the authorization evaluation is an operational overlay and cannot change those fields. @@ -344,6 +344,27 @@ operational overlay and cannot change those fields. generated schemas enforce the variants with `oneOf`. Only a new verifier artifact can clear a pending control obligation. - `execution` — `"not_run" | "succeeded" | "skipped" | "failed"`. +- `diff_status` — whether the compared change set was read at all. + `completeness` is `"complete"` / `"partial"` / `"unavailable"`; `reason` is + `null` only when complete, and otherwise `not_attempted`, `refs_missing`, + `merge_base_missing` (shallow checkout — deepening restores the merge base), + `unrelated_histories` (no common ancestor exists; no fetch can create one), + `objects_missing`, `metadata_limit_exceeded`, `body_limit_exceeded`, + `git_timeout`, or `git_failed`. `remediation` names + the repair and `fetch_repairable` says whether fetching can perform it. + **Only `"complete"` licenses reading a negative `trigger` result**; anything + else means the diff was not read, which is never evidence that a PR is + unrelated to agent capabilities. `null` means a pre-v0.7 artifact — unknown, + not complete. +- `trigger` — the run/skip evaluation. Read `evaluation_status` first: when it + is `"not_evaluated"`, `should_run` / `run_shipgate` / `skip` / `skip_reason` + are `null` and `next_action.kind` is `"input_required"`. `skip_reason` is + never `"no_match"` for inputs that were not fully read. `"evaluated"` on an + incomplete `diff_status` is not a contradiction: only *skip* verdicts are + withheld, so a `should_run: true` reached from evidence that did not depend + on the missing bytes is authoritative and must not be overridden. Read + `matched_rules` to see what carried it — a `force_run` match rests on the + manifest, not on the diff. - `merge_verdict` — `"mergeable"` / `"human_review_required"` / `"insufficient_evidence"` / `"blocked"` / `"unknown"`. Deterministic projection of `release_decision.decision` (`passed`→`mergeable`, diff --git a/docs/triggers.json b/docs/triggers.json index 6c566770..a918e57b 100644 --- a/docs/triggers.json +++ b/docs/triggers.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "schema_version": "0.2", + "schema_version": "0.3", "name": "agents-shipgate-triggers", "description": "Machine-readable trigger catalog for Agents Shipgate. Mirrors the AGENTS.md trigger table — a coding agent can fetch this and apply the rules to a PR diff or repo state to decide whether to run `agents-shipgate verify --preview --json` or the full verifier. Stable for 0.x: rule IDs, predicate vocabulary, and action enum will not change in minor versions.", "source_of_truth": "AGENTS.md#should-i-run-shipgate-on-this-pr", diff --git a/docs/verifier-schema.v0.7.json b/docs/verifier-schema.v0.7.json new file mode 100644 index 00000000..b10ecb6e --- /dev/null +++ b/docs/verifier-schema.v0.7.json @@ -0,0 +1,2814 @@ +{ + "$defs": { + "AgentActionRequiredControl": { + "additionalProperties": false, + "description": "Non-terminal state with one exact coding-agent-owned next step.", + "properties": { + "allowed_next_commands": { + "items": { + "minLength": 1, + "type": "string" + }, + "title": "Allowed Next Commands", + "type": "array" + }, + "completion_allowed": { + "const": false, + "default": false, + "title": "Completion Allowed", + "type": "boolean" + }, + "human_review": { + "$ref": "#/$defs/NoHumanReview" + }, + "must_stop": { + "const": false, + "default": false, + "title": "Must Stop", + "type": "boolean" + }, + "next_action": { + "$ref": "#/$defs/CodingAgentAction" + }, + "reason": { + "minLength": 1, + "title": "Reason", + "type": "string" + }, + "state": { + "const": "agent_action_required", + "title": "State", + "type": "string" + }, + "stop_reason": { + "default": null, + "title": "Stop Reason", + "type": "null" + }, + "verify_required": { + "default": false, + "title": "Verify Required", + "type": "boolean" + } + }, + "required": [ + "state", + "reason", + "completion_allowed", + "must_stop", + "verify_required", + "next_action", + "allowed_next_commands", + "human_review", + "stop_reason" + ], + "title": "AgentActionRequiredControl", + "type": "object" + }, + "AgentControl": { + "discriminator": { + "mapping": { + "agent_action_required": "#/$defs/AgentActionRequiredControl", + "complete": "#/$defs/CompleteAgentControl", + "human_review_required": "#/$defs/HumanReviewRequiredControl" + }, + "propertyName": "state" + }, + "oneOf": [ + { + "$ref": "#/$defs/CompleteAgentControl" + }, + { + "$ref": "#/$defs/AgentActionRequiredControl" + }, + { + "$ref": "#/$defs/HumanReviewRequiredControl" + } + ] + }, + "AuthorizationEvaluationV1": { + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "accepted" + } + } + }, + "then": { + "properties": { + "authorization_id": { + "not": { + "type": "null" + } + }, + "authorization_request_id": { + "not": { + "type": "null" + } + }, + "command": { + "not": { + "type": "null" + } + }, + "expires_at": { + "not": { + "type": "null" + } + }, + "issued_at": { + "not": { + "type": "null" + } + }, + "key_id": { + "not": { + "type": "null" + } + }, + "operation_id": { + "not": { + "type": "null" + } + }, + "principal": { + "not": { + "type": "null" + } + }, + "provider": { + "not": { + "type": "null" + } + }, + "reason_codes": { + "maxItems": 0 + }, + "trust_policy_id": { + "not": { + "type": "null" + } + } + }, + "required": [ + "authorization_id", + "authorization_request_id", + "trust_policy_id", + "key_id", + "provider", + "principal", + "operation_id", + "command", + "issued_at", + "expires_at" + ] + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "rejected", + "not_requested", + "not_applicable" + ] + } + } + }, + "then": { + "properties": { + "command": { + "type": "null" + } + } + } + }, + { + "if": { + "properties": { + "status": { + "const": "rejected" + } + } + }, + "then": { + "properties": { + "reason_codes": { + "minItems": 1 + } + }, + "required": [ + "reason_codes" + ] + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "not_requested", + "not_applicable" + ] + } + } + }, + "then": { + "properties": { + "authorization_id": { + "type": "null" + }, + "authorization_request_id": { + "type": "null" + }, + "command": { + "type": "null" + }, + "expires_at": { + "type": "null" + }, + "issued_at": { + "type": "null" + }, + "key_id": { + "type": "null" + }, + "operation_id": { + "type": "null" + }, + "principal": { + "type": "null" + }, + "provider": { + "type": "null" + }, + "trust_policy_id": { + "type": "null" + } + } + } + } + ], + "description": "Fail-closed authorization evaluation consumed by verifier projections.", + "properties": { + "authorization_id": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Authorization Id" + }, + "authorization_request_id": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Authorization Request Id" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Command" + }, + "expires_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expires At" + }, + "issued_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Issued At" + }, + "key_id": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Key Id" + }, + "operation_id": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Operation Id" + }, + "principal": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Principal" + }, + "provider": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provider" + }, + "reason_codes": { + "items": { + "type": "string" + }, + "title": "Reason Codes", + "type": "array", + "uniqueItems": true + }, + "schema_version": { + "const": "shipgate.human_authorization_evaluation/v1", + "default": "shipgate.human_authorization_evaluation/v1", + "title": "Schema Version", + "type": "string" + }, + "status": { + "enum": [ + "not_requested", + "accepted", + "rejected", + "not_applicable" + ], + "title": "Status", + "type": "string" + }, + "trust_policy_id": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Trust Policy Id" + } + }, + "required": [ + "status" + ], + "title": "AuthorizationEvaluationV1", + "type": "object" + }, + "BaselineDelta": { + "properties": { + "enabled": { + "title": "Enabled", + "type": "boolean" + }, + "matched_count": { + "default": 0, + "title": "Matched Count", + "type": "integer" + }, + "new_count": { + "default": 0, + "title": "New Count", + "type": "integer" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "resolved_count": { + "default": 0, + "title": "Resolved Count", + "type": "integer" + } + }, + "required": [ + "enabled" + ], + "title": "BaselineDelta", + "type": "object" + }, + "BindingCoverageDecision": { + "properties": { + "gap_count": { + "default": 0, + "title": "Gap Count", + "type": "integer" + }, + "pass_eligible": { + "default": false, + "title": "Pass Eligible", + "type": "boolean" + }, + "possible_tools": { + "default": 0, + "title": "Possible Tools", + "type": "integer" + }, + "reachable_tools": { + "default": 0, + "title": "Reachable Tools", + "type": "integer" + }, + "reason_counts": { + "additionalProperties": { + "type": "integer" + }, + "title": "Reason Counts", + "type": "object" + }, + "total_catalog_tools": { + "default": 0, + "title": "Total Catalog Tools", + "type": "integer" + }, + "unbound_tools": { + "default": 0, + "title": "Unbound Tools", + "type": "integer" + } + }, + "title": "BindingCoverageDecision", + "type": "object" + }, + "CodingAgentAction": { + "discriminator": { + "mapping": { + "configure": "#/$defs/CodingAgentCommandAction", + "discover": "#/$defs/CodingAgentCommandAction", + "fetch_base": "#/$defs/CodingAgentFetchBaseAction", + "initialize": "#/$defs/CodingAgentCommandAction", + "install": "#/$defs/CodingAgentCommandAction", + "repair": "#/$defs/CodingAgentCommandAction", + "rerun": "#/$defs/CodingAgentCommandAction", + "verify": "#/$defs/CodingAgentCommandAction" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/CodingAgentCommandAction" + }, + { + "$ref": "#/$defs/CodingAgentFetchBaseAction" + } + ] + }, + "CodingAgentCommandAction": { + "additionalProperties": false, + "description": "An executable, exact next step owned by the coding agent.", + "properties": { + "actor": { + "const": "coding_agent", + "default": "coding_agent", + "title": "Actor", + "type": "string" + }, + "command": { + "minLength": 1, + "title": "Command", + "type": "string" + }, + "expects": { + "default": null, + "title": "Expects", + "type": "null" + }, + "kind": { + "enum": [ + "verify", + "discover", + "configure", + "initialize", + "repair", + "install", + "rerun" + ], + "title": "Kind", + "type": "string" + }, + "why": { + "minLength": 1, + "title": "Why", + "type": "string" + } + }, + "required": [ + "actor", + "kind", + "command", + "expects", + "why" + ], + "title": "CodingAgentCommandAction", + "type": "object" + }, + "CodingAgentFetchBaseAction": { + "additionalProperties": false, + "description": "A structured input request when an exact fetch command is unavailable.\n\nShipgate never fetches refs itself. ``expects`` therefore names the exact\nref or artifact a caller must make available before rerunning verification.", + "properties": { + "actor": { + "const": "coding_agent", + "default": "coding_agent", + "title": "Actor", + "type": "string" + }, + "command": { + "default": null, + "title": "Command", + "type": "null" + }, + "expects": { + "minLength": 1, + "title": "Expects", + "type": "string" + }, + "kind": { + "const": "fetch_base", + "title": "Kind", + "type": "string" + }, + "why": { + "minLength": 1, + "title": "Why", + "type": "string" + } + }, + "required": [ + "actor", + "kind", + "command", + "expects", + "why" + ], + "title": "CodingAgentFetchBaseAction", + "type": "object" + }, + "CompleteAgentControl": { + "additionalProperties": false, + "description": "Terminal state: the coding agent may report the task complete.", + "properties": { + "allowed_next_commands": { + "items": { + "minLength": 1, + "type": "string" + }, + "maxItems": 0, + "title": "Allowed Next Commands", + "type": "array" + }, + "completion_allowed": { + "const": true, + "default": true, + "title": "Completion Allowed", + "type": "boolean" + }, + "human_review": { + "$ref": "#/$defs/NoHumanReview" + }, + "must_stop": { + "const": false, + "default": false, + "title": "Must Stop", + "type": "boolean" + }, + "next_action": { + "default": null, + "title": "Next Action", + "type": "null" + }, + "reason": { + "minLength": 1, + "title": "Reason", + "type": "string" + }, + "state": { + "const": "complete", + "title": "State", + "type": "string" + }, + "stop_reason": { + "default": null, + "title": "Stop Reason", + "type": "null" + }, + "verify_required": { + "const": false, + "default": false, + "title": "Verify Required", + "type": "boolean" + } + }, + "required": [ + "state", + "reason", + "completion_allowed", + "must_stop", + "verify_required", + "next_action", + "allowed_next_commands", + "human_review", + "stop_reason" + ], + "title": "CompleteAgentControl", + "type": "object" + }, + "ContributionRule": { + "additionalProperties": false, + "description": "Per-finding audit row explaining how a finding contributed to the\nrelease decision.\n\nAdditive in v0.17. Every finding in `report.findings` produces\nexactly one ContributionRule. Reading the contribution rule is\nsufficient to predict the gate outcome for that finding without\nre-deriving the decision logic; the set of valid `(rule, category)`\npairs is the contract documented in STABILITY.md \"Release decision\ntruth table\".", + "properties": { + "category": { + "enum": [ + "blocker", + "review_item", + "excluded" + ], + "title": "Category", + "type": "string" + }, + "check_id": { + "title": "Check Id", + "type": "string" + }, + "finding_id": { + "title": "Finding Id", + "type": "string" + }, + "fingerprint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Fingerprint" + }, + "rationale": { + "title": "Rationale", + "type": "string" + }, + "rule": { + "enum": [ + "policy_block_new", + "severity_block_new", + "policy_baseline_accepted", + "severity_baseline_accepted", + "review_required", + "sub_threshold", + "unsupported_evidence", + "suppressed" + ], + "title": "Rule", + "type": "string" + } + }, + "required": [ + "finding_id", + "check_id", + "category", + "rule", + "rationale" + ], + "title": "ContributionRule", + "type": "object" + }, + "EvidenceCoverageDecision": { + "properties": { + "binding_coverage": { + "$ref": "#/$defs/BindingCoverageDecision" + }, + "evidence_gaps": { + "items": { + "$ref": "#/$defs/EvidenceGap" + }, + "title": "Evidence Gaps", + "type": "array" + }, + "human_review_recommended": { + "title": "Human Review Recommended", + "type": "boolean" + }, + "identity_coverage": { + "$ref": "#/$defs/IdentityCoverageDecision" + }, + "level": { + "title": "Level", + "type": "string" + }, + "low_confidence_tool_count": { + "title": "Low Confidence Tool Count", + "type": "integer" + }, + "policy_gap_count": { + "default": 0, + "title": "Policy Gap Count", + "type": "integer" + }, + "semantic_coverage": { + "$ref": "#/$defs/SemanticCoverageDecision" + }, + "source_warning_count": { + "title": "Source Warning Count", + "type": "integer" + } + }, + "required": [ + "level", + "human_review_recommended", + "source_warning_count", + "low_confidence_tool_count" + ], + "title": "EvidenceCoverageDecision", + "type": "object" + }, + "EvidenceGap": { + "description": "v0.26: one structured row per measurable evidence gap.\n\n``insufficient_evidence`` previously diagnosed without prescribing;\neach gap names the degraded subject and the specific next action\nthat raises extraction confidence. Purely explanatory \u2014 gating\nstill uses only the counts (the gap list is a projection of them).", + "properties": { + "kind": { + "enum": [ + "low_confidence_tool", + "source_warning", + "incomplete_surface", + "missing_effect_evidence", + "inferred_effect_only", + "conflicting_effect_evidence", + "missing_authority_evidence", + "partial_authority_evidence", + "conflicting_authority_evidence", + "invalid_semantic_annotation", + "incomplete_tool_identity", + "conflicting_tool_identity", + "unresolved_tool_selector", + "ambiguous_tool_selector", + "ambiguous_legacy_tool_identity", + "invalid_tool_binding", + "missing_binding_evidence", + "partial_binding_evidence", + "conflicting_binding_evidence", + "ambiguous_root_agent", + "unresolved_agent_binding", + "unresolved_bound_tool", + "incomplete_handoff_graph", + "invalid_binding_annotation", + "invalid_evidence_provenance", + "inferred_policy_applicability", + "mixed_policy_evidence", + "unknown_policy_evidence", + "conflicting_policy_evidence" + ], + "title": "Kind", + "type": "string" + }, + "next_action": { + "$ref": "#/$defs/EvidenceGapAction" + }, + "source_ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Ref" + }, + "source_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Type" + }, + "subject": { + "title": "Subject", + "type": "string" + }, + "why": { + "title": "Why", + "type": "string" + } + }, + "required": [ + "kind", + "subject", + "why", + "next_action" + ], + "title": "EvidenceGap", + "type": "object" + }, + "EvidenceGapAction": { + "description": "One concrete, mechanically-executable step that closes a gap.\n\nMirrors the agent-mode ``next_actions[]`` error shape\n(``kind``/``command``/``path``/``why``/``expects``) so agents reuse\none routing vocabulary across error recovery and evidence repair.", + "properties": { + "accepted_values": { + "items": { + "type": "string" + }, + "title": "Accepted Values", + "type": "array" + }, + "auto_apply": { + "const": false, + "default": false, + "title": "Auto Apply", + "type": "boolean" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Command" + }, + "declaration_template": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Declaration Template" + }, + "expects": { + "title": "Expects", + "type": "string" + }, + "kind": { + "enum": [ + "declare_tool_inventory", + "provide_source", + "review_warning", + "declare_action_effect", + "declare_action_authority", + "provide_complete_inventory", + "resolve_semantic_conflict", + "declare_source_identity", + "qualify_tool_selector", + "provide_tool_binding", + "resolve_tool_identity_conflict", + "regenerate_identity_artifact", + "declare_agent_root", + "declare_agent_bindings", + "provide_static_binding_source", + "provide_complete_binding_graph", + "resolve_binding_conflict", + "regenerate_binding_artifact", + "provide_policy_evidence", + "review_policy_evidence", + "resolve_policy_evidence_conflict" + ], + "title": "Kind", + "type": "string" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "requires_human_review": { + "const": true, + "default": true, + "title": "Requires Human Review", + "type": "boolean" + }, + "suggested_patch_kind": { + "const": "manual", + "default": "manual", + "title": "Suggested Patch Kind", + "type": "string" + }, + "why": { + "title": "Why", + "type": "string" + } + }, + "required": [ + "kind", + "why", + "expects" + ], + "title": "EvidenceGapAction", + "type": "object" + }, + "FailPolicy": { + "properties": { + "ci_mode": { + "title": "Ci Mode", + "type": "string" + }, + "exit_code": { + "title": "Exit Code", + "type": "integer" + }, + "fail_on": { + "items": { + "enum": [ + "info", + "low", + "medium", + "high", + "critical" + ], + "type": "string" + }, + "title": "Fail On", + "type": "array" + }, + "new_findings_only": { + "default": false, + "title": "New Findings Only", + "type": "boolean" + }, + "would_fail_ci": { + "title": "Would Fail Ci", + "type": "boolean" + } + }, + "required": [ + "ci_mode", + "would_fail_ci", + "exit_code" + ], + "title": "FailPolicy", + "type": "object" + }, + "FindingSupport": { + "additionalProperties": false, + "description": "Authoritative support for finding confidence and release contribution.\n\nRule metadata may request a severity or block, but it cannot upgrade the\nunderlying evidence. ``support_hash`` binds baselines and audit surfaces\nto the predicate evidence that actually made the finding eligible.", + "properties": { + "blocking_eligible": { + "default": false, + "title": "Blocking Eligible", + "type": "boolean" + }, + "claim_ids": { + "items": { + "type": "string" + }, + "title": "Claim Ids", + "type": "array" + }, + "confidence": { + "default": "low", + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "evidence_bases": { + "items": { + "enum": [ + "reviewed_declaration", + "protocol_structure", + "typed_provider_fact", + "structural_scope", + "inferred_keyword", + "inferred_regex", + "protocol_default", + "unknown" + ], + "type": "string" + }, + "title": "Evidence Bases", + "type": "array" + }, + "policy_eligible": { + "default": false, + "title": "Policy Eligible", + "type": "boolean" + }, + "predicates": { + "items": { + "$ref": "#/$defs/PolicyPredicateEvidence" + }, + "title": "Predicates", + "type": "array" + }, + "status": { + "default": "matched", + "enum": [ + "matched", + "not_matched", + "indeterminate", + "conflicting" + ], + "title": "Status", + "type": "string" + }, + "support_hash": { + "title": "Support Hash", + "type": "string" + } + }, + "required": [ + "support_hash" + ], + "title": "FindingSupport", + "type": "object" + }, + "HumanControlAction": { + "additionalProperties": false, + "description": "A human-owned route. Human actions never expose executable commands.", + "properties": { + "actor": { + "const": "human", + "default": "human", + "title": "Actor", + "type": "string" + }, + "command": { + "default": null, + "title": "Command", + "type": "null" + }, + "expects": { + "default": null, + "title": "Expects", + "type": "null" + }, + "kind": { + "enum": [ + "review", + "stop" + ], + "title": "Kind", + "type": "string" + }, + "why": { + "minLength": 1, + "title": "Why", + "type": "string" + } + }, + "required": [ + "actor", + "kind", + "command", + "expects", + "why" + ], + "title": "HumanControlAction", + "type": "object" + }, + "HumanReviewRequiredControl": { + "additionalProperties": false, + "description": "Stopping state: no further coding-agent action is authorized.", + "properties": { + "allowed_next_commands": { + "items": { + "minLength": 1, + "type": "string" + }, + "maxItems": 0, + "title": "Allowed Next Commands", + "type": "array" + }, + "completion_allowed": { + "const": false, + "default": false, + "title": "Completion Allowed", + "type": "boolean" + }, + "human_review": { + "$ref": "#/$defs/RequiredHumanReview" + }, + "must_stop": { + "const": true, + "default": true, + "title": "Must Stop", + "type": "boolean" + }, + "next_action": { + "$ref": "#/$defs/HumanControlAction" + }, + "reason": { + "minLength": 1, + "title": "Reason", + "type": "string" + }, + "state": { + "const": "human_review_required", + "title": "State", + "type": "string" + }, + "stop_reason": { + "minLength": 1, + "title": "Stop Reason", + "type": "string" + }, + "verify_required": { + "default": false, + "title": "Verify Required", + "type": "boolean" + } + }, + "required": [ + "state", + "reason", + "completion_allowed", + "must_stop", + "verify_required", + "next_action", + "allowed_next_commands", + "human_review", + "stop_reason" + ], + "title": "HumanReviewRequiredControl", + "type": "object" + }, + "IdentityCoverageDecision": { + "properties": { + "ambiguous_name_count": { + "default": 0, + "title": "Ambiguous Name Count", + "type": "integer" + }, + "bound_tools": { + "default": 0, + "title": "Bound Tools", + "type": "integer" + }, + "canonical_tools": { + "default": 0, + "title": "Canonical Tools", + "type": "integer" + }, + "gap_count": { + "default": 0, + "title": "Gap Count", + "type": "integer" + }, + "pass_eligible_tools": { + "default": 0, + "title": "Pass Eligible Tools", + "type": "integer" + }, + "reason_counts": { + "additionalProperties": { + "type": "integer" + }, + "title": "Reason Counts", + "type": "object" + }, + "total_observations": { + "default": 0, + "title": "Total Observations", + "type": "integer" + } + }, + "title": "IdentityCoverageDecision", + "type": "object" + }, + "NoHumanReview": { + "additionalProperties": false, + "description": "Exact negative human-review projection for non-stopping states.", + "properties": { + "required": { + "const": false, + "default": false, + "title": "Required", + "type": "boolean" + }, + "required_reviewers": { + "items": { + "type": "string" + }, + "maxItems": 0, + "title": "Required Reviewers", + "type": "array" + }, + "why": { + "default": null, + "title": "Why", + "type": "null" + } + }, + "required": [ + "required", + "why", + "required_reviewers" + ], + "title": "NoHumanReview", + "type": "object" + }, + "PolicyPredicateEvidence": { + "additionalProperties": false, + "description": "One tri-state policy predicate and the evidence that supports it.", + "properties": { + "claim_ids": { + "items": { + "type": "string" + }, + "title": "Claim Ids", + "type": "array" + }, + "confidence": { + "default": "low", + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "evidence_bases": { + "items": { + "enum": [ + "reviewed_declaration", + "protocol_structure", + "typed_provider_fact", + "structural_scope", + "inferred_keyword", + "inferred_regex", + "protocol_default", + "unknown" + ], + "type": "string" + }, + "title": "Evidence Bases", + "type": "array" + }, + "expected": { + "default": null, + "title": "Expected" + }, + "observed": { + "default": null, + "title": "Observed" + }, + "policy_eligible": { + "default": false, + "title": "Policy Eligible", + "type": "boolean" + }, + "predicate": { + "title": "Predicate", + "type": "string" + }, + "status": { + "enum": [ + "matched", + "not_matched", + "indeterminate", + "conflicting" + ], + "title": "Status", + "type": "string" + }, + "why": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Why" + } + }, + "required": [ + "predicate", + "status" + ], + "title": "PolicyPredicateEvidence", + "type": "object" + }, + "ReleaseDecision": { + "properties": { + "baseline_delta": { + "$ref": "#/$defs/BaselineDelta" + }, + "blockers": { + "items": { + "$ref": "#/$defs/ReleaseDecisionItem" + }, + "title": "Blockers", + "type": "array" + }, + "contribution_rules": { + "items": { + "$ref": "#/$defs/ContributionRule" + }, + "title": "Contribution Rules", + "type": "array" + }, + "decision": { + "enum": [ + "blocked", + "review_required", + "insufficient_evidence", + "passed" + ], + "title": "Decision", + "type": "string" + }, + "evidence_coverage": { + "$ref": "#/$defs/EvidenceCoverageDecision" + }, + "fail_policy": { + "$ref": "#/$defs/FailPolicy" + }, + "reason": { + "title": "Reason", + "type": "string" + }, + "review_items": { + "items": { + "$ref": "#/$defs/ReleaseDecisionItem" + }, + "title": "Review Items", + "type": "array" + }, + "runtime_behavior_verified": { + "const": false, + "default": false, + "title": "Runtime Behavior Verified", + "type": "boolean" + }, + "static_analysis_only": { + "const": true, + "default": true, + "title": "Static Analysis Only", + "type": "boolean" + }, + "static_verdict_disclaimer": { + "default": "This verdict covers deterministic static evidence only. Agents Shipgate did not execute the agent or prove runtime behavior, tool routing, credential enforcement, or safety.", + "title": "Static Verdict Disclaimer", + "type": "string" + } + }, + "required": [ + "decision", + "reason", + "evidence_coverage", + "baseline_delta", + "fail_policy" + ], + "title": "ReleaseDecision", + "type": "object" + }, + "ReleaseDecisionItem": { + "properties": { + "baseline_status": { + "anyOf": [ + { + "enum": [ + "new", + "matched", + "resolved" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Baseline Status" + }, + "blocks_release": { + "default": false, + "title": "Blocks Release", + "type": "boolean" + }, + "capability_refs": { + "items": { + "type": "string" + }, + "title": "Capability Refs", + "type": "array" + }, + "capability_trace_refs": { + "items": { + "type": "string" + }, + "title": "Capability Trace Refs", + "type": "array" + }, + "check_id": { + "title": "Check Id", + "type": "string" + }, + "fingerprint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Fingerprint" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Id" + }, + "policy_evidence_source": { + "anyOf": [ + { + "$ref": "#/$defs/SourceReference" + }, + { + "type": "null" + } + ], + "default": null + }, + "severity": { + "enum": [ + "info", + "low", + "medium", + "high", + "critical" + ], + "title": "Severity", + "type": "string" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/SourceReference" + }, + { + "type": "null" + } + ], + "default": null + }, + "support": { + "anyOf": [ + { + "$ref": "#/$defs/FindingSupport" + }, + { + "type": "null" + } + ], + "default": null + }, + "title": { + "title": "Title", + "type": "string" + } + }, + "required": [ + "check_id", + "severity", + "title" + ], + "title": "ReleaseDecisionItem", + "type": "object" + }, + "RequiredHumanReview": { + "additionalProperties": false, + "description": "Human-review evidence carried by the stopping state.", + "properties": { + "required": { + "const": true, + "default": true, + "title": "Required", + "type": "boolean" + }, + "required_reviewers": { + "items": { + "minLength": 1, + "type": "string" + }, + "title": "Required Reviewers", + "type": "array" + }, + "why": { + "minLength": 1, + "title": "Why", + "type": "string" + } + }, + "required": [ + "required", + "why", + "required_reviewers" + ], + "title": "RequiredHumanReview", + "type": "object" + }, + "SemanticCoverageDecision": { + "description": "v0.29 pass eligibility across the normalized action surface.\n\nUnlike extraction-confidence thresholds, semantic gaps are\nzero-tolerance: any non-pass-eligible unknown/partial/conflicting\ndimension prevents ``passed``. Known authority review concerns (for\nexample ambient or unscoped credentials) are counted separately so\nthey deterministically route to ``review_required`` rather than\n``insufficient_evidence``.", + "properties": { + "gap_count": { + "default": 0, + "title": "Gap Count", + "type": "integer" + }, + "pass_eligible_actions": { + "default": 0, + "title": "Pass Eligible Actions", + "type": "integer" + }, + "reason_counts": { + "additionalProperties": { + "type": "integer" + }, + "title": "Reason Counts", + "type": "object" + }, + "review_concern_count": { + "default": 0, + "title": "Review Concern Count", + "type": "integer" + }, + "total_actions": { + "default": 0, + "title": "Total Actions", + "type": "integer" + } + }, + "title": "SemanticCoverageDecision", + "type": "object" + }, + "SourceReference": { + "additionalProperties": true, + "properties": { + "end_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "End Line" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Location" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "pointer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Pointer" + }, + "ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Ref" + }, + "start_column": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Start Column" + }, + "start_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Start Line" + }, + "type": { + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "SourceReference", + "type": "object" + }, + "VerifierCapabilityChange": { + "additionalProperties": false, + "description": "One reviewer-facing capability change projected for verifier output.", + "properties": { + "change_bucket": { + "enum": [ + "added", + "modified", + "removed" + ], + "title": "Change Bucket", + "type": "string" + }, + "change_type": { + "title": "Change Type", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "impact": { + "default": "informational", + "enum": [ + "blocks_release", + "review_required", + "insufficient_evidence", + "informational", + "none" + ], + "title": "Impact", + "type": "string" + }, + "rationale": { + "title": "Rationale", + "type": "string" + }, + "related_finding_ids": { + "items": { + "type": "string" + }, + "title": "Related Finding Ids", + "type": "array" + }, + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Path" + }, + "source_start_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Start Line" + }, + "subject": { + "title": "Subject", + "type": "string" + }, + "subject_kind": { + "title": "Subject Kind", + "type": "string" + } + }, + "required": [ + "id", + "change_type", + "change_bucket", + "subject_kind", + "subject", + "rationale" + ], + "title": "VerifierCapabilityChange", + "type": "object" + }, + "VerifierCapabilityReview": { + "additionalProperties": false, + "description": "Derived capability-review rollup for PR comments and Action outputs.\n\nThis is a projection only. It never gates independently of\n``report.json.release_decision.decision``.", + "properties": { + "added": { + "default": 0, + "title": "Added", + "type": "integer" + }, + "modified": { + "default": 0, + "title": "Modified", + "type": "integer" + }, + "notes": { + "items": { + "type": "string" + }, + "title": "Notes", + "type": "array" + }, + "policy_weakened": { + "default": false, + "title": "Policy Weakened", + "type": "boolean" + }, + "removed": { + "default": 0, + "title": "Removed", + "type": "integer" + }, + "top_changes": { + "items": { + "$ref": "#/$defs/VerifierCapabilityChange" + }, + "title": "Top Changes", + "type": "array" + }, + "trust_root_touched": { + "default": false, + "title": "Trust Root Touched", + "type": "boolean" + } + }, + "title": "VerifierCapabilityReview", + "type": "object" + }, + "VerifierDiffStatus": { + "additionalProperties": false, + "description": "Whether the compared change set was actually read, and why not.\n\nEmitted on every verifier artifact so automation never has to infer input\nhealth from a verdict. ``completeness: \"complete\"`` is the only value that\nlicenses reading a negative trigger result \u2014 anything else means the\nevidence the verdict would rest on was missing, and the artifact says so\ninstead of reporting \"nothing in this PR signals a tool-surface change\".", + "properties": { + "completeness": { + "default": "complete", + "enum": [ + "complete", + "partial", + "unavailable", + "unknown" + ], + "title": "Completeness", + "type": "string" + }, + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Detail" + }, + "fetch_repairable": { + "default": false, + "title": "Fetch Repairable", + "type": "boolean" + }, + "reason": { + "anyOf": [ + { + "enum": [ + "not_attempted", + "refs_missing", + "merge_base_missing", + "unrelated_histories", + "objects_missing", + "metadata_limit_exceeded", + "body_limit_exceeded", + "git_timeout", + "git_failed" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Reason" + }, + "remediation": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Remediation" + } + }, + "title": "VerifierDiffStatus", + "type": "object" + }, + "VerifierFixTask": { + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "actor": { + "const": "human" + } + }, + "required": [ + "actor" + ] + }, + "then": { + "properties": { + "safe_to_attempt": { + "const": false + } + } + } + }, + { + "if": { + "properties": { + "actor": { + "const": "coding_agent" + }, + "safe_to_attempt": { + "const": true + } + }, + "required": [ + "actor", + "safe_to_attempt" + ] + }, + "then": { + "properties": { + "verification_command": { + "minLength": 1, + "pattern": "\\S", + "type": "string" + } + }, + "required": [ + "verification_command" + ] + } + } + ], + "description": "The single repair task a verify run hands to whoever acts next.\n\nRouting is deterministic and projected from the head scan \u2014 never an LLM\njudgment. ``coding_agent`` + ``safe_to_attempt=True`` means the gating\ngaps are mechanical (every gating finding is ``autofix_safe``): the agent\nmay fix them and re-run ``verification_command``. ``human`` +\n``safe_to_attempt=False`` means an authority gap a coding agent must not\ninvent its way past \u2014 missing approval/idempotency evidence, a weakened\npolicy, or a touched trust root. ``forbidden_shortcuts`` are the\nreward-hacking moves that are never acceptable for either actor.\n``patches`` (v0.12+) carries the machine-applicable suggested patches for\nthe gating findings when verify ran with ``--suggest-patches`` and the\ntask routes to the coding agent.", + "properties": { + "actor": { + "enum": [ + "coding_agent", + "human" + ], + "title": "Actor", + "type": "string" + }, + "allowed_repairs": { + "items": { + "$ref": "#/$defs/VerifierRepair" + }, + "title": "Allowed Repairs", + "type": "array" + }, + "forbidden_repairs": { + "items": { + "$ref": "#/$defs/VerifierRepair" + }, + "title": "Forbidden Repairs", + "type": "array" + }, + "forbidden_shortcuts": { + "items": { + "type": "string" + }, + "title": "Forbidden Shortcuts", + "type": "array" + }, + "instructions": { + "items": { + "type": "string" + }, + "title": "Instructions", + "type": "array" + }, + "patches": { + "items": { + "$ref": "#/$defs/VerifierFixTaskPatch" + }, + "title": "Patches", + "type": "array" + }, + "safe_to_attempt": { + "title": "Safe To Attempt", + "type": "boolean" + }, + "verification_command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Verification Command" + } + }, + "required": [ + "actor", + "safe_to_attempt" + ], + "title": "VerifierFixTask", + "type": "object" + }, + "VerifierFixTaskPatch": { + "additionalProperties": false, + "description": "A machine-applicable patch projected into the fix task.\n\nRepair aid only \u2014 never a gate input. ``patch`` carries the\ndiscriminated Patch payload (``set_pointer`` / ``append_pointer`` /\n``remove_pointer``) exactly as the head scan emitted it; ``manual``\npatches are intentionally excluded because their guidance already\nappears in ``instructions``.", + "properties": { + "check_id": { + "default": "", + "title": "Check Id", + "type": "string" + }, + "finding_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Finding Id" + }, + "patch": { + "additionalProperties": true, + "title": "Patch", + "type": "object" + } + }, + "title": "VerifierFixTaskPatch", + "type": "object" + }, + "VerifierRepair": { + "additionalProperties": false, + "description": "One deterministic repair affordance or prohibition.\n\nThe verifier owns the actor and safety boundary. These rows are not model\nsuggestions: they are a structured projection of remediation metadata and\ntrust-root rules so coding agents can distinguish mechanical fixes from\nhuman-only authority decisions.", + "properties": { + "actor": { + "enum": [ + "coding_agent", + "human" + ], + "title": "Actor", + "type": "string" + }, + "check_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Check Id" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Command" + }, + "finding_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Finding Id" + }, + "id": { + "title": "Id", + "type": "string" + }, + "kind": { + "title": "Kind", + "type": "string" + }, + "reason": { + "title": "Reason", + "type": "string" + }, + "target": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Target" + } + }, + "required": [ + "id", + "actor", + "kind", + "reason" + ], + "title": "VerifierRepair", + "type": "object" + } + }, + "$id": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/verifier-schema.v0.7.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "decision": { + "const": "passed" + } + }, + "required": [ + "decision" + ] + }, + "then": { + "properties": { + "applicability": { + "const": "verified" + }, + "can_merge_without_human": { + "const": true + }, + "capability_review": { + "properties": { + "policy_weakened": { + "const": false + }, + "trust_root_touched": { + "const": false + } + } + }, + "control": { + "properties": { + "state": { + "const": "complete" + } + }, + "required": [ + "state" + ] + }, + "execution": { + "const": "succeeded" + }, + "fix_task": { + "type": "null" + }, + "head_status": { + "const": "succeeded" + }, + "merge_verdict": { + "const": "mergeable" + }, + "release_decision": { + "properties": { + "blockers": { + "maxItems": 0 + }, + "decision": { + "const": "passed" + }, + "evidence_coverage": { + "properties": { + "evidence_gaps": { + "maxItems": 0 + }, + "human_review_recommended": { + "const": false + } + } + }, + "review_items": { + "maxItems": 0 + } + }, + "type": "object" + } + } + } + }, + { + "else": { + "properties": { + "control": { + "properties": { + "state": { + "enum": [ + "agent_action_required", + "human_review_required" + ] + } + }, + "required": [ + "state" + ] + } + } + }, + "if": { + "properties": { + "can_merge_without_human": { + "const": true + } + }, + "required": [ + "can_merge_without_human" + ] + }, + "then": { + "oneOf": [ + { + "properties": { + "applicability": { + "const": "verified" + }, + "decision": { + "const": "passed" + }, + "execution": { + "const": "succeeded" + } + } + }, + { + "properties": { + "applicability": { + "const": "not_applicable" + }, + "decision": { + "type": "null" + }, + "execution": { + "const": "skipped" + } + } + } + ], + "properties": { + "control": { + "properties": { + "state": { + "const": "complete" + } + }, + "required": [ + "state" + ] + } + } + } + }, + { + "if": { + "properties": { + "authorization": { + "properties": { + "status": { + "const": "accepted" + } + }, + "required": [ + "status" + ] + } + }, + "required": [ + "authorization" + ] + }, + "then": { + "properties": { + "applicability": { + "const": "verified" + }, + "can_merge_without_human": { + "const": false + }, + "control": { + "properties": { + "allowed_next_commands": { + "maxItems": 1, + "minItems": 1 + }, + "completion_allowed": { + "const": false + }, + "next_action": { + "properties": { + "kind": { + "const": "repair" + } + }, + "required": [ + "kind" + ] + }, + "state": { + "const": "agent_action_required" + } + }, + "required": [ + "state", + "completion_allowed", + "next_action", + "allowed_next_commands" + ] + }, + "decision": { + "const": "review_required" + }, + "execution": { + "const": "succeeded" + }, + "fix_task": { + "type": "null" + }, + "head_status": { + "const": "succeeded" + }, + "merge_verdict": { + "const": "human_review_required" + }, + "release_decision": { + "properties": { + "decision": { + "const": "review_required" + } + }, + "required": [ + "decision" + ], + "type": "object" + } + }, + "required": [ + "execution", + "head_status", + "release_decision", + "decision", + "merge_verdict", + "applicability", + "can_merge_without_human", + "control", + "fix_task" + ] + } + } + ], + "description": "JSON Schema for verifier.json. Generated from agents_shipgate.schemas.verifier.VerifierArtifact. Do not edit by hand.", + "properties": { + "agent_summary": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Agent Summary" + }, + "applicability": { + "default": "not_evaluated", + "enum": [ + "not_evaluated", + "verified", + "not_applicable", + "failed" + ], + "title": "Applicability", + "type": "string" + }, + "artifacts": { + "additionalProperties": { + "type": "string" + }, + "title": "Artifacts", + "type": "object" + }, + "authorization": { + "$ref": "#/$defs/AuthorizationEvaluationV1" + }, + "base_notes": { + "items": { + "type": "string" + }, + "title": "Base Notes", + "type": "array" + }, + "base_ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Base Ref" + }, + "base_report_json": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Base Report Json" + }, + "base_status": { + "default": "not_requested", + "enum": [ + "not_requested", + "skipped", + "diff_from_provided", + "ref_missing", + "archive_failed", + "missing_manifest", + "scan_failed", + "cache_hit", + "succeeded" + ], + "title": "Base Status", + "type": "string" + }, + "base_tree_sha": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Base Tree Sha" + }, + "can_merge_without_human": { + "default": false, + "title": "Can Merge Without Human", + "type": "boolean" + }, + "capability_review": { + "$ref": "#/$defs/VerifierCapabilityReview" + }, + "changed_files": { + "items": { + "type": "string" + }, + "title": "Changed Files", + "type": "array" + }, + "config": { + "title": "Config", + "type": "string" + }, + "control": { + "$ref": "#/$defs/AgentControl" + }, + "decision": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Decision" + }, + "decision_id": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Decision Id" + }, + "diff_status": { + "$ref": "#/$defs/VerifierDiffStatus" + }, + "diff_text_available": { + "default": false, + "title": "Diff Text Available", + "type": "boolean" + }, + "engine_requirement_id": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Engine Requirement Id" + }, + "execution": { + "default": "not_run", + "enum": [ + "not_run", + "succeeded", + "skipped", + "failed" + ], + "title": "Execution", + "type": "string" + }, + "executor_id": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Executor Id" + }, + "fix_task": { + "anyOf": [ + { + "$ref": "#/$defs/VerifierFixTask" + }, + { + "type": "null" + } + ], + "default": null + }, + "forbidden_actions": { + "items": { + "type": "string" + }, + "title": "Forbidden Actions", + "type": "array" + }, + "forbidden_file_edits": { + "items": { + "type": "string" + }, + "title": "Forbidden File Edits", + "type": "array" + }, + "head_exit_code": { + "default": 0, + "title": "Head Exit Code", + "type": "integer" + }, + "head_ref": { + "default": "HEAD", + "title": "Head Ref", + "type": "string" + }, + "head_report_json": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Head Report Json" + }, + "head_status": { + "default": "not_run", + "enum": [ + "not_run", + "succeeded", + "skipped", + "failed" + ], + "title": "Head Status", + "type": "string" + }, + "head_tree_sha": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Head Tree Sha" + }, + "headline": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Headline" + }, + "input_set_id": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Input Set Id" + }, + "merge_verdict": { + "default": "unknown", + "enum": [ + "mergeable", + "human_review_required", + "insufficient_evidence", + "blocked", + "unknown" + ], + "title": "Merge Verdict", + "type": "string" + }, + "mode": { + "default": "advisory", + "title": "Mode", + "type": "string" + }, + "release_decision": { + "anyOf": [ + { + "$ref": "#/$defs/ReleaseDecision" + }, + { + "type": "null" + } + ], + "default": null + }, + "request_id": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Request Id" + }, + "reviewer_summary": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Reviewer Summary" + }, + "runtime_behavior_verified": { + "const": false, + "default": false, + "title": "Runtime Behavior Verified", + "type": "boolean" + }, + "static_analysis_only": { + "const": true, + "default": true, + "title": "Static Analysis Only", + "type": "boolean" + }, + "static_verdict_disclaimer": { + "default": "This verdict covers deterministic static evidence only. Agents Shipgate did not execute the agent or prove runtime behavior, tool routing, credential enforcement, or safety.", + "title": "Static Verdict Disclaimer", + "type": "string" + }, + "subject_id": { + "anyOf": [ + { + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Subject Id" + }, + "trigger": { + "additionalProperties": true, + "title": "Trigger", + "type": "object" + }, + "verifier_schema_version": { + "const": "0.7", + "default": "0.7", + "title": "Verifier Schema Version", + "type": "string" + }, + "workspace": { + "title": "Workspace", + "type": "string" + } + }, + "required": [ + "workspace", + "config", + "diff_status", + "control", + "authorization" + ], + "title": "Agents Shipgate Verifier Artifact v0.7", + "type": "object" +} diff --git a/llms-full.txt b/llms-full.txt index 302317e9..f5f09d05 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -340,7 +340,7 @@ agents-shipgate trigger --base origin/main --head HEAD --json agents-shipgate trigger --list-rules --json ``` -The command emits a stable JSON verdict: `should_run` (alias of `run_shipgate`), `force_run`, `dry_run_recommended`, `skip_reason`, `matched_rules`, `changed_files`, and `diff_tokens`. The developer entry point `python -m agents_shipgate.triggers shipgate.yaml prompts/refund.md` is preserved. +The command emits a stable JSON verdict: `should_run` (alias of `run_shipgate`), `force_run`, `dry_run_recommended`, `skip_reason`, `matched_rules`, `changed_files`, `diff_tokens`, plus `input_status` and `evaluation_status` (catalog schema `0.3`). When the diff could not be read in full, a *skip* verdict is withheld: `evaluation_status` is `not_evaluated` and `should_run`/`run_shipgate`/`skip`/`skip_reason` are `null`, so an unread diff is never reported as `no_match`. A *run* verdict is still published, because rule matching is monotone — evidence that already matched cannot be un-matched by the bytes that are missing — and it arrives as `evaluation_status: evaluated` with `should_run: true`. That evidence may be a rule matched on the change set or `force_run` from a manifest that is present regardless of the diff, so read `matched_rules` before attributing it. Branch on `evaluation_status`, not on `should_run` alone. The developer entry point `python -m agents_shipgate.triggers shipgate.yaml prompts/refund.md` is preserved. **Stop conditions.** Stop and do not run `init` only when **all** of these hold: @@ -1101,7 +1101,7 @@ Downstream repos generated with - Current report schema: `0.34` — [`docs/report-schema.v0.34.json`](report-schema.v0.34.json) - Current packet schema: `0.12` — [`docs/packet-schema.v0.12.json`](packet-schema.v0.12.json) - Current shared agent result schema: `agent_result_v2` — [`docs/agent-result-schema.v2.json`](agent-result-schema.v2.json) -- Current verifier schema: `0.6` — [`docs/verifier-schema.v0.6.json`](verifier-schema.v0.6.json) +- Current verifier schema: `0.7` — [`docs/verifier-schema.v0.7.json`](verifier-schema.v0.7.json) - Current verify-run schema: `shipgate.verify_run/v3` — [`docs/verify-run-schema.v3.json`](verify-run-schema.v3.json) - Current verification identity schemas: [`plan v1`](verification-plan-schema.v1.json), [`unit result v1`](verification-unit-result-schema.v1.json), [`artifact manifest v1`](verification-artifact-manifest-schema.v1.json), and [`terminal receipt v1`](verification-receipt-schema.v1.json) - Current human-authorization schemas: request, signed grant, verifier evaluation, and external trust policy v1 — [`docs/human-authorization-schema.v1.json`](human-authorization-schema.v1.json) @@ -1117,7 +1117,7 @@ Downstream repos generated with - Current registry schema: `0.4` — [`docs/registry-schema.v0.4.json`](registry-schema.v0.4.json) - Current org evidence bundle schema: `shipgate.org_evidence_bundle/v2` — [`docs/org-evidence-bundle-schema.v2.json`](org-evidence-bundle-schema.v2.json) - Current host-grants inventory, baseline, and drift schemas: `0.2` — [`inventory`](host-grants-inventory-schema.v0.2.json), [`baseline`](host-grants-baseline-schema.v0.2.json), [`drift`](host-grants-drift-schema.v0.2.json) -- Current trigger catalog schema: `0.2` — [`docs/triggers.json`](triggers.json) +- Current trigger catalog schema: `0.3` — [`docs/triggers.json`](triggers.json) - Current governance benchmark catalog schema: `0.2` — [`docs/governance-benchmark-catalog-schema.v0.2.json`](governance-benchmark-catalog-schema.v0.2.json) - Current governance benchmark result schema: `0.2` — [`docs/governance-benchmark-result-schema.v0.2.json`](governance-benchmark-result-schema.v0.2.json) - Frozen-reference report schemas: frozen [`v0.33`](report-schema.v0.33.json), frozen [`v0.32`](report-schema.v0.32.json), frozen [`v0.31`](report-schema.v0.31.json), frozen [`v0.30`](report-schema.v0.30.json), and older versions listed in [`docs/INDEX.md`](INDEX.md#reference) @@ -1355,8 +1355,8 @@ from existing artifacts with: agents-shipgate agent handoff --from agents-shipgate-reports/verifier.json --json ``` -In `agents-shipgate-reports/verifier.json`, read the v0.6 fields below (full -schema [`docs/verifier-schema.v0.6.json`](verifier-schema.v0.6.json)). **Lead +In `agents-shipgate-reports/verifier.json`, read the v0.7 fields below (full +schema [`docs/verifier-schema.v0.7.json`](verifier-schema.v0.7.json)). **Lead with `control.state`.** Every release and merge field below is a mirror or deterministic projection of `report.json`; the authorization evaluation is an operational overlay and cannot change those fields. @@ -1368,6 +1368,27 @@ operational overlay and cannot change those fields. generated schemas enforce the variants with `oneOf`. Only a new verifier artifact can clear a pending control obligation. - `execution` — `"not_run" | "succeeded" | "skipped" | "failed"`. +- `diff_status` — whether the compared change set was read at all. + `completeness` is `"complete"` / `"partial"` / `"unavailable"`; `reason` is + `null` only when complete, and otherwise `not_attempted`, `refs_missing`, + `merge_base_missing` (shallow checkout — deepening restores the merge base), + `unrelated_histories` (no common ancestor exists; no fetch can create one), + `objects_missing`, `metadata_limit_exceeded`, `body_limit_exceeded`, + `git_timeout`, or `git_failed`. `remediation` names + the repair and `fetch_repairable` says whether fetching can perform it. + **Only `"complete"` licenses reading a negative `trigger` result**; anything + else means the diff was not read, which is never evidence that a PR is + unrelated to agent capabilities. `null` means a pre-v0.7 artifact — unknown, + not complete. +- `trigger` — the run/skip evaluation. Read `evaluation_status` first: when it + is `"not_evaluated"`, `should_run` / `run_shipgate` / `skip` / `skip_reason` + are `null` and `next_action.kind` is `"input_required"`. `skip_reason` is + never `"no_match"` for inputs that were not fully read. `"evaluated"` on an + incomplete `diff_status` is not a contradiction: only *skip* verdicts are + withheld, so a `should_run: true` reached from evidence that did not depend + on the missing bytes is authoritative and must not be overridden. Read + `matched_rules` to see what carried it — a `force_run` match rests on the + manifest, not on the diff. - `merge_verdict` — `"mergeable"` / `"human_review_required"` / `"insufficient_evidence"` / `"blocked"` / `"unknown"`. Deterministic projection of `release_decision.decision` (`passed`→`mergeable`, diff --git a/plugins/claude-code/skills/agents-shipgate/SKILL.md b/plugins/claude-code/skills/agents-shipgate/SKILL.md index f60c44c0..93c8815e 100644 --- a/plugins/claude-code/skills/agents-shipgate/SKILL.md +++ b/plugins/claude-code/skills/agents-shipgate/SKILL.md @@ -74,7 +74,7 @@ For non-GitHub CI (GitLab, CircleCI, Jenkins, Azure Pipelines, Buildkite, Bitbuc - **CLI surface** follows the current 0.x contract line — see https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/STABILITY.md. - **Installed CLI contract**: when available, run `agents-shipgate contract --json` to verify local schema versions, capability/research surfaces, `release_decision.decision`, and manual-review signal fields. Older installs should use [`docs/agent-contract-current.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md) or upgrade before automating against the local contract command. -- **Verifier JSON**: `verifier_schema_version: "0.6"`. Switch on `control.state`, then read `merge_verdict`, `can_merge_without_human`, `control.next_action`, `fix_task`, `capability_review.top_changes`, `trust_root_touched`, and `policy_weakened` before summarizing an AI-generated PR. `merge_verdict` is a deterministic projection; the gate remains `report.json.release_decision.decision`. +- **Verifier JSON**: `verifier_schema_version: "0.7"`. Switch on `control.state`, then read `merge_verdict`, `can_merge_without_human`, `control.next_action`, `fix_task`, `capability_review.top_changes`, `trust_root_touched`, and `policy_weakened` before summarizing an AI-generated PR. `merge_verdict` is a deterministic projection; the gate remains `report.json.release_decision.decision`. Check `diff_status.completeness` before you believe any negative result: only `"complete"` means the PR diff was actually read. Anything else (`reason` is one of `not_attempted`, `refs_missing`, `merge_base_missing`, `unrelated_histories`, `objects_missing`, `metadata_limit_exceeded`, `body_limit_exceeded`, `git_timeout`, `git_failed`) means evidence was missing: follow `remediation`, and never report the PR as unrelated to agent capabilities. Then read `trigger.evaluation_status` for what that cost. `"not_evaluated"` (with `trigger.should_run` `null`) means no verdict exists. `"evaluated"` on an incomplete diff is not a contradiction — evidence that did not depend on the missing bytes already proved Shipgate should run — so honor `should_run: true` instead of overriding it, and still recover the diff before trusting a merge verdict. That evidence is either a rule matched on the change set or, in an already-adopted repository, `force_run: true` from the manifest alone; check `matched_rules` before attributing the verdict to anything the diff showed. - **Verification receipt**: `verification-receipt.json` uses `schema_version: "shipgate.verification_receipt/v1"` and is written last. Validate it before trusting any projected verdict; it content-addresses the request, executor, unit result, decision, and complete artifact set. - **Verify run JSON**: `verify-run.json` uses `schema_version: "shipgate.verify_run/v3"`, embeds the content-addressed plan and executor, and binds unit-result and decision IDs. `run_id` is an exact compatibility alias of `request_id`; do not treat the run projection as a second gate. - **Report JSON**: `report_schema_version: "0.34"`. Read `release_decision.decision` first. A `passed` decision requires a complete root-reachable static binding graph plus complete, conflict-free identity, effect, and authority evidence for every reachable action; it does not prove runtime behavior. Preserve `release_decision.static_analysis_only=true`, `runtime_behavior_verified=false`, and `static_verdict_disclaimer` in summaries. Read `release_decision.evidence_coverage.binding_coverage`, `semantic_coverage`, `identity_coverage`, and `policy_gap_count`, then work every `evidence_gaps[].next_action` in order. Binding, semantic, and policy-applicability gaps are not Findings and cannot be suppressed, baselined, severity-overridden, cleared by `--no-heuristics`, or satisfied by `human_ack`; binding, effect, and authority declarations are human assertions and must never be auto-written. Use `tool_catalog[]` for diagnostics and `tool_inventory[]` for the proven reachable surface. The current schema is [`docs/report-schema.v0.34.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/report-schema.v0.34.json); v0.33 is a frozen compatibility reference. See the [current agent contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md), [verification identity contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/verification-reproducibility.md), and [evidence-backed passed contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/passed-verdict-contract.md). diff --git a/plugins/claude-code/skills/agents-shipgate/prompts/decide-shipgate-relevance.md b/plugins/claude-code/skills/agents-shipgate/prompts/decide-shipgate-relevance.md index 5087607f..43e0dac1 100644 --- a/plugins/claude-code/skills/agents-shipgate/prompts/decide-shipgate-relevance.md +++ b/plugins/claude-code/skills/agents-shipgate/prompts/decide-shipgate-relevance.md @@ -22,7 +22,7 @@ the rules to the changed file list. - **Local repo** (already adopted Shipgate): read `docs/triggers.json` directly. - **Remote** (target repo without Shipgate): fetch `https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/triggers.json`. - - The catalog has `schema_version: "0.2"`; match `surface_class` instead of maintaining a parallel path list. + - The catalog has `schema_version: "0.3"`; match `surface_class` instead of maintaining a parallel path list. 3. **Apply the rules.** Two equivalent options: @@ -52,7 +52,18 @@ the rules to the changed file list. manually). If your repo already has a manifest, also pass `--manifest-present` so the `force_run` rule can fire. The output shape is `{run_shipgate, dry_run_recommended, - matched_rules, stop_conditions_fired, rationale, schema_version}`. + matched_rules, stop_conditions_fired, rationale, schema_version, + input_status, evaluation_status}`. + When `evaluation_status` is `not_evaluated`, `run_shipgate` is `null`: + the diff could not be read, so there is no verdict. Report the missing + input and stop — never treat it as "this PR is not agent-related". + An `input_status` other than `complete` alongside + `evaluation_status: evaluated` is not a contradiction: evidence that did + not depend on the missing bytes already proved Shipgate should run — a + rule matched on the change set, or `force_run` from a manifest that is + present regardless of the diff. Honor that verdict, read `matched_rules` + before saying what established it, and say the diff still needs + recovering. 4. **Emit the decision.** Always reply in this exact JSON shape so downstream automation can parse you: diff --git a/prompts/decide-shipgate-relevance.md b/prompts/decide-shipgate-relevance.md index 5087607f..43e0dac1 100644 --- a/prompts/decide-shipgate-relevance.md +++ b/prompts/decide-shipgate-relevance.md @@ -22,7 +22,7 @@ the rules to the changed file list. - **Local repo** (already adopted Shipgate): read `docs/triggers.json` directly. - **Remote** (target repo without Shipgate): fetch `https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/triggers.json`. - - The catalog has `schema_version: "0.2"`; match `surface_class` instead of maintaining a parallel path list. + - The catalog has `schema_version: "0.3"`; match `surface_class` instead of maintaining a parallel path list. 3. **Apply the rules.** Two equivalent options: @@ -52,7 +52,18 @@ the rules to the changed file list. manually). If your repo already has a manifest, also pass `--manifest-present` so the `force_run` rule can fire. The output shape is `{run_shipgate, dry_run_recommended, - matched_rules, stop_conditions_fired, rationale, schema_version}`. + matched_rules, stop_conditions_fired, rationale, schema_version, + input_status, evaluation_status}`. + When `evaluation_status` is `not_evaluated`, `run_shipgate` is `null`: + the diff could not be read, so there is no verdict. Report the missing + input and stop — never treat it as "this PR is not agent-related". + An `input_status` other than `complete` alongside + `evaluation_status: evaluated` is not a contradiction: evidence that did + not depend on the missing bytes already proved Shipgate should run — a + rule matched on the change set, or `force_run` from a manifest that is + present regardless of the diff. Honor that verdict, read `matched_rules` + before saying what established it, and say the diff still needs + recovering. 4. **Emit the decision.** Always reply in this exact JSON shape so downstream automation can parse you: diff --git a/skills/agents-shipgate/SKILL.md b/skills/agents-shipgate/SKILL.md index f60c44c0..93c8815e 100644 --- a/skills/agents-shipgate/SKILL.md +++ b/skills/agents-shipgate/SKILL.md @@ -74,7 +74,7 @@ For non-GitHub CI (GitLab, CircleCI, Jenkins, Azure Pipelines, Buildkite, Bitbuc - **CLI surface** follows the current 0.x contract line — see https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/STABILITY.md. - **Installed CLI contract**: when available, run `agents-shipgate contract --json` to verify local schema versions, capability/research surfaces, `release_decision.decision`, and manual-review signal fields. Older installs should use [`docs/agent-contract-current.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md) or upgrade before automating against the local contract command. -- **Verifier JSON**: `verifier_schema_version: "0.6"`. Switch on `control.state`, then read `merge_verdict`, `can_merge_without_human`, `control.next_action`, `fix_task`, `capability_review.top_changes`, `trust_root_touched`, and `policy_weakened` before summarizing an AI-generated PR. `merge_verdict` is a deterministic projection; the gate remains `report.json.release_decision.decision`. +- **Verifier JSON**: `verifier_schema_version: "0.7"`. Switch on `control.state`, then read `merge_verdict`, `can_merge_without_human`, `control.next_action`, `fix_task`, `capability_review.top_changes`, `trust_root_touched`, and `policy_weakened` before summarizing an AI-generated PR. `merge_verdict` is a deterministic projection; the gate remains `report.json.release_decision.decision`. Check `diff_status.completeness` before you believe any negative result: only `"complete"` means the PR diff was actually read. Anything else (`reason` is one of `not_attempted`, `refs_missing`, `merge_base_missing`, `unrelated_histories`, `objects_missing`, `metadata_limit_exceeded`, `body_limit_exceeded`, `git_timeout`, `git_failed`) means evidence was missing: follow `remediation`, and never report the PR as unrelated to agent capabilities. Then read `trigger.evaluation_status` for what that cost. `"not_evaluated"` (with `trigger.should_run` `null`) means no verdict exists. `"evaluated"` on an incomplete diff is not a contradiction — evidence that did not depend on the missing bytes already proved Shipgate should run — so honor `should_run: true` instead of overriding it, and still recover the diff before trusting a merge verdict. That evidence is either a rule matched on the change set or, in an already-adopted repository, `force_run: true` from the manifest alone; check `matched_rules` before attributing the verdict to anything the diff showed. - **Verification receipt**: `verification-receipt.json` uses `schema_version: "shipgate.verification_receipt/v1"` and is written last. Validate it before trusting any projected verdict; it content-addresses the request, executor, unit result, decision, and complete artifact set. - **Verify run JSON**: `verify-run.json` uses `schema_version: "shipgate.verify_run/v3"`, embeds the content-addressed plan and executor, and binds unit-result and decision IDs. `run_id` is an exact compatibility alias of `request_id`; do not treat the run projection as a second gate. - **Report JSON**: `report_schema_version: "0.34"`. Read `release_decision.decision` first. A `passed` decision requires a complete root-reachable static binding graph plus complete, conflict-free identity, effect, and authority evidence for every reachable action; it does not prove runtime behavior. Preserve `release_decision.static_analysis_only=true`, `runtime_behavior_verified=false`, and `static_verdict_disclaimer` in summaries. Read `release_decision.evidence_coverage.binding_coverage`, `semantic_coverage`, `identity_coverage`, and `policy_gap_count`, then work every `evidence_gaps[].next_action` in order. Binding, semantic, and policy-applicability gaps are not Findings and cannot be suppressed, baselined, severity-overridden, cleared by `--no-heuristics`, or satisfied by `human_ack`; binding, effect, and authority declarations are human assertions and must never be auto-written. Use `tool_catalog[]` for diagnostics and `tool_inventory[]` for the proven reachable surface. The current schema is [`docs/report-schema.v0.34.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/report-schema.v0.34.json); v0.33 is a frozen compatibility reference. See the [current agent contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md), [verification identity contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/verification-reproducibility.md), and [evidence-backed passed contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/passed-verdict-contract.md). diff --git a/skills/agents-shipgate/prompts/decide-shipgate-relevance.md b/skills/agents-shipgate/prompts/decide-shipgate-relevance.md index 5087607f..43e0dac1 100644 --- a/skills/agents-shipgate/prompts/decide-shipgate-relevance.md +++ b/skills/agents-shipgate/prompts/decide-shipgate-relevance.md @@ -22,7 +22,7 @@ the rules to the changed file list. - **Local repo** (already adopted Shipgate): read `docs/triggers.json` directly. - **Remote** (target repo without Shipgate): fetch `https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/triggers.json`. - - The catalog has `schema_version: "0.2"`; match `surface_class` instead of maintaining a parallel path list. + - The catalog has `schema_version: "0.3"`; match `surface_class` instead of maintaining a parallel path list. 3. **Apply the rules.** Two equivalent options: @@ -52,7 +52,18 @@ the rules to the changed file list. manually). If your repo already has a manifest, also pass `--manifest-present` so the `force_run` rule can fire. The output shape is `{run_shipgate, dry_run_recommended, - matched_rules, stop_conditions_fired, rationale, schema_version}`. + matched_rules, stop_conditions_fired, rationale, schema_version, + input_status, evaluation_status}`. + When `evaluation_status` is `not_evaluated`, `run_shipgate` is `null`: + the diff could not be read, so there is no verdict. Report the missing + input and stop — never treat it as "this PR is not agent-related". + An `input_status` other than `complete` alongside + `evaluation_status: evaluated` is not a contradiction: evidence that did + not depend on the missing bytes already proved Shipgate should run — a + rule matched on the change set, or `force_run` from a manifest that is + present regardless of the diff. Honor that verdict, read `matched_rules` + before saying what established it, and say the diff still needs + recovering. 4. **Emit the decision.** Always reply in this exact JSON shape so downstream automation can parse you: diff --git a/src/agents_shipgate/cli/attest.py b/src/agents_shipgate/cli/attest.py index 258ad4f8..3e369489 100644 --- a/src/agents_shipgate/cli/attest.py +++ b/src/agents_shipgate/cli/attest.py @@ -132,7 +132,7 @@ def _attest_command( receipt_path = source.with_name("verification-receipt.json") receipt_payload = _load_optional_json_object(receipt_path) receipt_sha256 = None - if verifier.get("verifier_schema_version") in {"0.5", "0.6"}: + if verifier.get("verifier_schema_version") in {"0.5", "0.6", "0.7"}: if not receipt_payload: raise typer.Exit(3) receipt = VerificationReceipt.model_validate(receipt_payload) diff --git a/src/agents_shipgate/cli/discovery/agent_instructions/renderers/local_contract.py b/src/agents_shipgate/cli/discovery/agent_instructions/renderers/local_contract.py index ecceec2e..52a2fc83 100644 --- a/src/agents_shipgate/cli/discovery/agent_instructions/renderers/local_contract.py +++ b/src/agents_shipgate/cli/discovery/agent_instructions/renderers/local_contract.py @@ -11,11 +11,16 @@ def render_file() -> str: return render_local_agent_contract() -# Exact render shipped by local contract schema v6. Keeping this hash lets -# first-adoption reruns upgrade an untouched managed file to v7 without -# overwriting user-authored JSON. +# Exact renders shipped by earlier releases. Keeping these hashes lets a rerun +# upgrade an untouched managed file in place without overwriting user-authored +# JSON. The file body changes whenever any advertised sub-schema version moves, +# so every outgoing render is appended here — not only schema-version bumps of +# the contract file itself. PRIOR_RENDER_SHA256: tuple[str, ...] = ( + # local contract schema v6 "85d33d005d35f933b72e32c2d370efc2680e09d2ebe0c9997931c8ab4f352738", + # v7 before verifier 0.6 -> 0.7 and trigger catalog 0.2 -> 0.3 + "6041d5fc42ee4be37596c9c13b9752a8a511bb18bc987b32b0ffb49160ee6d93", ) diff --git a/src/agents_shipgate/cli/org.py b/src/agents_shipgate/cli/org.py index e31e7889..cad32726 100644 --- a/src/agents_shipgate/cli/org.py +++ b/src/agents_shipgate/cli/org.py @@ -228,7 +228,7 @@ def org_bundle( ) receipt = _load_optional_json_object(receipt_path) verified_receipt: VerificationReceipt | None = None - if verifier.get("verifier_schema_version") in {"0.5", "0.6"}: + if verifier.get("verifier_schema_version") in {"0.5", "0.6", "0.7"}: try: if receipt is None: raise ValueError("current verifier evidence is missing verification-receipt.json") diff --git a/src/agents_shipgate/cli/trigger.py b/src/agents_shipgate/cli/trigger.py index f80a28ba..34924243 100644 --- a/src/agents_shipgate/cli/trigger.py +++ b/src/agents_shipgate/cli/trigger.py @@ -34,6 +34,7 @@ from agents_shipgate.core.errors import ConfigError, InputParseError from agents_shipgate.triggers import ( _git_diff_context, + _verdict_label, evaluate, load_triggers, ) @@ -202,7 +203,7 @@ def trigger( typer.echo(json.dumps(result, indent=2)) return - verdict = "RUN" if result["should_run"] else "SKIP" + verdict = _verdict_label(result) typer.echo(f"Verdict: {verdict}") typer.echo(f"Rationale: {result['rationale']}") if result["dry_run_recommended"]: diff --git a/src/agents_shipgate/cli/verify/git.py b/src/agents_shipgate/cli/verify/git.py index ce0303cf..2d4903e3 100644 --- a/src/agents_shipgate/cli/verify/git.py +++ b/src/agents_shipgate/cli/verify/git.py @@ -25,6 +25,8 @@ _DIFF_CONFIG_LIMIT = 1024 * 1024 _DIFF_METADATA_LIMIT = 8 * 1024 * 1024 _DIFF_BODY_LIMIT = 32 * 1024 * 1024 +_GIT_STDERR_LIMIT = 8 * 1024 +_GIT_STDERR_EXCERPT_CHARS = 240 _TEXT_CAPABILITY_SUFFIXES = frozenset( { ".json", @@ -61,6 +63,140 @@ def __init__(self, paths: list[str]) -> None: ) +# Why a requested diff could not be read in full. These are input-acquisition +# states, not verdicts: none of them says anything about what the PR contains. +# ``refs_missing``/``merge_base_missing``/``objects_missing`` are repairable by +# making history or objects locally available; the rest are not. +DiffInputReason = Literal[ + "not_attempted", + "refs_missing", + "merge_base_missing", + "unrelated_histories", + "objects_missing", + "metadata_limit_exceeded", + "body_limit_exceeded", + "git_timeout", + "git_failed", +] + +# Mirrors ``BoundaryChangeSet.completeness`` so ``check`` and ``verify`` speak +# one vocabulary for partially-read inputs. +DiffCompleteness = Literal["complete", "partial", "unavailable"] + +_FETCHABLE_DIFF_REASONS: frozenset[str] = frozenset( + {"refs_missing", "merge_base_missing", "objects_missing"} +) + +_DIFF_REASON_REMEDIATION: dict[str, str] = { + "not_attempted": ( + "Verification stopped before it read any diff, so nothing is known " + "about the change set. Clear the reported blocker and rerun." + ), + "refs_missing": ( + "Fetch the missing ref locally (for example " + "`git fetch --no-tags origin `), then rerun." + ), + "merge_base_missing": ( + "This checkout is shallow, so the merge base the two refs share was " + "truncated away. Deepen history (`git fetch --deepen=`, or " + "`git fetch --unshallow` / checkout with `fetch-depth: 0`), then rerun." + ), + "unrelated_histories": ( + "The two refs share no common ancestor and this checkout is not " + "shallow, so no fetch can create one. Confirm the base names the right " + "comparison point — a force-push or a rewritten branch produces this — " + "then rerun." + ), + "objects_missing": ( + "This checkout is a partial clone and the objects the diff needs were " + "never fetched. Verification runs with GIT_NO_LAZY_FETCH=1 and will " + "not fetch them implicitly. Hydrate them (for example " + "`git fetch --refetch origin`, or clone without `--filter`), then " + "rerun." + ), + "metadata_limit_exceeded": ( + "The change set exceeds Shipgate's static diff-metadata bound. Split " + "the change, or exclude generated output from the compared range." + ), + "body_limit_exceeded": ( + "The unified diff exceeds Shipgate's static diff-body bound. Changed " + "paths were still collected; split the change or exclude generated " + "output to recover the textual evidence." + ), + "git_timeout": ( + "Git did not finish within the static timeout. Inspect repository " + "size and local Git health before rerunning; fetching refs will not " + "repair it." + ), + "git_failed": ( + "Inspect the reported Git failure before rerunning; fetching refs " + "cannot repair a deterministic input failure." + ), +} + + +@dataclass(frozen=True) +class DiffContext: + """One diff-acquisition attempt and exactly how complete its result is. + + ``completeness`` is the contract. ``complete`` means every changed path and + the full unified-diff body were read; ``partial`` means the changed paths + are authoritative but the textual body is missing or unproven; and + ``unavailable`` means nothing about the change set was established. A caller + must never treat ``partial`` or ``unavailable`` evidence as proof that a PR + is unrelated to agent capabilities. + """ + + changed_files: tuple[str, ...] = () + diff_text: str = "" + completeness: DiffCompleteness = "complete" + reason: DiffInputReason | None = None + detail: str = "" + + @property + def remediation(self) -> str: + if self.reason is None: + return "" + return _DIFF_REASON_REMEDIATION[self.reason] + + @property + def fetch_repairable(self) -> bool: + """Whether making refs/objects available locally can repair this.""" + + return self.reason in _FETCHABLE_DIFF_REASONS + + @property + def note(self) -> str: + """One safe operator-facing line for ``base_notes``.""" + + if self.completeness == "complete": + return "" + scope = ( + "Changed paths were collected but the diff body could not be read" + if self.completeness == "partial" + else "The diff could not be read" + ) + detail = "" + if self.detail: + terminated = self.detail.rstrip() + if terminated and terminated[-1] not in ".!?": + terminated += "." + detail = f" Git reported: {terminated}" + return f"{scope} ({self.reason}).{detail} {self.remediation}" + + +class DiffInputError(ConfigError): + """A diff that could not be read in full, with its classified reason.""" + + def __init__(self, context: DiffContext) -> None: + self.context = context + super().__init__(context.note.strip()) + + +class _UnavailableRevisionError(ConfigError): + """A revision expression that names refs this checkout does not have.""" + + _SAFE_DIFF_CONFIG = [ "-c", "core.fsmonitor=false", @@ -461,20 +597,63 @@ def git_path(workspace: Path, path: str) -> Path: def diff_context(workspace: Path, base: str, head: str) -> tuple[list[str], str]: + """Return committed-ref diff paths and body, or raise on any shortfall. + + Callers that can act on a partially-read diff should use + :func:`collect_diff_context` instead. This wrapper stays strict so a caller + that cannot represent partial evidence never silently reasons over it. + """ + + return _require_complete(collect_diff_context(workspace, base, head)) + + +def diff_revspec_context(workspace: Path, revspec: str) -> tuple[list[str], str]: + """Return deterministic committed-ref diff paths and body, or raise.""" + + return _require_complete(collect_revspec_diff_context(workspace, revspec)) + + +def _require_complete(context: DiffContext) -> tuple[list[str], str]: + if context.completeness != "complete": + raise DiffInputError(context) + return list(context.changed_files), context.diff_text + + +def collect_diff_context(workspace: Path, base: str, head: str) -> DiffContext: + """Collect the ``base...head`` diff and report exactly how complete it is.""" + base_commit = commit_sha(workspace, base) head_commit = commit_sha(workspace, head) if base_commit is None or head_commit is None: - raise ConfigError("Git diff refs are unavailable locally") - revspec = f"{base_commit}...{head_commit}" - return diff_revspec_context(workspace, revspec) + missing = base if base_commit is None else head + return DiffContext( + completeness="unavailable", + reason="refs_missing", + detail=f"Git ref {missing!r} is not available locally.", + ) + return collect_revspec_diff_context(workspace, f"{base_commit}...{head_commit}") -def diff_revspec_context(workspace: Path, revspec: str) -> tuple[list[str], str]: - """Return deterministic committed-ref diff paths and body.""" +def collect_revspec_diff_context(workspace: Path, revspec: str) -> DiffContext: + """Collect a deterministic committed-ref diff without discarding evidence. + + Metadata and body are read separately and reported separately. A body that + cannot be read no longer throws away the changed-path evidence that was + successfully collected — a blobless clone, for instance, answers + ``--name-status`` fully while failing the textual diff, and those paths are + exactly what tells a caller the PR touches an agent surface. + """ _reject_unbound_diff_configuration(workspace) - revspec = _resolved_diff_revspec(workspace, revspec) - names = _run_git_bounded_output( + try: + revspec = _resolved_diff_revspec(workspace, revspec) + except _UnavailableRevisionError as exc: + return DiffContext( + completeness="unavailable", + reason="refs_missing", + detail=str(exc), + ) + names = _run_git_bounded_result( workspace, [ *_SAFE_DIFF_CONFIG, @@ -486,7 +665,15 @@ def diff_revspec_context(workspace: Path, revspec: str) -> tuple[list[str], str] ], max_output_bytes=_DIFF_METADATA_LIMIT, ) - body = _run_git_bounded_output( + if names.payload is None: + reason, detail = _classify_diff_failure( + names, limit_reason="metadata_limit_exceeded", workspace=workspace + ) + return DiffContext( + completeness="unavailable", reason=reason, detail=detail + ) + paths = tuple(sorted(_paths_from_name_status(names.payload))) + body = _run_git_bounded_result( workspace, [ *_SAFE_DIFF_CONFIG, @@ -496,17 +683,34 @@ def diff_revspec_context(workspace: Path, revspec: str) -> tuple[list[str], str] ], max_output_bytes=_DIFF_BODY_LIMIT, ) - if names is None or body is None: - raise ConfigError("Git diff exceeded static output bounds or could not be read.") - paths = sorted(_paths_from_name_status(names)) - diff_text = _decode_diff_body(body) + if body.payload is None: + reason, detail = _classify_diff_failure( + body, limit_reason="body_limit_exceeded", workspace=workspace + ) + return DiffContext( + changed_files=paths, + completeness="partial", + reason=reason, + detail=detail, + ) + diff_text = _decode_diff_body(body.payload) try: _reject_binary_capability_paths(workspace, revspec) except BinaryCapabilityDiffError as exc: - exc.changed_paths = tuple(paths) + exc.changed_paths = paths exc.diff_text = diff_text raise - return paths, diff_text + except DiffInputError as exc: + # The binary-hiding guard could not run, so the body is not proven to + # contain every capability path's text. Keep it, but never as complete. + return DiffContext( + changed_files=paths, + diff_text=diff_text, + completeness="partial", + reason=exc.context.reason, + detail=exc.context.detail, + ) + return DiffContext(changed_files=paths, diff_text=diff_text) def _paths_from_name_status(payload: bytes) -> list[str]: @@ -552,7 +756,7 @@ def _reject_binary_capability_paths( ) -> None: """Fail closed when a source-like path is hidden behind a binary marker.""" - payload = _run_git_bounded_output( + result = _run_git_bounded_result( workspace, [ *_SAFE_DIFF_CONFIG, @@ -567,10 +771,13 @@ def _reject_binary_capability_paths( ], max_output_bytes=_DIFF_METADATA_LIMIT, ) + payload = result.payload if payload is None: - raise ConfigError( - "Git binary-path metadata exceeded static output bounds or could " - "not be read." + reason, detail = _classify_diff_failure( + result, limit_reason="metadata_limit_exceeded", workspace=workspace + ) + raise DiffInputError( + DiffContext(completeness="unavailable", reason=reason, detail=detail) ) hidden: list[str] = [] for record in payload.split(b"\0"): @@ -900,7 +1107,7 @@ def working_tree_context( pathspec = _worktree_pathspec(workspace, exclude) if reject_index_hidden: _reject_index_hidden_capability_paths(workspace, pathspec=pathspec) - names = _run_git_bounded_output( + names = _run_git_bounded_result( workspace, [ *_SAFE_DIFF_CONFIG, @@ -914,30 +1121,18 @@ def working_tree_context( ], max_output_bytes=_DIFF_METADATA_LIMIT, ) - body = _run_git_bounded_output( - workspace, - [ - *_SAFE_DIFF_CONFIG, - "diff", - *_DETERMINISTIC_DIFF_OPTIONS, - "HEAD", - "--", - *pathspec, - ], - max_output_bytes=_DIFF_BODY_LIMIT, - ) - if names is None or body is None: - raise ConfigError( - "Git worktree diff exceeded static output bounds or could not be read." + if names.payload is None: + reason, detail = _classify_diff_failure( + names, limit_reason="metadata_limit_exceeded", workspace=workspace ) - paths = sorted(_paths_from_name_status(names)) - diff_text = _decode_diff_body(body) - try: - _reject_binary_capability_paths(workspace, "HEAD", pathspec=pathspec) - except BinaryCapabilityDiffError as exc: - exc.changed_paths = tuple(paths) - exc.diff_text = diff_text - raise + raise DiffInputError( + DiffContext(completeness="unavailable", reason=reason, detail=detail) + ) + paths = sorted(_paths_from_name_status(names.payload)) + # The untracked inventory is cheap path metadata, independent of the diff + # body. Collecting it here rather than after the body read means a body + # that cannot be read still hands back the complete set of changed paths — + # a brand-new capability file appears in no `git diff` at all. untracked = _run_git_bounded_output( workspace, [ @@ -961,6 +1156,50 @@ def working_tree_context( path = os.fsdecode(raw_path) if path not in paths: paths.append(path) + body = _run_git_bounded_result( + workspace, + [ + *_SAFE_DIFF_CONFIG, + "diff", + *_DETERMINISTIC_DIFF_OPTIONS, + "HEAD", + "--", + *pathspec, + ], + max_output_bytes=_DIFF_BODY_LIMIT, + ) + if body.payload is None: + reason, detail = _classify_diff_failure( + body, limit_reason="body_limit_exceeded", workspace=workspace + ) + raise DiffInputError( + DiffContext( + changed_files=tuple(paths), + completeness="partial", + reason=reason, + detail=detail, + ) + ) + diff_text = _decode_diff_body(body.payload) + try: + _reject_binary_capability_paths(workspace, "HEAD", pathspec=pathspec) + except BinaryCapabilityDiffError as exc: + exc.changed_paths = tuple(paths) + exc.diff_text = diff_text + raise + except DiffInputError as exc: + # The binary-hiding guard could not run, so the body is not proven to + # cover every capability path. Carry what was read: a caller that can + # act on partial evidence should not have to re-collect it. + raise DiffInputError( + DiffContext( + changed_files=tuple(paths), + diff_text=diff_text, + completeness="partial", + reason=exc.context.reason, + detail=exc.context.detail, + ) + ) from exc return paths, diff_text @@ -1342,7 +1581,9 @@ def _resolved_diff_revspec(workspace: Path, revspec: str) -> str: raise ConfigError(f"Unsupported Git diff revision expression: {revspec!r}") commits = [commit_sha(workspace, part) for part in parts] if any(commit is None for commit in commits): - raise ConfigError(f"Git diff revision is unavailable: {revspec!r}") + raise _UnavailableRevisionError( + f"Git diff revision is unavailable: {revspec!r}" + ) return separator.join(commit for commit in commits if commit is not None) @@ -1387,6 +1628,16 @@ def _run_git_dir( ) +@dataclass(frozen=True) +class _BoundedGitResult: + """One bounded Git read, keeping why it failed instead of only that it did.""" + + payload: bytes | None + exceeded: bool = False + timed_out: bool = False + stderr: str = "" + + def _run_git_bounded_output( workspace: Path, args: list[str], @@ -1398,20 +1649,48 @@ def _run_git_bounded_output( ) -> bytes | None: """Run read-only Git plumbing without buffering unbounded stdout.""" + return _run_git_bounded_result( + workspace, + args, + max_output_bytes=max_output_bytes, + timeout=timeout, + allowed_returncodes=allowed_returncodes, + input=input, + ).payload + + +def _run_git_bounded_result( + workspace: Path, + args: list[str], + *, + max_output_bytes: int, + timeout: int = 60, + allowed_returncodes: tuple[int, ...] = (0,), + input: bytes | None = None, +) -> _BoundedGitResult: + """Run bounded Git plumbing and retain a bounded stderr excerpt. + + stderr is drained on its own thread under a small cap for the same reason + stdout is: an unread pipe deadlocks the child, and an unbounded one is a + memory hazard. The excerpt exists only so an input failure can be + classified and explained; it never becomes evidence. + """ + cmd = ["git", "--no-replace-objects", "-C", str(workspace), *args] env = _git_object_environment() try: process = subprocess.Popen( # noqa: S603 - fixed local Git argv, no shell. cmd, env=env, - stderr=subprocess.DEVNULL, + stderr=subprocess.PIPE, stdin=subprocess.PIPE if input is not None else subprocess.DEVNULL, stdout=subprocess.PIPE, ) - except OSError: - return None + except OSError as exc: + return _BoundedGitResult(payload=None, stderr=str(exc)) output = bytearray() + errors = bytearray() exceeded = False read_failed = False @@ -1433,8 +1712,20 @@ def _drain_stdout() -> None: except OSError: read_failed = True + def _drain_stderr() -> None: + assert process.stderr is not None + try: + while chunk := process.stderr.read(4096): + remaining = _GIT_STDERR_LIMIT - len(errors) + if remaining > 0: + errors.extend(chunk[:remaining]) + except OSError: + pass + reader = threading.Thread(target=_drain_stdout, daemon=True) reader.start() + error_reader = threading.Thread(target=_drain_stderr, daemon=True) + error_reader.start() write_failed = False def _write_stdin() -> None: @@ -1459,20 +1750,141 @@ def _write_stdin() -> None: process.kill() process.wait() reader.join() + error_reader.join() if writer is not None: writer.join() - return None + return _BoundedGitResult( + payload=None, + timed_out=True, + stderr=_decode_git_stderr(errors), + ) reader.join() + error_reader.join() if writer is not None: writer.join() + stderr_text = _decode_git_stderr(errors) if ( returncode not in allowed_returncodes or exceeded or read_failed or write_failed ): + return _BoundedGitResult( + payload=None, + exceeded=exceeded, + stderr=stderr_text, + ) + return _BoundedGitResult(payload=bytes(output), stderr=stderr_text) + + +def _decode_git_stderr(payload: bytes | bytearray) -> str: + """Return Git's diagnostic text as one safe, bounded, single-line string.""" + + text = bytes(payload).decode("utf-8", errors="replace") + collapsed = " ".join( + part + for part in "".join( + character if character.isprintable() else " " for character in text + ).split() + ) + if len(collapsed) > _GIT_STDERR_EXCERPT_CHARS: + collapsed = collapsed[:_GIT_STDERR_EXCERPT_CHARS].rstrip() + "…" + return collapsed + + +def _history_is_truncated(workspace: Path) -> bool | None: + """Whether this checkout's commit history is shallow. + + ``None`` when Git could not answer. That is not "no": it is the one case + where the caller has no basis to claim the histories are unrelated, so it + must not synthesize either repair. + """ + + result = _run_git( + workspace, ["rev-parse", "--is-shallow-repository"], check=False + ) + if result.returncode != 0: return None - return bytes(output) + answer = result.stdout.strip().casefold() + if answer == "true": + return True + if answer == "false": + return False + return None + + +def _redact_local_paths(text: str, workspace: Path) -> str: + """Keep local filesystem layout out of a diagnostic that ships in JSON.""" + + if not text: + return text + for absolute in (str(workspace.resolve()), str(workspace), str(Path.home())): + if absolute and absolute != os.sep: + text = text.replace(absolute, "") + return text + + +def _classify_diff_failure( + result: _BoundedGitResult, + *, + limit_reason: DiffInputReason, + workspace: Path | None = None, +) -> tuple[DiffInputReason, str]: + """Map one failed bounded Git read to a stable reason and a safe detail. + + Classification reads Git's own diagnostic rather than guessing from the + repository shape, so a shallow clone with no merge base and a partial clone + with unfetched blobs stop being reported as the same failure. + """ + + if result.exceeded: + return limit_reason, "" + detail = ( + _redact_local_paths(result.stderr, workspace) + if workspace is not None + else result.stderr + ) + if result.timed_out: + return "git_timeout", detail + lowered = result.stderr.casefold() + if "no merge base" in lowered: + # "No merge base" has two causes with opposite repairs, and Git reports + # them identically. A shallow checkout truncated a merge base that does + # exist — deepening restores it. Two genuinely unrelated roots have no + # common ancestor at all, and routing that to another fetch loops an + # agent forever, so it goes to whoever chose the base ref. + truncated = ( + _history_is_truncated(workspace) if workspace is not None else None + ) + if truncated is True: + return "merge_base_missing", detail + if truncated is False: + return "unrelated_histories", detail + return "git_failed", detail + if any( + marker in lowered + for marker in ( + "promisor remote", + "lazy fetching disabled", + "missing blob", + "unable to read", + "cannot read object", + "object file is empty", + ) + ): + return "objects_missing", detail + if any( + marker in lowered + for marker in ( + "unknown revision", + "ambiguous argument", + "not a valid object name", + "bad revision", + "bad object", + ) + ): + return "refs_missing", detail + return "git_failed", detail def _run_git( @@ -1560,11 +1972,15 @@ def staged_paths_under(workspace: Path, subdir: str) -> list[str]: "active_replace_refs", "archive_tree", "BinaryCapabilityDiffError", + "collect_diff_context", + "collect_revspec_diff_context", "commit_date", "commit_sha", "DefaultBaseDetection", "detect_default_base", "detect_default_base_with_notes", + "DiffContext", + "DiffInputError", "diff_context", "diff_revspec_context", "ensure_git_workspace", diff --git a/src/agents_shipgate/cli/verify/orchestrator.py b/src/agents_shipgate/cli/verify/orchestrator.py index 828e2cb4..a301f284 100644 --- a/src/agents_shipgate/cli/verify/orchestrator.py +++ b/src/agents_shipgate/cli/verify/orchestrator.py @@ -84,6 +84,7 @@ VerifierArtifact, VerifierBaseStatus, VerifierCapabilityReview, + VerifierDiffStatus, VerifierFixTask, applicability_for, merge_verdict_for, @@ -93,18 +94,28 @@ VerifyRunOutcome, build_verify_run_artifact, ) -from agents_shipgate.triggers import evaluate +from agents_shipgate.triggers import ( + ACTION_FORCE_RUN as TRIGGER_ACTION_FORCE_RUN, +) +from agents_shipgate.triggers import ( + INPUT_COMPLETE, + INPUT_PARTIAL, + INPUT_UNAVAILABLE, + evaluate, +) from .capability_review import build_capability_review from .fix_task import FORBIDDEN_SHORTCUTS, build_fix_task, is_pure_adoption_review from .git import ( + DiffContext, + DiffInputError, active_replace_refs, archive_tree, carries_manifest_like_yaml, + collect_diff_context, commit_date, commit_sha, detect_default_base_with_notes, - diff_context, ensure_git_workspace, git_path, merge_base_sha, @@ -263,6 +274,9 @@ def run_verify( diff_text="", manifest_present=False, user_requested=True, + # verify stopped before reading any diff, so the evaluator has no + # change set to reason about and must not report "no rules matched". + input_status=INPUT_UNAVAILABLE, ) message = ( f"Shipgate config not found at {_display_path(config_path, git_root)}. " @@ -279,6 +293,15 @@ def run_verify( trigger=trigger, base_status="not_requested", base_tree=None, + diff_status=VerifierDiffStatus( + completeness="unavailable", + reason="not_attempted", + detail="verify stopped at the missing manifest.", + remediation=( + "Point --config at the manifest, or initialize Shipgate, " + "then rerun." + ), + ), base_report=None, base_notes=[], report=None, @@ -326,7 +349,11 @@ def run_verify( base_capability_lock: CapabilityLockFileV1 | None = None base_notes: list[str] = [] diff_unavailable = False - diff_failure_action: HumanControlAction | None = None + # Every collector that fell short, paired with what its repair would need. + # The action and the headline are derived once from the worst of them, so + # a published ``diff_status`` can never disagree with the repair it + # authorizes. + diff_failures: list[tuple[DiffContext, str]] = [] head_exists = ref_exists(git_root, head) if not head_exists: @@ -335,6 +362,7 @@ def run_verify( diff_text="", manifest_present=True, user_requested=True, + input_status=INPUT_UNAVAILABLE, ) message = f"Head ref does not exist locally: {head}" verifier = _build_verifier( @@ -347,6 +375,15 @@ def run_verify( trigger=trigger, base_status="ref_missing", base_tree=None, + diff_status=VerifierDiffStatus( + completeness="unavailable", + reason="refs_missing", + detail=message, + remediation=( + "Fetch the head ref locally, then rerun verify." + ), + fetch_repairable=True, + ), base_report=None, base_notes=[message], report=None, @@ -420,24 +457,34 @@ def run_verify( if base: base_exists = ref_exists(git_root, base) if base_exists: - try: - changed_files, diff_text = diff_context(git_root, base, head) - except Exception as exc: # noqa: BLE001 - diff context degrades only. + collected = _collect_diff(git_root, base, head) + changed_files = list(collected.changed_files) + diff_text = collected.diff_text + if collected.completeness != "complete": + # The refs resolved, so the shortfall is about history depth, + # object availability, or Git itself — each of which has a + # different repair. Report which one instead of a single + # "could not be read". diff_unavailable = True base_status = "archive_failed" - detail = f"Could not collect diff for {base}...{head}: {exc}" - base_notes.append(detail) - diff_failure_action = HumanControlAction( - kind="review", - why=( - f"{detail}. The refs are present; fetching cannot repair " - "this deterministic input failure. Inspect the reported " - "Git configuration/resource issue before rerunning." - ), + diff_failures.append((collected, f"{base}...{head}")) + base_notes.append( + f"Could not collect the {base}...{head} diff in full. " + f"{collected.note}" ) else: diff_unavailable = True base_status = "ref_missing" + diff_failures.append( + ( + DiffContext( + completeness="unavailable", + reason="refs_missing", + detail=f"Base ref {base!r} is not available locally.", + ), + base, + ) + ) base_notes.append( f"Base ref {base!r} is not available locally; run with fetch-depth: 0 " "or fetch the base before verify." @@ -461,16 +508,33 @@ def run_verify( ) except Exception as exc: # noqa: BLE001 - local context degrades only. diff_unavailable = True - detail = f"Could not collect working-tree diff context: {exc}" - base_notes.append(detail) - diff_failure_action = HumanControlAction( - kind="review", - why=( - f"{detail}. Inspect the deterministic worktree-input " - "failure before rerunning; fetching refs cannot repair it." - ), + worktree_failure = _as_diff_context(exc) + # Whatever the failed collector did read still counts. Reporting + # "changed paths were collected" in the notes while handing the + # trigger an empty list would lose the path-rule match the paths + # exist to produce. + changed_files = _dedupe_paths( + [*changed_files, *worktree_failure.changed_files] + ) + diff_text = _join_diff_text(diff_text, worktree_failure.diff_text) + diff_failures.append((worktree_failure, head)) + base_notes.append( + f"Could not collect working-tree diff context. " + f"{worktree_failure.note}" ) + # A worktree shortfall is never softened by a committed-ref diff that did + # read cleanly: the two are unioned into one change set, so the union is + # only as complete as its weakest half — and the repair Shipgate authorizes + # has to be the repair for *that* half. Deriving it incrementally published + # a fetch_base action beside a diff_status no fetch could repair. + diff_input, diff_failure_expects = _worst_diff_failure(diff_failures) + diff_failure_action = ( + _diff_failure_action(diff_input, expects=diff_failure_expects) + if diff_input is not None + else None + ) + trigger = evaluate( paths=changed_files, diff_text=diff_text, @@ -479,6 +543,7 @@ def run_verify( # trigger stop-conditions from treating the canonical PR command as # passive repo discovery. user_requested=True, + input_status=_trigger_input_status(diff_input), ) verifier = _build_verifier( git_root=git_root, @@ -490,6 +555,7 @@ def run_verify( trigger=trigger, base_status=base_status, base_tree=base_tree, + diff_status=_diff_status_artifact(diff_input), base_report=base_report, base_notes=base_notes, report=None, @@ -512,6 +578,7 @@ def run_verify( trigger=trigger, base_status=base_status, base_tree=base_tree, + diff_status=_diff_status_artifact(diff_input), base_report=base_report, base_notes=base_notes, report=None, @@ -520,6 +587,7 @@ def run_verify( out_dir=out_dir, ci_mode=ci_mode, first_next_action_override=diff_failure_action, + headline_override=_diff_failure_headline(diff_input), worktree=not archive_head, rerun_options=rerun_options, ) @@ -554,6 +622,7 @@ def run_verify( trigger=trigger, base_status=base_status, base_tree=base_tree, + diff_status=_diff_status_artifact(diff_input), base_report=base_report, base_notes=base_notes, report=None, @@ -795,6 +864,7 @@ def capture_capability_lock(lock: CapabilityLockFileV1) -> None: trigger=trigger, base_status=base_status, base_tree=base_tree, + diff_status=_diff_status_artifact(diff_input), head_tree=head_tree, base_report=base_report, base_notes=base_notes, @@ -1740,6 +1810,137 @@ def _derive_verifier_control( ) +_TRIGGER_INPUT_STATUS: dict[str, str] = { + "complete": INPUT_COMPLETE, + "partial": INPUT_PARTIAL, + "unavailable": INPUT_UNAVAILABLE, +} + + +def _collect_diff(git_root: Path, base: str, head: str) -> DiffContext: + """Collect a committed-ref diff, turning any surprise into a typed state.""" + + try: + return collect_diff_context(git_root, base, head) + except Exception as exc: # noqa: BLE001 - diff context degrades only. + return _as_diff_context(exc) + + +def _as_diff_context(exc: Exception) -> DiffContext: + """Represent an unexpected collection failure in the same typed vocabulary.""" + + if isinstance(exc, DiffInputError): + return exc.context + return DiffContext( + completeness="unavailable", + reason="git_failed", + detail=str(exc) or exc.__class__.__name__, + ) + + +_DIFF_COMPLETENESS_ORDER = {"complete": 0, "partial": 1, "unavailable": 2} + + +def _worst_diff_failure( + failures: list[tuple[DiffContext, str]], +) -> tuple[DiffContext | None, str]: + """Pick the one failure the artifact must report, with its repair target. + + Halves of a single change set are unioned, so the union is only as complete + as its weakest half. Among equally incomplete halves the one a fetch cannot + repair wins: authorizing another fetch against a deterministic failure is + the loop this ordering exists to prevent. + """ + + if not failures: + return None, "" + return max( + failures, + key=lambda pair: ( + _DIFF_COMPLETENESS_ORDER[pair[0].completeness], + 0 if pair[0].fetch_repairable else 1, + ), + ) + + +def _matched_diff_evidence(trigger: dict[str, Any]) -> bool: + """Whether any matched rule was decided by the change set itself. + + ``force_run`` fires from the presence of a manifest, which is repository + state rather than diff evidence. Separating the two keeps a headline from + claiming a diff showed something when no diff was read. + """ + + return any( + match.get("action") != TRIGGER_ACTION_FORCE_RUN + for match in trigger.get("matched_rules", []) + if isinstance(match, dict) + ) + + +def _diff_failure_headline(context: DiffContext | None) -> str | None: + """Summarize a diff-input failure in the terms its control route uses. + + The generic failed-scan headline says "human review required" for every + unknown verdict, which contradicts an artifact whose control state is + ``agent_action_required`` with a ``fetch_base`` next action. The headline + and the route are both derived from the same classified failure here. + """ + + if context is None: + return None + if context.fetch_repairable: + return ( + f"Shipgate could not read the PR diff ({context.reason}); the " + "history it needs is not available locally yet, so no verdict was " + "reached. Make it available, then rerun verify." + ) + return ( + f"Shipgate could not read the PR diff ({context.reason}); fetching " + "cannot repair this, so no verdict was reached and a human must " + "resolve the input." + ) + + +def _diff_status_artifact(context: DiffContext | None) -> VerifierDiffStatus: + """Project one diff-acquisition attempt onto the verifier artifact.""" + + context = context or DiffContext() + return VerifierDiffStatus( + completeness=context.completeness, + reason=context.reason, + detail=context.detail or None, + remediation=context.remediation or None, + fetch_repairable=context.fetch_repairable, + ) + + +def _trigger_input_status(context: DiffContext | None) -> str: + if context is None: + return INPUT_COMPLETE + return _TRIGGER_INPUT_STATUS[context.completeness] + + +def _diff_failure_action( + context: DiffContext, *, expects: str +) -> AgentControlAction: + """Route a diff-input failure to the action that can actually repair it. + + A missing merge base or an unfetched partial-clone object is repaired by + making history available locally — that is agent work, not review work. + Everything else is a deterministic failure that fetching cannot touch, so + it goes to a human with the Git diagnostic attached. + """ + + if context.fetch_repairable: + return CodingAgentFetchBaseAction( + kind="fetch_base", + expects=expects, + why=context.note, + ) + return HumanControlAction(kind="review", why=context.note) + + def _build_verifier( *, git_root: Path, @@ -1751,6 +1952,7 @@ def _build_verifier( trigger: dict[str, Any], base_status: VerifierBaseStatus, base_tree: str | None, + diff_status: VerifierDiffStatus | None = None, head_tree: str | None = None, base_report: Path | None, base_notes: list[str], @@ -1857,6 +2059,7 @@ def _build_verifier( head_ref=head, changed_files=changed_files, diff_text_available=bool(diff_text), + diff_status=diff_status or VerifierDiffStatus(), trigger=trigger, base_status=base_status, base_tree_sha=base_tree, @@ -3126,30 +3329,25 @@ def run_preview( changed_files: list[str] = [] diff_text = "" notes: list[str] = [] - diff_unavailable = False - diff_failure_requires_review = False - if base or head: + diff_input: DiffContext | None = None + if base: try: git_root = ensure_git_workspace(root) - head_ref = head or "HEAD" - if base: - if ref_exists(git_root, base) and ref_exists(git_root, head_ref): - changed_files, diff_text = diff_context(git_root, base, head_ref) - else: - diff_unavailable = True - notes.append( - "Preview diff unavailable: base/head ref is not available locally." - ) + collected = _collect_diff(git_root, base, head or "HEAD") except Exception as exc: # noqa: BLE001 - preview must never crash. - diff_unavailable = True - diff_failure_requires_review = True - notes.append(f"Preview diff unavailable: {exc}") + collected = _as_diff_context(exc) + changed_files = list(collected.changed_files) + diff_text = collected.diff_text + if collected.completeness != "complete": + diff_input = collected + notes.append(f"Preview diff unavailable: {collected.note}") trigger = evaluate( paths=changed_files, diff_text=diff_text, manifest_present=manifest_present, user_requested=True, + input_status=_trigger_input_status(diff_input), ) # Trigger previews may recommend detect/init as a generic recovery path. @@ -3167,24 +3365,51 @@ def run_preview( pr_comment_style=pr_comment_style, ) - if diff_unavailable and manifest_present and diff_failure_requires_review: - why = ( - "Preview could not collect the requested deterministic diff even " - "though ref availability was not the problem. Inspect the reported " - "Git configuration/resource failure before rerunning." - ) - next_action = HumanControlAction(kind="review", why=why) - headline = "Shipgate preview could not safely inspect the requested PR diff." - elif diff_unavailable and manifest_present: - next_action: AgentControlAction = CodingAgentFetchBaseAction( - kind="fetch_base", - expects=base or head or "the requested base and head refs", - why=( - "Preview could not inspect the requested PR diff; make the base " - "and head refs available locally, then rerun preview or verify." + # A diff Shipgate could not read outranks every adoption route below it, + # whether or not this workspace has a manifest. Falling through to "Shipgate + # is not configured here" would answer a question nobody asked and bury the + # fact that the PR was never inspected — and an unadopted repo reached over a + # shallow or blobless clone is exactly where this failure lands. + if diff_input is not None: + next_action: AgentControlAction = _diff_failure_action( + diff_input, + expects=( + f"{base}...{head or 'HEAD'}" + if base + else (head or "the requested base and head refs") ), ) - headline = "Shipgate preview could not inspect the requested PR diff." + read = ( + "could only partly read" + if diff_input.completeness == "partial" + else "could not read" + ) + # Partial evidence can still carry a sound run verdict — a matched path + # rule needs no diff body — and the evaluator publishes it. Saying "no + # relevance verdict was reached" alongside `should_run: true` would make + # the headline contradict the artifact it summarizes. But the run may + # rest on evidence that has nothing to do with the diff: an adopted + # repository force-runs on the manifest alone, with no paths read at + # all, so naming the paths there would attribute the verdict to + # evidence that does not exist. + if not trigger.get("run_shipgate"): + outcome = "no relevance verdict was reached" + elif _matched_diff_evidence(trigger): + outcome = ( + "the change it did read already shows an agent-capability " + "surface, so relevance is established; recover the full diff " + "before trusting any merge verdict" + ) + else: + outcome = ( + "this workspace is already configured for Shipgate, so " + "verification must run regardless; recover the full diff " + "before trusting any merge verdict" + ) + headline = ( + f"Shipgate preview {read} the requested PR diff " + f"({diff_input.reason}); {outcome}." + ) elif manifest_present: next_action = CodingAgentCommandAction( kind="verify", @@ -3238,6 +3463,7 @@ def run_preview( head_ref=head or "HEAD", changed_files=changed_files, diff_text_available=bool(diff_text), + diff_status=_diff_status_artifact(diff_input), trigger=trigger, base_status="not_requested", base_notes=notes, diff --git a/src/agents_shipgate/schemas/contract.py b/src/agents_shipgate/schemas/contract.py index 1af2ba00..23d9b917 100644 --- a/src/agents_shipgate/schemas/contract.py +++ b/src/agents_shipgate/schemas/contract.py @@ -64,7 +64,7 @@ AGENT_BOUNDARY_RESULT_SCHEMA_PATH: Literal["docs/agent-boundary-result-schema.v1.json"] = ( "docs/agent-boundary-result-schema.v1.json" ) -TRIGGER_CATALOG_SCHEMA_VERSION: Literal["0.2"] = "0.2" +TRIGGER_CATALOG_SCHEMA_VERSION: Literal["0.3"] = "0.3" # Fields of the SHARED agent result (``agent_result_schema_path``). The graded # ``pending_review[]`` obligation is deliberately absent: it exists only on # ``shipgate.agent_boundary_result/v1``, because adding it to the shared base diff --git a/src/agents_shipgate/schemas/verifier.py b/src/agents_shipgate/schemas/verifier.py index 85aaeb45..b3cb6f97 100644 --- a/src/agents_shipgate/schemas/verifier.py +++ b/src/agents_shipgate/schemas/verifier.py @@ -24,6 +24,32 @@ ] VerifierExecution = Literal["not_run", "succeeded", "skipped", "failed"] VerifierHeadStatus = VerifierExecution +# How completely the compared change set was read, and — when it was not read +# in full — why. This is an input-acquisition fact, never a verdict: an +# unreadable diff says nothing about what the PR contains, so a consumer must +# not read anything but ``complete`` as evidence that a PR is unrelated to +# agent capabilities. +# ``unknown`` is reachable only through legacy normalization: a pre-v0.7 +# artifact recorded no input health at all, and saying so is the one honest +# answer. Current emitters never produce it. Like every value other than +# ``complete`` it withholds permission to read a negative trigger verdict. +DiffCompleteness = Literal["complete", "partial", "unavailable", "unknown"] +DiffInputReason = Literal[ + # Verification stopped before it read any diff (e.g. no manifest to gate + # against). Nothing failed in Git; nothing about the change set is known. + "not_attempted", + "refs_missing", + # A shallow checkout truncated a merge base that does exist (deepen), as + # against ``unrelated_histories``, where no common ancestor exists at all + # and no fetch can create one. + "merge_base_missing", + "unrelated_histories", + "objects_missing", + "metadata_limit_exceeded", + "body_limit_exceeded", + "git_timeout", + "git_failed", +] MergeVerdict = Literal[ "mergeable", "human_review_required", @@ -288,6 +314,67 @@ class VerifierCapabilityReview(BaseModel): notes: list[str] = Field(default_factory=list) +# Only these three describe history or objects that a fetch can make local. +# The rest are deterministic failures that another fetch cannot touch, so a +# ``fetch_repairable`` claim about them is rejected at construction rather than +# published as an instruction that loops. +_FETCH_REPAIRABLE_REASONS = frozenset( + {"refs_missing", "merge_base_missing", "objects_missing"} +) + + +class VerifierDiffStatus(BaseModel): + """Whether the compared change set was actually read, and why not. + + Emitted on every verifier artifact so automation never has to infer input + health from a verdict. ``completeness: "complete"`` is the only value that + licenses reading a negative trigger result — anything else means the + evidence the verdict would rest on was missing, and the artifact says so + instead of reporting "nothing in this PR signals a tool-surface change". + """ + + model_config = ConfigDict(extra="forbid") + + completeness: DiffCompleteness = "complete" + # Present exactly when the diff was read neither completely nor not-at-all: + # ``complete`` has nothing to explain, and ``unknown`` has no record to + # explain it with. + reason: DiffInputReason | None = None + # Bounded, path-redacted excerpt of Git's own diagnostic. Diagnostics only. + detail: str | None = None + # The precise repair, e.g. deepen history or hydrate partial-clone objects. + remediation: str | None = None + # Whether making refs/objects available locally can repair the failure. + # ``False`` routes to a human instead of another fetch attempt. + fetch_repairable: bool = False + + @model_validator(mode="after") + def _reason_tracks_completeness(self) -> VerifierDiffStatus: + explainable = self.completeness in {"partial", "unavailable"} + if explainable != (self.reason is not None): + raise ValueError( + "VerifierDiffStatus.reason must be present exactly when the " + "diff was partially read or unavailable" + ) + if self.completeness != "complete" and self.fetch_repairable and ( + self.reason not in _FETCH_REPAIRABLE_REASONS + ): + raise ValueError( + f"VerifierDiffStatus.fetch_repairable is not true for " + f"{self.reason!r}: fetching cannot repair it" + ) + return self + + @classmethod + def unknown(cls) -> VerifierDiffStatus: + """The input health of an artifact that predates v0.7 reporting.""" + + return cls( + completeness="unknown", + detail="This artifact predates verifier v0.7 input-health reporting.", + ) + + AgentStopReason = Literal[ "self_approval_prohibited", "blocked_findings", @@ -487,7 +574,7 @@ class VerifierArtifact(BaseModel): }, ) - verifier_schema_version: Literal["0.6"] = "0.6" + verifier_schema_version: Literal["0.7"] = "0.7" static_analysis_only: Literal[True] = True runtime_behavior_verified: Literal[False] = False static_verdict_disclaimer: str = STATIC_VERDICT_DISCLAIMER @@ -503,6 +590,11 @@ class VerifierArtifact(BaseModel): head_ref: str = "HEAD" changed_files: list[str] = Field(default_factory=list) diff_text_available: bool = False + # Required, so a current artifact cannot omit the input-health contract: + # a payload with no ``diff_status`` would be indistinguishable from one + # that read its diff cleanly. Pre-v0.7 artifacts are normalized to + # ``VerifierDiffStatus.unknown()`` on the legacy path instead. + diff_status: VerifierDiffStatus trigger: dict[str, Any] = Field(default_factory=dict) base_status: VerifierBaseStatus = "not_requested" base_tree_sha: str | None = None @@ -541,14 +633,20 @@ def _normalize_legacy_control(cls, data: Any) -> Any: return data normalized = dict(data) legacy_version = normalized.get("verifier_schema_version") - legacy = legacy_version in {"0.1", "0.2", "0.3", "0.4", "0.5"} + legacy = legacy_version in {"0.1", "0.2", "0.3", "0.4", "0.5", "0.6"} if not legacy: - # Current v0.6 artifacts must already carry the authoritative + # Current v0.7 artifacts must already carry the authoritative # control union. Silently synthesizing a missing or malformed # current control would turn an internal consistency failure into - # a trusted handoff. Only the frozen v0.2 reader is normalized. + # a trusted handoff. Only frozen prior readers are normalized. return normalized - normalized["verifier_schema_version"] = "0.6" + normalized["verifier_schema_version"] = "0.7" + # A pre-v0.7 artifact recorded nothing about whether its diff was + # readable. Defaulting that to ``complete`` would manufacture the one + # claim the whole field exists to stop. + normalized.setdefault( + "diff_status", VerifierDiffStatus.unknown().model_dump(mode="json") + ) normalized.setdefault( "authorization", AuthorizationEvaluationV1.not_requested().model_dump(mode="json"), diff --git a/src/agents_shipgate/triggers.py b/src/agents_shipgate/triggers.py index 0a1116f3..1463e48a 100644 --- a/src/agents_shipgate/triggers.py +++ b/src/agents_shipgate/triggers.py @@ -53,6 +53,26 @@ {ACTION_FORCE_RUN, ACTION_RUN, ACTION_SKIP, ACTION_DRY_RUN} ) +# How complete the diff evidence handed to the evaluator is. Mirrors +# ``BoundaryChangeSet.completeness`` and ``DiffContext.completeness`` so every +# input path in the product describes a partially-read diff the same way. +# +# complete — every changed path and the full diff body were read +# partial — some evidence is missing (typically: paths but no body) +# unavailable — nothing about the change set was established +# +# Rule matching is monotone in path and diff evidence: adding evidence can only +# add matches. So a *run* verdict reached from incomplete evidence stays sound, +# while any *skip* verdict does not — the missing bytes are exactly what would +# have flipped it. That asymmetry is what ``evaluation_status`` reports. +INPUT_COMPLETE = "complete" +INPUT_PARTIAL = "partial" +INPUT_UNAVAILABLE = "unavailable" +VALID_INPUT_STATUSES = frozenset({INPUT_COMPLETE, INPUT_PARTIAL, INPUT_UNAVAILABLE}) + +EVALUATION_EVALUATED = "evaluated" +EVALUATION_NOT_EVALUATED = "not_evaluated" + # Semantic class of the surface a rule describes. Rule IDs are stable audit # labels, not a type system: consumers must switch on ``surface_class`` instead # of maintaining ID allow-lists that silently miss newly-added adapters. @@ -166,7 +186,7 @@ def _contains_detect_returns(pred: Any) -> bool: def _next_action( *, - run: bool, + run: bool | None, dry_run_recommended: bool, skip_reason: str | None, manifest_present: bool, @@ -183,6 +203,11 @@ def _next_action( coding agent can route setup. ``command`` is ``None`` when no action is warranted. """ + if run is None: + # The verdict was withheld because the diff was never read. The only + # honest next step is to repair the input, and the caller that failed + # to read it is the one that knows how — so no command is invented here. + return {"kind": "input_required", "command": None, "why": rationale} if run: if manifest_present: return { @@ -321,32 +346,53 @@ def evaluate( detect_result: dict[str, Any] | None = None, user_requested: bool = False, triggers: dict[str, Any] | None = None, + input_status: str = INPUT_COMPLETE, ) -> dict[str, Any]: """Evaluate the trigger catalog against a snapshot of repo state. + ``input_status`` declares how complete that snapshot is (see + ``INPUT_COMPLETE`` / ``INPUT_PARTIAL`` / ``INPUT_UNAVAILABLE``). A caller + that could not read the PR diff must say so: without it the evaluator sees + an empty path list and an empty diff body, which are indistinguishable from + a PR that genuinely changed nothing relevant, and it would report + ``skip_reason: "no_match"`` — "nothing in this PR signals a tool-surface + change" — about a PR it never read. + Returns a dict with: - ``schema_version`` (str) — the trigger catalog's schema version. - - ``should_run`` (bool) — friendly alias of ``run_shipgate`` (same - value); kept so consumers reading either field agree. - - ``run_shipgate`` (bool) — final verdict. + - ``input_status`` (str) — echoed back: ``complete``, ``partial`` or + ``unavailable``. + - ``evaluation_status`` (str) — ``evaluated`` when the verdict is + supported by the evidence that was actually read, otherwise + ``not_evaluated``. It is ``not_evaluated`` exactly when the inputs + were incomplete *and* the rules that did run produced no reason to + run: that combination proves nothing, so no verdict is published. + - ``should_run`` (bool|None) — friendly alias of ``run_shipgate`` (same + value); kept so consumers reading either field agree. ``None`` when + ``evaluation_status`` is ``not_evaluated``. + - ``run_shipgate`` (bool|None) — final verdict; ``None`` when not + evaluated. - ``force_run`` (bool) — a ``force_run`` rule matched and was not overridden by the stop block (opted-in repo → run on every PR). - ``dry_run_recommended`` (bool) — true when a ``dry_run`` rule fired and no ``run_shipgate``/``force_run``/``skip_shipgate`` rule did. Callers that want to be helpful can propose a non-mutating ``scan`` even though ``run_shipgate`` is false. - - ``skip`` (bool) — inverse of ``should_run``; convenience for - consumers that branch on the skip case. - - ``skip_reason`` (str|None) — ``None`` when running; otherwise a - stable token: ``stop_conditions``, ``skip_rule``, ``dry_run_only`` - or ``no_match``. + - ``skip`` (bool|None) — inverse of ``should_run``; convenience for + consumers that branch on the skip case. ``None`` when not evaluated. + - ``skip_reason`` (str|None) — ``None`` when running *and* when the + verdict was withheld; otherwise a stable token: ``stop_conditions``, + ``skip_rule``, ``dry_run_only`` or ``no_match``. ``no_match`` is + never emitted for inputs that were not fully read. - ``stop_conditions_fired`` (bool) — whether the explicit stop block held; this beats every rule action. - ``stop_conditions_evaluated`` (bool) — whether the stop block could be fully evaluated. ``False`` when the block references - ``detect_returns`` but no ``detect_result`` was supplied; in that - case the evaluator never stops (``stop_conditions_fired`` stays + ``detect_returns`` but no ``detect_result`` was supplied, and + ``False`` whenever ``input_status`` is not ``complete`` because the + block reasons over the very path evidence that is missing. In those + cases the evaluator never stops (``stop_conditions_fired`` stays ``False``) and the caller knows the stop verdict is unknown rather than "evaluated and did not hold". - ``rationale`` (str) — single-sentence explanation. @@ -356,16 +402,24 @@ def evaluate( are present in ``diff_text`` (sorted, de-duplicated). - ``next_action`` (dict) — the single recommended next step as ``{kind, command, why}`` (``kind`` is ``command``/``stop``/ - ``none``); a deterministic projection of the verdict. + ``none``/``input_required``); a deterministic projection of the + verdict. Action precedence (highest first): ``stop_conditions`` → skip; ``force_run`` → run (overrides skip; used by manifest-present); ``skip_shipgate`` → skip (beats ``run_shipgate``); ``run_shipgate`` - → run; ``dry_run`` → skip + ``dry_run_recommended``. + → run; ``dry_run`` → skip + ``dry_run_recommended``. Incomplete input + then withholds any resulting skip. """ if triggers is None: triggers = load_triggers() + if input_status not in VALID_INPUT_STATUSES: + raise ConfigError( + f"Unknown trigger input_status {input_status!r}; expected one of " + f"{sorted(VALID_INPUT_STATUSES)}." + ) paths = paths or [] + inputs_complete = input_status == INPUT_COMPLETE matched: list[dict[str, Any]] = [] for rule in triggers.get("rules", []): @@ -395,8 +449,12 @@ def evaluate( # cannot conclude "non-agent project" — so we never stop on it, and we # report stop_conditions_evaluated=False so consumers can tell the # difference between "evaluated, did not hold" and "could not evaluate". - stop_conditions_evaluated = bool(stop_payload) and ( - detect_result is not None or not _contains_detect_returns(stop_payload) + stop_conditions_evaluated = ( + bool(stop_payload) + and inputs_complete + and ( + detect_result is not None or not _contains_detect_returns(stop_payload) + ) ) stop_fired = stop_conditions_evaluated and _eval_predicate( stop_payload, @@ -457,10 +515,37 @@ def evaluate( "No rules matched; nothing in this PR signals a tool-surface change." ) + verdict: bool | None = run + evaluation_status = EVALUATION_EVALUATED + if not inputs_complete and not run: + # Everything below the run verdicts rests on evidence that was never + # read. "No rules matched" and "only docs changed" are claims about a + # diff; without the diff they are claims about nothing. Withhold the + # verdict rather than publish an unfalsifiable skip. + verdict = None + skip_reason = None + # The advisory dry-run recommendation is derived from the same withheld + # skip, so it is suppressed too. Nothing is lost: the rule that fired + # is still listed in ``matched_rules``. + dry_run_recommended = False + evaluation_status = EVALUATION_NOT_EVALUATED + missing = ( + "the change set could not be read, so there was no path or diff " + "evidence to match against" + if input_status == INPUT_UNAVAILABLE + else "the change set was read only in part, so every rule that " + "depends on the missing evidence could not fire" + ) + rationale = ( + f"Trigger rules were not evaluated: {missing}. That is not " + "evidence the PR is unrelated to agent capabilities — repair the " + "diff input and re-evaluate." + ) + # ``should_run`` is a friendlier alias of ``run_shipgate`` (identical # value); both are kept so 0.x consumers reading either field agree. next_action = _next_action( - run=run, + run=verdict, dry_run_recommended=dry_run_recommended, skip_reason=skip_reason, manifest_present=manifest_present, @@ -472,9 +557,11 @@ def evaluate( ) return { "schema_version": triggers.get("schema_version"), - "should_run": run, - "run_shipgate": run, - "skip": not run, + "input_status": input_status, + "evaluation_status": evaluation_status, + "should_run": verdict, + "run_shipgate": verdict, + "skip": None if verdict is None else not verdict, "force_run": has_force_run and not stop_fired, "dry_run_recommended": dry_run_recommended, "skip_reason": skip_reason, @@ -488,6 +575,14 @@ def evaluate( } +def _verdict_label(result: dict[str, Any]) -> str: + """Render the run/skip/withheld verdict for human output.""" + + if result.get("evaluation_status") == EVALUATION_NOT_EVALUATED: + return "NOT EVALUATED" + return "RUN" if result.get("run_shipgate") else "SKIP" + + def _git_diff_context( revspec: str | None, *, cwd: Path | None = None ) -> tuple[list[str], str]: @@ -661,7 +756,7 @@ def main(argv: list[str] | None = None) -> int: print(json.dumps(result, indent=2)) return 0 - verdict = "RUN" if result["run_shipgate"] else "SKIP" + verdict = _verdict_label(result) print(f"Verdict: {verdict}") print(f"Rationale: {result['rationale']}") if result["matched_rules"]: diff --git a/tests/golden/codex_boundary_result/agents_requirement_removed.json b/tests/golden/codex_boundary_result/agents_requirement_removed.json index af25c711..fd60623c 100644 --- a/tests/golden/codex_boundary_result/agents_requirement_removed.json +++ b/tests/golden/codex_boundary_result/agents_requirement_removed.json @@ -108,7 +108,9 @@ ], "source_artifacts": {}, "trigger": { - "schema_version": "0.2", + "schema_version": "0.3", + "input_status": "complete", + "evaluation_status": "evaluated", "should_run": true, "run_shipgate": true, "skip": false, diff --git a/tests/golden/codex_boundary_result/docs_only.json b/tests/golden/codex_boundary_result/docs_only.json index ac80cdd1..2fa28b64 100644 --- a/tests/golden/codex_boundary_result/docs_only.json +++ b/tests/golden/codex_boundary_result/docs_only.json @@ -74,7 +74,9 @@ ], "source_artifacts": {}, "trigger": { - "schema_version": "0.2", + "schema_version": "0.3", + "input_status": "complete", + "evaluation_status": "evaluated", "should_run": false, "run_shipgate": false, "skip": true, diff --git a/tests/golden/codex_boundary_result/github_action_removed.json b/tests/golden/codex_boundary_result/github_action_removed.json index 2cb1b681..33daa88f 100644 --- a/tests/golden/codex_boundary_result/github_action_removed.json +++ b/tests/golden/codex_boundary_result/github_action_removed.json @@ -115,7 +115,9 @@ ], "source_artifacts": {}, "trigger": { - "schema_version": "0.2", + "schema_version": "0.3", + "input_status": "complete", + "evaluation_status": "evaluated", "should_run": true, "run_shipgate": true, "skip": false, diff --git a/tests/golden/codex_boundary_result/malformed_toml.json b/tests/golden/codex_boundary_result/malformed_toml.json index 0d631194..29733ace 100644 --- a/tests/golden/codex_boundary_result/malformed_toml.json +++ b/tests/golden/codex_boundary_result/malformed_toml.json @@ -111,7 +111,9 @@ ], "source_artifacts": {}, "trigger": { - "schema_version": "0.2", + "schema_version": "0.3", + "input_status": "complete", + "evaluation_status": "evaluated", "should_run": true, "run_shipgate": true, "skip": false, diff --git a/tests/golden/codex_boundary_result/mcp_auto_approve_write.json b/tests/golden/codex_boundary_result/mcp_auto_approve_write.json index 03807409..6d26eeb6 100644 --- a/tests/golden/codex_boundary_result/mcp_auto_approve_write.json +++ b/tests/golden/codex_boundary_result/mcp_auto_approve_write.json @@ -119,7 +119,9 @@ ], "source_artifacts": {}, "trigger": { - "schema_version": "0.2", + "schema_version": "0.3", + "input_status": "complete", + "evaluation_status": "evaluated", "should_run": true, "run_shipgate": true, "skip": false, diff --git a/tests/golden/codex_boundary_result/network_wildcard.json b/tests/golden/codex_boundary_result/network_wildcard.json index 4459c646..5bd9a7dc 100644 --- a/tests/golden/codex_boundary_result/network_wildcard.json +++ b/tests/golden/codex_boundary_result/network_wildcard.json @@ -113,7 +113,9 @@ ], "source_artifacts": {}, "trigger": { - "schema_version": "0.2", + "schema_version": "0.3", + "input_status": "complete", + "evaluation_status": "evaluated", "should_run": true, "run_shipgate": true, "skip": false, diff --git a/tests/golden/codex_boundary_result/python_refactor.json b/tests/golden/codex_boundary_result/python_refactor.json index 072215c7..a9f19911 100644 --- a/tests/golden/codex_boundary_result/python_refactor.json +++ b/tests/golden/codex_boundary_result/python_refactor.json @@ -74,7 +74,9 @@ ], "source_artifacts": {}, "trigger": { - "schema_version": "0.2", + "schema_version": "0.3", + "input_status": "complete", + "evaluation_status": "evaluated", "should_run": false, "run_shipgate": false, "skip": true, diff --git a/tests/golden/codex_boundary_result/unknown_permission_key.json b/tests/golden/codex_boundary_result/unknown_permission_key.json index 4ea8fb64..3abe3433 100644 --- a/tests/golden/codex_boundary_result/unknown_permission_key.json +++ b/tests/golden/codex_boundary_result/unknown_permission_key.json @@ -108,7 +108,9 @@ ], "source_artifacts": {}, "trigger": { - "schema_version": "0.2", + "schema_version": "0.3", + "input_status": "complete", + "evaluation_status": "evaluated", "should_run": true, "run_shipgate": true, "skip": false, diff --git a/tests/integration/github_action/test_agent_result.py b/tests/integration/github_action/test_agent_result.py index bcd815cb..7ce8ccef 100644 --- a/tests/integration/github_action/test_agent_result.py +++ b/tests/integration/github_action/test_agent_result.py @@ -34,6 +34,7 @@ from agents_shipgate.schemas.verifier import ( VerifierArtifact, VerifierCapabilityReview, + VerifierDiffStatus, VerifierFixTask, ) from scripts.github_action_outputs import extract_outputs, merge_verdict_policy_exit_code @@ -461,6 +462,7 @@ def _verifier( ) return VerifierArtifact( workspace="/tmp/workspace", + diff_status=VerifierDiffStatus(), config="shipgate.yaml", base_ref="origin/main", head_ref="HEAD", diff --git a/tests/test_adapter_static_only.py b/tests/test_adapter_static_only.py index fe554e01..1c71ded1 100644 --- a/tests/test_adapter_static_only.py +++ b/tests/test_adapter_static_only.py @@ -257,9 +257,9 @@ class AllowedException: AllowedException( relative_path="cli/verify/git.py", surface="attr_call:subprocess.Popen", - line=1404, + line=1682, snippet=( - "subprocess.Popen(cmd, env=env, stderr=subprocess.DEVNULL, " + "subprocess.Popen(cmd, env=env, stderr=subprocess.PIPE, " "stdin=subprocess.PIPE if input is not None else " "subprocess.DEVNULL, stdout=subprocess.PIPE)" ), @@ -268,13 +268,15 @@ class AllowedException: "incrementally and kills them at a hard byte or wall-clock " "bound. It covers diff/name/attribute/inventory reads and " "retained-manifest discovery without a shell, user-code " - "execution, or fetch." + "execution, or fetch. stderr is piped (not discarded) and " + "drained on its own thread under a small cap so an input " + "failure can be classified; the excerpt is diagnostic only." ), ), AllowedException( relative_path="cli/verify/git.py", surface="attr_call:subprocess.run", - line=1514, + line=1926, snippet=( "subprocess.run(cmd, capture_output=capture_output, check=check, " "env=env, input=input, stderr=stderr, stdin=stdin, stdout=stdout, " @@ -396,7 +398,7 @@ class AllowedException: AllowedException( relative_path="triggers.py", surface="attr_call:importlib.resources.files", - line=97, + line=117, snippet="files('agents_shipgate')", rationale=( "Resolves the bundled trigger catalog (docs/triggers.json) " diff --git a/tests/test_agent_handoff.py b/tests/test_agent_handoff.py index 5280d353..07fe71fe 100644 --- a/tests/test_agent_handoff.py +++ b/tests/test_agent_handoff.py @@ -151,7 +151,8 @@ def _authorized_verifier_payload() -> dict: release["decision"] = "review_required" release["reason"] = "A protected workflow change requires human review." release["review_items"] = release.pop("blockers") - payload["verifier_schema_version"] = "0.6" + payload["verifier_schema_version"] = "0.7" + payload["diff_status"] = {"completeness": "complete"} payload["decision"] = "review_required" payload["merge_verdict"] = "human_review_required" payload["control"] = derive_agent_control( @@ -384,6 +385,7 @@ def test_preview_handoff_carries_standing_forbidden_lists() -> None: "merge_verdict": "unknown", "applicability": "not_evaluated", "can_merge_without_human": False, + "diff_status": {"completeness": "complete"}, "authorization": AuthorizationEvaluationV1.not_requested().model_dump(mode="json"), "control": derive_agent_control( reason="Configure Agents Shipgate before verification.", diff --git a/tests/test_agent_instructions_apply.py b/tests/test_agent_instructions_apply.py index e27322fb..32f43509 100644 --- a/tests/test_agent_instructions_apply.py +++ b/tests/test_agent_instructions_apply.py @@ -204,7 +204,7 @@ def test_local_contract_renderer_has_required_fields() -> None: assert payload["host_grants_inventory_schema_version"] == "0.2" assert payload["host_grants_baseline_schema_version"] == "0.2" assert payload["host_grants_drift_schema_version"] == "0.2" - assert payload["trigger_catalog_schema_version"] == "0.2" + assert payload["trigger_catalog_schema_version"] == "0.3" assert payload["gating_signal"] == "release_decision.decision" assert payload["default_paths"]["local_contract"] == ".shipgate/agent-contract.json" assert payload["verifier_read_order"] == [ diff --git a/tests/test_agent_instructions_renderers.py b/tests/test_agent_instructions_renderers.py index 64c55e45..1a3fd47b 100644 --- a/tests/test_agent_instructions_renderers.py +++ b/tests/test_agent_instructions_renderers.py @@ -45,7 +45,7 @@ REPO_ROOT = Path(__file__).resolve().parent.parent EXPECTED_CLAUDE_CODE_SKILL_RENDER_SHA256 = { ".claude/skills/agents-shipgate/SKILL.md": ( - "58ea3b6bba89078ec54d6b5493ffebf9250d9619fbacef5090285b009e58cdcd" + "f6771ad16589854d51604cf35af0d78022b6a28e5ffa97a1d0c429ee6b6107da" ), ".claude/skills/agents-shipgate/ci-recipes/advisory-pr-comment.yml": ( # Renders {{ shipgate_version }}; changes on every version bump. @@ -55,7 +55,7 @@ "53296f41b7c2bc8538555a4361707de8b990748b7a5d80ae4ce066af83af8fa7" ), ".claude/skills/agents-shipgate/prompts/decide-shipgate-relevance.md": ( - "370a81cf1c35212584702ca89c5476f3cd6c19aaaf8b4bb9f57c18476f0d13ef" + "4bc3d245f6a12937807ba59d1cae5085664361168233042d00422a43a056789d" ), ".claude/skills/agents-shipgate/prompts/explain-finding-to-user.md": ( "18031ed870b3c937a2996173820639ef441afe0a45e8171f16468826cd389829" @@ -169,7 +169,7 @@ def test_local_contract_renderer_exposes_agent_operational_fields() -> None: assert payload["primary_commands"]["host_audit"].startswith("shipgate audit --host") assert "verify_local" not in payload["primary_commands"] assert payload["commands"]["verify_local"].startswith("agents-shipgate verify") - assert payload["verifier_schema_version"] == "0.6" + assert payload["verifier_schema_version"] == "0.7" assert payload["verify_run_schema_version"] == "shipgate.verify_run/v3" assert payload["agent_handoff_schema_version"] == "shipgate.agent_handoff/v6" assert payload["agent_handoff_schema_path"] == "docs/agent-handoff-schema.v6.json" @@ -187,7 +187,7 @@ def test_local_contract_renderer_exposes_agent_operational_fields() -> None: assert payload["host_grants_inventory_schema_version"] == "0.2" assert payload["host_grants_baseline_schema_version"] == "0.2" assert payload["host_grants_drift_schema_version"] == "0.2" - assert payload["trigger_catalog_schema_version"] == "0.2" + assert payload["trigger_catalog_schema_version"] == "0.3" assert payload["agent_result_control_fields"] == [ "decision", "control", diff --git a/tests/test_agent_mode.py b/tests/test_agent_mode.py index b0379967..b743ae01 100644 --- a/tests/test_agent_mode.py +++ b/tests/test_agent_mode.py @@ -226,7 +226,7 @@ def test_verify_json_shortcut_prints_verifier_artifact(tmp_path: Path) -> None: assert result.exit_code == 0, result.output payload = json.loads(result.output) - assert payload["verifier_schema_version"] == "0.6" + assert payload["verifier_schema_version"] == "0.7" assert payload["merge_verdict"] == "insufficient_evidence" assert payload["can_merge_without_human"] is False assert payload["control"]["state"] == "human_review_required" @@ -275,7 +275,7 @@ def test_verify_format_json_still_prints_full_verifier_artifact( assert result.exit_code == 0, result.output payload = json.loads(result.output) - assert payload["verifier_schema_version"] == "0.6" + assert payload["verifier_schema_version"] == "0.7" assert payload["execution"] == "succeeded" assert payload["head_status"] == "succeeded" assert payload["trigger"]["run_shipgate"] is True @@ -292,7 +292,7 @@ def test_verify_agent_environment_defaults_to_verifier_json( assert result.exit_code == 0, result.output payload = json.loads(result.output) - assert payload["verifier_schema_version"] == "0.6" + assert payload["verifier_schema_version"] == "0.7" assert payload["merge_verdict"] == "insufficient_evidence" diff --git a/tests/test_authorization_verifier_scenarios.py b/tests/test_authorization_verifier_scenarios.py index e9860993..2d73a309 100644 --- a/tests/test_authorization_verifier_scenarios.py +++ b/tests/test_authorization_verifier_scenarios.py @@ -9,7 +9,12 @@ from agents_shipgate.schemas.agent_control import HumanControlAction from agents_shipgate.schemas.disclaimers import STATIC_VERDICT_DISCLAIMER from agents_shipgate.schemas.human_authorization import AuthorizationEvaluationV1 -from agents_shipgate.schemas.verifier import VerifierArtifact, VerifierFixTask, map_merge_verdict +from agents_shipgate.schemas.verifier import ( + VerifierArtifact, + VerifierDiffStatus, + VerifierFixTask, + map_merge_verdict, +) AUTHORIZED_COMMAND = ( "git push --force-with-lease=refs/heads/codex/human-authorization-state:" @@ -107,6 +112,7 @@ def _verifier( reason = "Shipgate could not complete verification." return VerifierArtifact( workspace="/tmp/repo", + diff_status=VerifierDiffStatus(), config="shipgate.yaml", execution="failed", head_status="failed", @@ -122,6 +128,7 @@ def _verifier( reason = f"Release decision is {decision}." return VerifierArtifact( workspace="/tmp/repo", + diff_status=VerifierDiffStatus(), config="shipgate.yaml", execution="succeeded", head_status="succeeded", diff --git a/tests/test_diff_input_status.py b/tests/test_diff_input_status.py new file mode 100644 index 00000000..9b5e8cd9 --- /dev/null +++ b/tests/test_diff_input_status.py @@ -0,0 +1,828 @@ +"""An unreadable diff must never be reported as "nothing here is agent-related". + +Regression coverage for the class of bug where the diff-acquisition layer +collapsed every failure into one message, the caller then evaluated the trigger +catalog against empty inputs, and the artifact published +``skip_reason: "no_match"`` — "nothing in this PR signals a tool-surface +change" — about a PR the verifier had never read. + +The three input shapes exercised here are the ones that occur in practice: + +1. no reachable merge base (a shallow clone, or unrelated histories); +2. a partial clone whose blobs were never fetched, with lazy fetching disabled + as verification's static/no-implicit-network boundary requires; +3. an agent-related diff whose body exceeds the static diff-body bound. + +In every one of them the changed-path evidence and the diff body have +different availability, so the tests assert on both. +""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest +from pydantic import ValidationError +from typer.testing import CliRunner + +from agents_shipgate.cli.main import app +from agents_shipgate.cli.verify import git as verify_git +from agents_shipgate.cli.verify.git import ( + DiffInputError, + collect_diff_context, + diff_revspec_context, +) +from agents_shipgate.triggers import evaluate + +runner = CliRunner() + +ADK_AGENT_SOURCE = """\ +from google.adk.agents import LlmAgent +from google.adk.tools import FunctionTool + + +def issue_refund(order_id: str, amount: float) -> dict: + return {"order_id": order_id, "amount": amount} + + +refund_tool = FunctionTool(issue_refund) +root_agent = LlmAgent(name="support", tools=[refund_tool]) +""" + + +MINIMAL_MANIFEST = """\ +version: "0.1" +project: + name: test +agent: + name: test-agent + declared_purpose: + - test +environment: + target: local +tool_sources: + - id: tools + type: mcp + path: tools.json +""" + + +def _git(root: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(root), *args], + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def _repo(path: Path) -> Path: + path.mkdir(parents=True, exist_ok=True) + _git(path, "init", "-q", "-b", "main") + _git(path, "config", "user.email", "test@example.test") + _git(path, "config", "user.name", "Test") + return path + + +def _commit(root: Path, message: str) -> None: + _git(root, "add", "-A") + _git(root, "commit", "-qm", message) + + +def _unrelated_histories(tmp_path: Path) -> Path: + """A repo whose two branches share no commit — no merge base exists.""" + + repo = _repo(tmp_path / "repo") + (repo / "README.md").write_text("base\n", encoding="utf-8") + _commit(repo, "base") + + _git(repo, "checkout", "-q", "--orphan", "detached-base") + _git(repo, "rm", "-rq", "--cached", ".") + (repo / "README.md").unlink() + (repo / "OTHER.md").write_text("unrelated root\n", encoding="utf-8") + _commit(repo, "unrelated root") + _git(repo, "checkout", "-q", "main") + + agent = repo / "src" / "agent.py" + agent.parent.mkdir(parents=True, exist_ok=True) + agent.write_text(ADK_AGENT_SOURCE, encoding="utf-8") + _commit(repo, "add adk agent") + return repo + + +def _shallow_clone(tmp_path: Path) -> Path: + """A shallow clone whose truncated history hides a merge base that exists. + + The `actions/checkout` default shape: head fetched at depth 1, base + fetched at depth 1, no reachable common ancestor between them. + """ + + origin = _repo(tmp_path / "origin") + (origin / "README.md").write_text("one\n", encoding="utf-8") + _commit(origin, "c1") + (origin / "README.md").write_text("two\n", encoding="utf-8") + _commit(origin, "c2") + _git(origin, "branch", "base-ref") + agent = origin / "src" / "agent.py" + agent.parent.mkdir(parents=True, exist_ok=True) + agent.write_text(ADK_AGENT_SOURCE, encoding="utf-8") + _commit(origin, "add adk agent") + + clone = tmp_path / "clone" + subprocess.run( + ["git", "clone", "-q", "--depth", "1", "--no-local", f"file://{origin}", str(clone)], + check=True, + capture_output=True, + text=True, + ) + _git(clone, "config", "user.email", "test@example.test") + _git(clone, "config", "user.name", "Test") + _git(clone, "fetch", "-q", "--depth", "1", "origin", "base-ref:base-ref") + assert _git(clone, "rev-parse", "--is-shallow-repository") == "true" + return clone + + +def _blobless_clone( + tmp_path: Path, + *, + capability_path: str = "src/agent.py", + capability_text: str = ADK_AGENT_SOURCE, +) -> Path: + """A partial clone missing the blobs the base side of the diff needs.""" + + origin = _repo(tmp_path / "origin") + (origin / "README.md").write_text("base\n", encoding="utf-8") + _commit(origin, "base") + _git(origin, "branch", "base-ref") + + agent = origin / capability_path + agent.parent.mkdir(parents=True, exist_ok=True) + agent.write_text(capability_text, encoding="utf-8") + # An existing file must also change, so the base side owns a blob the + # clone never fetches. A pure addition would leave nothing missing. + (origin / "README.md").write_text("base, revised\n", encoding="utf-8") + _commit(origin, "add adk agent") + + _git(origin, "config", "uploadpack.allowfilter", "true") + _git(origin, "config", "uploadpack.allowanysha1inwant", "true") + + clone = tmp_path / "clone" + try: + subprocess.run( + [ + "git", + "clone", + "-q", + "--filter=blob:none", + "--no-local", + f"file://{origin}", + str(clone), + ], + check=True, + capture_output=True, + text=True, + ) + except subprocess.CalledProcessError as exc: # pragma: no cover - host policy + pytest.skip(f"local Git refused the partial clone: {exc.stderr}") + applied = subprocess.run( + ["git", "-C", str(clone), "config", "--get", "remote.origin.partialclonefilter"], + check=False, + capture_output=True, + text=True, + ) + if applied.stdout.strip() != "blob:none": # pragma: no cover - host policy + pytest.skip("local Git did not apply the blob:none partial-clone filter") + _git(clone, "config", "user.email", "test@example.test") + _git(clone, "config", "user.name", "Test") + _git(clone, "fetch", "-q", "origin", "base-ref:base-ref") + return clone + + +# --- 1. no merge base ------------------------------------------------------ + + +def test_shallow_history_reports_a_repairable_missing_merge_base( + tmp_path: Path, +) -> None: + clone = _shallow_clone(tmp_path) + + context = collect_diff_context(clone, "base-ref", "HEAD") + + assert context.completeness == "unavailable" + assert context.reason == "merge_base_missing" + assert "no merge base" in context.detail + # Deepening history really does restore the merge base here, so this is + # agent work rather than review work. + assert context.fetch_repairable is True + assert "deepen" in context.remediation.casefold() + + +def test_deepening_a_shallow_clone_actually_repairs_the_diff( + tmp_path: Path, +) -> None: + """The remediation must be the one that works, not the one that reads well.""" + + clone = _shallow_clone(tmp_path) + assert collect_diff_context(clone, "base-ref", "HEAD").reason == "merge_base_missing" + + _git(clone, "fetch", "-q", "--deepen=10", "origin", "main", "base-ref") + + repaired = collect_diff_context(clone, "base-ref", "HEAD") + assert repaired.completeness == "complete" + assert "src/agent.py" in repaired.changed_files + + +def test_unrelated_histories_are_never_routed_to_another_fetch( + tmp_path: Path, +) -> None: + """Git reports both causes as "no merge base"; only one is fetch-repairable. + + Two orphan roots in a complete checkout share no ancestor at all, so + `--deepen`/`--unshallow` can never produce one. Routing this to `fetch_base` + would loop an agent forever. + """ + + repo = _unrelated_histories(tmp_path) + assert _git(repo, "rev-parse", "--is-shallow-repository") == "false" + + context = collect_diff_context(repo, "detached-base", "HEAD") + + assert context.completeness == "unavailable" + assert context.reason == "unrelated_histories" + assert "no merge base" in context.detail + assert context.fetch_repairable is False + assert "no fetch can create one" in context.remediation + + +def test_unrelated_histories_route_a_verify_run_to_a_human(tmp_path: Path) -> None: + repo = _unrelated_histories(tmp_path) + (repo / "shipgate.yaml").write_text('version: "0.1"\n', encoding="utf-8") + _commit(repo, "adopt shipgate") + + result = runner.invoke( + app, + [ + "verify", + "--workspace", + str(repo), + "--base", + "detached-base", + "--head", + "HEAD", + "--format", + "json", + ], + ) + + assert result.exit_code == 2, result.output + payload = json.loads(result.output) + assert payload["diff_status"]["reason"] == "unrelated_histories" + assert payload["diff_status"]["fetch_repairable"] is False + assert payload["control"]["next_action"]["kind"] != "fetch_base" + assert payload["can_merge_without_human"] is False + + +def test_preview_withholds_the_verdict_when_no_merge_base_exists( + tmp_path: Path, +) -> None: + clone = _shallow_clone(tmp_path) + + result = runner.invoke( + app, + [ + "verify", + "--workspace", + str(clone), + "--preview", + "--base", + "base-ref", + "--head", + "HEAD", + "--json", + ], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["diff_status"]["completeness"] == "unavailable" + assert payload["diff_status"]["reason"] == "merge_base_missing" + assert payload["trigger"]["evaluation_status"] == "not_evaluated" + assert payload["trigger"]["should_run"] is None + assert payload["trigger"]["skip_reason"] is None + assert "no_match" not in json.dumps(payload["trigger"]) + assert payload["merge_verdict"] == "unknown" + assert payload["can_merge_without_human"] is False + + +def test_verify_fails_closed_and_names_the_missing_merge_base(tmp_path: Path) -> None: + clone = _shallow_clone(tmp_path) + (clone / "shipgate.yaml").write_text('version: "0.1"\n', encoding="utf-8") + _commit(clone, "adopt shipgate") + + result = runner.invoke( + app, + [ + "verify", + "--workspace", + str(clone), + "--base", + "base-ref", + "--head", + "HEAD", + "--format", + "json", + ], + ) + + assert result.exit_code == 2, result.output + payload = json.loads(result.output) + assert payload["diff_status"]["reason"] == "merge_base_missing" + assert payload["base_status"] == "archive_failed" + assert payload["merge_verdict"] == "unknown" + assert payload["can_merge_without_human"] is False + assert any("merge_base_missing" in note for note in payload["base_notes"]) + # The refs are present; only history depth is, so fetching is the repair. + assert payload["control"]["next_action"]["kind"] == "fetch_base" + + +# --- 2. partial clone, objects never fetched ------------------------------- + + +def test_partial_clone_keeps_changed_paths_when_blobs_are_missing( + tmp_path: Path, +) -> None: + clone = _blobless_clone(tmp_path) + + context = collect_diff_context(clone, "base-ref", "HEAD") + + # `--name-status` answers fully in a blobless clone even though the + # textual diff cannot be produced, and those paths are precisely what + # says the PR touches an agent surface. + assert context.completeness == "partial" + assert context.reason == "objects_missing" + assert "src/agent.py" in context.changed_files + assert context.fetch_repairable is True + assert "GIT_NO_LAZY_FETCH" in context.remediation + + +def test_preview_routes_a_blobless_clone_to_hydrating_objects(tmp_path: Path) -> None: + clone = _blobless_clone(tmp_path) + + result = runner.invoke( + app, + [ + "verify", + "--workspace", + str(clone), + "--preview", + "--base", + "base-ref", + "--head", + "HEAD", + "--json", + ], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["diff_status"]["completeness"] == "partial" + assert payload["diff_status"]["reason"] == "objects_missing" + assert "src/agent.py" in payload["changed_files"] + assert payload["trigger"]["evaluation_status"] == "not_evaluated" + assert payload["trigger"]["skip_reason"] is None + assert payload["control"]["next_action"]["kind"] == "fetch_base" + assert payload["can_merge_without_human"] is False + + +def test_unconfigured_workspace_still_reports_the_diff_failure(tmp_path: Path) -> None: + """The cold-start case this class of failure actually comes from. + + Shallow and blobless clones of un-adopted repositories are the normal + shape of first contact. Routing them to "Shipgate is not configured in + this workspace" answers a question nobody asked and hides the fact that + the PR was never inspected. + """ + + clone = _blobless_clone(tmp_path) + assert not (clone / "shipgate.yaml").exists() + + result = runner.invoke( + app, + [ + "verify", + "--workspace", + str(clone), + "--preview", + "--base", + "base-ref", + "--head", + "HEAD", + "--json", + ], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["control"]["next_action"]["kind"] != "initialize" + assert "not configured" not in payload["headline"] + assert "objects_missing" in payload["headline"] + assert payload["diff_status"]["reason"] == "objects_missing" + + +# --- 3. an agent diff whose body is unreadable ----------------------------- + + +def test_body_limit_keeps_paths_and_never_reports_no_match( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A binary-heavy PR must not lose its `agent.py` evidence. + + The Google ADK shape: one small `agent.py` that adds an `LlmAgent` and + `FunctionTool` bindings, shipped alongside demo assets large enough to + push the aggregate diff past the static body bound. + """ + + repo = _repo(tmp_path / "repo") + (repo / "README.md").write_text("base\n", encoding="utf-8") + _commit(repo, "base") + _git(repo, "branch", "base-ref") + + agent = repo / "src" / "agent.py" + agent.parent.mkdir(parents=True, exist_ok=True) + agent.write_text(ADK_AGENT_SOURCE, encoding="utf-8") + (repo / "docs").mkdir() + (repo / "docs" / "demo.txt").write_text("x" * 200_000 + "\n", encoding="utf-8") + _commit(repo, "add adk agent plus demo assets") + + monkeypatch.setattr(verify_git, "_DIFF_BODY_LIMIT", 4096) + + context = collect_diff_context(repo, "base-ref", "HEAD") + + assert context.completeness == "partial" + assert context.reason == "body_limit_exceeded" + assert "src/agent.py" in context.changed_files + assert context.fetch_repairable is False + + # The path evidence alone does not fire the ADK rule (it keys on the + # `FunctionTool(` token in the body), so the honest answer is "not + # evaluated" — never "nothing in this PR signals a tool-surface change". + trigger = evaluate( + paths=list(context.changed_files), + diff_text=context.diff_text, + manifest_present=False, + user_requested=True, + input_status="partial", + ) + assert trigger["evaluation_status"] == "not_evaluated" + assert trigger["should_run"] is None + assert trigger["skip_reason"] is None + assert trigger["next_action"]["kind"] == "input_required" + + +def test_a_readable_agent_diff_still_runs(tmp_path: Path) -> None: + """The control: with the body readable, the same PR routes to a run.""" + + repo = _repo(tmp_path / "repo") + (repo / "README.md").write_text("base\n", encoding="utf-8") + _commit(repo, "base") + _git(repo, "branch", "base-ref") + + agent = repo / "src" / "agent.py" + agent.parent.mkdir(parents=True, exist_ok=True) + agent.write_text(ADK_AGENT_SOURCE, encoding="utf-8") + _commit(repo, "add adk agent") + + context = collect_diff_context(repo, "base-ref", "HEAD") + + assert context.completeness == "complete" + assert context.reason is None + trigger = evaluate( + paths=list(context.changed_files), + diff_text=context.diff_text, + manifest_present=False, + user_requested=True, + ) + assert trigger["evaluation_status"] == "evaluated" + assert trigger["should_run"] is True + assert "TRIGGER-FUNCTION-TOOL-DECORATOR" in { + match["id"] for match in trigger["matched_rules"] + } + + +# --- the strict wrapper keeps its contract --------------------------------- + + +def test_strict_wrapper_refuses_to_hand_back_partial_evidence( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Callers that cannot represent partial input must not receive it silently.""" + + repo = _repo(tmp_path / "repo") + (repo / "README.md").write_text("base\n", encoding="utf-8") + _commit(repo, "base") + _git(repo, "branch", "base-ref") + (repo / "README.md").write_text("x" * 200_000 + "\n", encoding="utf-8") + _commit(repo, "large edit") + + monkeypatch.setattr(verify_git, "_DIFF_BODY_LIMIT", 4096) + + with pytest.raises(DiffInputError) as excinfo: + diff_revspec_context(repo, "base-ref...HEAD") + + assert excinfo.value.context.reason == "body_limit_exceeded" + assert excinfo.value.context.changed_files == ("README.md",) + + +def test_diagnostics_do_not_leak_the_local_checkout_path(tmp_path: Path) -> None: + repo = _unrelated_histories(tmp_path) + + context = collect_diff_context(repo, "detached-base", "HEAD") + + assert str(repo) not in context.detail + assert str(repo) not in context.note + + +# --- the artifact may not contradict itself -------------------------------- + + +def test_partial_evidence_that_proves_relevance_keeps_its_run_verdict( + tmp_path: Path, +) -> None: + """A path rule needs no diff body, so partial input can still decide "run". + + The evaluator publishes that verdict deliberately. The headline and + `control.reason` summarize the same artifact and must not answer + "no relevance verdict was reached" over the top of `should_run: true`. + """ + + clone = _blobless_clone( + tmp_path, + capability_path="tools/new_mcp.json", + capability_text='{"mcpServers": {}}\n', + ) + + result = runner.invoke( + app, + [ + "verify", + "--workspace", + str(clone), + "--preview", + "--base", + "base-ref", + "--head", + "HEAD", + "--json", + ], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + trigger = payload["trigger"] + + assert payload["diff_status"]["completeness"] == "partial" + assert "tools/new_mcp.json" in payload["changed_files"] + assert trigger["evaluation_status"] == "evaluated" + assert trigger["should_run"] is True + assert "TRIGGER-MCP-EXPORT-CHANGED" in { + match["id"] for match in trigger["matched_rules"] + } + + for surface in (payload["headline"], payload["control"]["reason"]): + assert "no relevance verdict" not in surface + assert "relevance is established" in surface + # The diff still has to be recovered before any merge verdict is trusted. + assert payload["merge_verdict"] == "unknown" + assert payload["can_merge_without_human"] is False + + +def test_verify_merges_partial_worktree_paths_into_the_change_set( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A failed worktree read still contributes the paths it did collect. + + Reporting "changed paths were collected" in `base_notes` while handing the + trigger an empty list loses exactly the path-rule match those paths exist + to produce. + """ + + repo = _repo(tmp_path / "repo") + (repo / "shipgate.yaml").write_text(MINIMAL_MANIFEST, encoding="utf-8") + (repo / "tools.json").write_text('{"tools": []}\n', encoding="utf-8") + (repo / "README.md").write_text("base\n", encoding="utf-8") + _commit(repo, "base") + _git(repo, "branch", "base-ref") + + # Uncommitted: one capability path plus enough text to blow the body bound. + (repo / "tools").mkdir() + (repo / "tools" / "new_mcp.json").write_text('{"mcpServers": {}}\n', encoding="utf-8") + (repo / "README.md").write_text("x" * 200_000 + "\n", encoding="utf-8") + + monkeypatch.setattr(verify_git, "_DIFF_BODY_LIMIT", 4096) + + # No --head: that is what makes verify read the working tree rather than an + # archived committed tree, which is the collector under test. + result = runner.invoke( + app, + [ + "verify", + "--workspace", + str(repo), + "--base", + "base-ref", + "--format", + "json", + ], + ) + + assert result.exit_code == 2, result.output + payload = json.loads(result.output) + + assert payload["diff_status"]["completeness"] == "partial" + assert payload["diff_status"]["reason"] == "body_limit_exceeded" + assert any("paths were collected" in note for note in payload["base_notes"]) + # The claim in base_notes and the published change set must agree. + assert "tools/new_mcp.json" in payload["changed_files"] + assert "TRIGGER-MCP-EXPORT-CHANGED" in { + match["id"] for match in payload["trigger"]["matched_rules"] + } + assert payload["merge_verdict"] == "unknown" + assert payload["can_merge_without_human"] is False + + +# --- the status, the repair, and the headline must agree ------------------- + + +def test_a_current_artifact_cannot_omit_its_input_health(tmp_path: Path) -> None: + """`diff_status` is the input-health contract; dropping it must not validate. + + A v0.7 payload with no `diff_status` would be indistinguishable from one + that read its diff cleanly — the exact claim the field exists to prevent. + """ + + from agents_shipgate.schemas.verifier import VerifierArtifact, VerifierDiffStatus + + repo = _repo(tmp_path / "repo") + (repo / "README.md").write_text("base\n", encoding="utf-8") + _commit(repo, "base") + result = runner.invoke( + app, ["verify", "--workspace", str(repo), "--preview", "--json"] + ) + assert result.exit_code == 0, result.output + emitted = json.loads(result.output) + assert emitted["diff_status"]["completeness"] == "complete" + + without = {k: v for k, v in emitted.items() if k != "diff_status"} + with pytest.raises(ValidationError): + VerifierArtifact.model_validate(without) + + # A pre-v0.7 artifact legitimately has none, and normalizes to "unknown" — + # which is not "complete", so it still withholds trust in a negative result. + legacy = dict(without) + legacy["verifier_schema_version"] = "0.6" + normalized = VerifierArtifact.model_validate(legacy) + assert normalized.verifier_schema_version == "0.7" + assert normalized.diff_status == VerifierDiffStatus.unknown() + assert normalized.diff_status.completeness == "unknown" + assert normalized.diff_status.reason is None + + +def test_fetch_repairable_cannot_be_claimed_for_a_deterministic_failure() -> None: + from agents_shipgate.schemas.verifier import VerifierDiffStatus + + with pytest.raises(ValueError, match="fetching cannot repair"): + VerifierDiffStatus( + completeness="unavailable", + reason="unrelated_histories", + fetch_repairable=True, + ) + + +def test_the_worst_failure_decides_both_status_and_repair( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A fetchable committed failure must not authorize a fetch for a worse one. + + Committed diff fails fetch-repairably (`refs_missing`); the worktree then + fails deterministically. Deriving the action incrementally published + `fetch_base` beside a `diff_status` no fetch could repair — a loop. + """ + + repo = _repo(tmp_path / "repo") + (repo / "shipgate.yaml").write_text(MINIMAL_MANIFEST, encoding="utf-8") + (repo / "tools.json").write_text('{"tools": []}\n', encoding="utf-8") + (repo / "README.md").write_text("base\n", encoding="utf-8") + _commit(repo, "base") + (repo / "README.md").write_text("uncommitted\n", encoding="utf-8") + + def _explode(*args: object, **kwargs: object) -> tuple[list[str], str]: + raise RuntimeError("simulated deterministic worktree failure") + + monkeypatch.setattr( + "agents_shipgate.cli.verify.orchestrator.working_tree_context", _explode + ) + + result = runner.invoke( + app, + [ + "verify", + "--workspace", + str(repo), + "--base", + "does-not-exist", + "--format", + "json", + ], + ) + + assert result.exit_code == 2, result.output + payload = json.loads(result.output) + status = payload["diff_status"] + action = payload["control"]["next_action"] + + assert status["reason"] == "git_failed" + assert status["fetch_repairable"] is False + # The published status and the authorized repair may not disagree. + assert action["kind"] != "fetch_base" + assert payload["can_merge_without_human"] is False + + +def test_the_failure_headline_matches_the_control_route(tmp_path: Path) -> None: + """A `fetch_base` route may not be summarized as "human review required".""" + + repo = _repo(tmp_path / "repo") + (repo / "shipgate.yaml").write_text(MINIMAL_MANIFEST, encoding="utf-8") + (repo / "tools.json").write_text('{"tools": []}\n', encoding="utf-8") + (repo / "README.md").write_text("base\n", encoding="utf-8") + _commit(repo, "base") + + result = runner.invoke( + app, + [ + "verify", + "--workspace", + str(repo), + "--base", + "origin/main", + "--head", + "HEAD", + "--format", + "json", + ], + ) + + assert result.exit_code == 2, result.output + payload = json.loads(result.output) + control = payload["control"] + + assert payload["diff_status"]["reason"] == "refs_missing" + assert control["state"] == "agent_action_required" + assert control["next_action"]["kind"] == "fetch_base" + assert control["human_review"]["required"] is False + for surface in (payload["headline"], control["reason"]): + assert "human review required" not in surface.casefold() + assert "refs_missing" in surface + + +def test_a_force_run_verdict_is_not_attributed_to_unread_paths( + tmp_path: Path, +) -> None: + """An adopted repo force-runs on the manifest, with no paths read at all.""" + + repo = _repo(tmp_path / "repo") + (repo / "shipgate.yaml").write_text(MINIMAL_MANIFEST, encoding="utf-8") + (repo / "tools.json").write_text('{"tools": []}\n', encoding="utf-8") + (repo / "README.md").write_text("base\n", encoding="utf-8") + _commit(repo, "base") + + result = runner.invoke( + app, + [ + "verify", + "--workspace", + str(repo), + "--preview", + "--base", + "origin/main", + "--head", + "HEAD", + "--json", + ], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + trigger = payload["trigger"] + + assert payload["changed_files"] == [] + assert trigger["force_run"] is True + assert trigger["should_run"] is True + assert {match["id"] for match in trigger["matched_rules"]} == { + "TRIGGER-EXISTING-MANIFEST-PRESENT" + } + for surface in (payload["headline"], payload["control"]["reason"]): + assert "already shows an agent-capability surface" not in surface + assert "already configured for Shipgate" in surface diff --git a/tests/test_human_authorization.py b/tests/test_human_authorization.py index 363840a7..1907ec65 100644 --- a/tests/test_human_authorization.py +++ b/tests/test_human_authorization.py @@ -812,7 +812,7 @@ def test_wire_schema_constrains_keys_signatures_and_push_syntax( @pytest.mark.parametrize( "schema_name", - ["verifier-schema.v0.6.json", "agent-handoff-schema.v6.json"], + ["verifier-schema.v0.7.json", "agent-handoff-schema.v6.json"], ) def test_embedded_authorization_evaluation_schemas_are_fail_closed( schema_name: str, diff --git a/tests/test_local_contract.py b/tests/test_local_contract.py index c3b6ca41..4f0c137b 100644 --- a/tests/test_local_contract.py +++ b/tests/test_local_contract.py @@ -112,7 +112,7 @@ def test_local_agent_contract_is_minimal_agent_operational_payload() -> None: ] assert payload["verifier_read_order"][-2:] == ["request_id", "decision_id"] assert payload["gating_signal"] == GATING_SIGNAL - assert payload["verifier_schema_version"] == "0.6" + assert payload["verifier_schema_version"] == "0.7" assert payload["verify_run_schema_version"] == "shipgate.verify_run/v3" assert payload["human_authorization_request_schema_version"] == ( "shipgate.human_authorization_request/v1" @@ -146,7 +146,7 @@ def test_local_agent_contract_is_minimal_agent_operational_payload() -> None: assert payload["host_grants_inventory_schema_version"] == "0.2" assert payload["host_grants_baseline_schema_version"] == "0.2" assert payload["host_grants_drift_schema_version"] == "0.2" - assert payload["trigger_catalog_schema_version"] == "0.2" + assert payload["trigger_catalog_schema_version"] == "0.3" assert payload["agent_result_schema_version"] == "agent_result_v2" assert payload["agent_result_schema_path"] == "docs/agent-result-schema.v2.json" assert payload["agent_result_control_fields"] == [ diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 100715d4..10e3f8c0 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -241,6 +241,7 @@ def test_mcp_handoff_handler_is_read_only(tmp_path: Path) -> None: "config": "shipgate.yaml", "execution": "succeeded", "head_status": "succeeded", + "diff_status": {"completeness": "complete"}, "release_decision": { "decision": "passed", "reason": "All required static verification passed.", diff --git a/tests/test_public_surface_contract.py b/tests/test_public_surface_contract.py index 2899f208..44cabc60 100644 --- a/tests/test_public_surface_contract.py +++ b/tests/test_public_surface_contract.py @@ -1033,8 +1033,8 @@ def test_triggers_json_loads_via_canonical_loader(): reaches a different verdict than this loader, that's a drift bug — catch it by exercising the loader during CI.""" triggers = load_triggers() - assert triggers["schema_version"] == "0.2", ( - "docs/triggers.json schema_version moved off 0.2; bump the " + assert triggers["schema_version"] == "0.3", ( + "docs/triggers.json schema_version moved off 0.3; bump the " "test constant deliberately so external consumers are notified." ) assert isinstance(triggers.get("rules"), list) and triggers["rules"], ( diff --git a/tests/test_safety_qualification.py b/tests/test_safety_qualification.py index 2916a91b..70d7db21 100644 --- a/tests/test_safety_qualification.py +++ b/tests/test_safety_qualification.py @@ -40,7 +40,10 @@ VerificationTask, content_id, ) -from agents_shipgate.schemas.verifier import VerifierArtifact +from agents_shipgate.schemas.verifier import ( + VerifierArtifact, + VerifierDiffStatus, +) from agents_shipgate.schemas.verify_run import ( VerifyRunOutcome, build_verify_run_artifact, @@ -358,6 +361,7 @@ def _fixture( ) verifier = VerifierArtifact( workspace=".", + diff_status=VerifierDiffStatus(), request_id=plan.request_id, subject_id=plan.subject.subject_id, input_set_id=plan.inputs.input_set_id, diff --git a/tests/test_trigger_command.py b/tests/test_trigger_command.py index 4ee1d483..1890d70a 100644 --- a/tests/test_trigger_command.py +++ b/tests/test_trigger_command.py @@ -12,6 +12,7 @@ from typer.testing import CliRunner from agents_shipgate.cli.main import app +from agents_shipgate.core.errors import ConfigError from agents_shipgate.triggers import ( SURFACE_CLASS_CAPABILITY, SURFACE_CLASS_HOST_BOUNDARY, @@ -118,7 +119,7 @@ def test_trigger_subcommand_json_shape(tmp_path): assert result.exit_code == 0, result.stdout payload = json.loads(result.stdout) assert M1_KEYS <= set(payload) - assert payload["schema_version"] == "0.2" + assert payload["schema_version"] == "0.3" assert payload["should_run"] is True assert payload["force_run"] is True # shipgate.yaml present in workspace assert payload["skip_reason"] is None @@ -159,7 +160,7 @@ def test_trigger_subcommand_list_rules_json(): result = runner.invoke(app, ["trigger", "--list-rules", "--json"]) assert result.exit_code == 0 catalog = json.loads(result.stdout) - assert catalog["schema_version"] == "0.2" + assert catalog["schema_version"] == "0.3" rule_ids = {r["id"] for r in catalog["rules"]} assert "TRIGGER-N8N-WORKFLOW-CHANGED" in rule_ids @@ -453,3 +454,66 @@ def test_unknown_action_falls_through_to_no_match(): res = evaluate(paths=["a.py"], triggers=cat) assert res["should_run"] is False assert {m["id"] for m in res["matched_rules"]} == {"R"} + + +# --- input completeness: a verdict may not outrun its evidence ------------- + + +def test_complete_inputs_keep_the_evaluated_no_match_verdict(): + result = evaluate(paths=["src/internal/util.py"], input_status="complete") + assert result["input_status"] == "complete" + assert result["evaluation_status"] == "evaluated" + assert result["should_run"] is False + assert result["skip_reason"] == "no_match" + + +@pytest.mark.parametrize("status", ["partial", "unavailable"]) +def test_incomplete_inputs_withhold_the_skip_verdict(status): + """`no_match` is a claim about a diff. Without the diff there is no claim.""" + + result = evaluate(paths=[], diff_text="", input_status=status) + assert result["evaluation_status"] == "not_evaluated" + assert result["should_run"] is None + assert result["run_shipgate"] is None + assert result["skip"] is None + assert result["skip_reason"] is None + assert result["next_action"]["kind"] == "input_required" + assert result["next_action"]["command"] is None + assert "not evidence" in result["rationale"] + + +def test_incomplete_inputs_still_publish_a_run_verdict(): + """Rule matching is monotone: more evidence only adds matches. + + A run reached from partial evidence therefore stays sound, and suppressing + it would turn a fail-closed gap into a missed gate. + """ + + result = evaluate( + paths=["shipgate.yaml"], + diff_text="", + manifest_present=True, + input_status="partial", + ) + assert result["evaluation_status"] == "evaluated" + assert result["should_run"] is True + assert result["input_status"] == "partial" + assert result["next_action"]["kind"] == "command" + + +def test_incomplete_inputs_cannot_fire_stop_conditions(): + """The stop block reasons over the very path evidence that is missing.""" + + result = evaluate( + paths=[], + detect_result={"is_agent_project": False, "suggested_sources": []}, + input_status="unavailable", + ) + assert result["stop_conditions_evaluated"] is False + assert result["stop_conditions_fired"] is False + assert result["evaluation_status"] == "not_evaluated" + + +def test_unknown_input_status_is_rejected(): + with pytest.raises(ConfigError): + evaluate(paths=[], input_status="mostly") diff --git a/tests/test_verdict_contract.py b/tests/test_verdict_contract.py index 164f1b25..4dcfabc5 100644 --- a/tests/test_verdict_contract.py +++ b/tests/test_verdict_contract.py @@ -30,6 +30,7 @@ from agents_shipgate.schemas.verifier import ( _DECISION_TO_VERDICT, VerifierArtifact, + VerifierDiffStatus, applicability_for, map_merge_verdict, merge_verdict_for, @@ -152,6 +153,7 @@ def _artifact(**overrides) -> VerifierArtifact: "config": "shipgate.yaml", "head_status": "succeeded", "authorization": AuthorizationEvaluationV1.not_requested(), + "diff_status": VerifierDiffStatus(), } base.update(overrides) return VerifierArtifact(**base) @@ -255,6 +257,7 @@ def test_artifact_rejects_applicability_inconsistent_with_substrate() -> None: with pytest.raises(ValidationError): VerifierArtifact( workspace="/tmp/w", + diff_status=VerifierDiffStatus(), config="shipgate.yaml", head_status="succeeded", execution="succeeded", diff --git a/tests/test_verifier_control_contract.py b/tests/test_verifier_control_contract.py index b415c1f5..e2891e17 100644 --- a/tests/test_verifier_control_contract.py +++ b/tests/test_verifier_control_contract.py @@ -18,7 +18,11 @@ from agents_shipgate.schemas.agent_control import CodingAgentCommandAction, HumanControlAction from agents_shipgate.schemas.disclaimers import STATIC_VERDICT_DISCLAIMER from agents_shipgate.schemas.human_authorization import AuthorizationEvaluationV1 -from agents_shipgate.schemas.verifier import VerifierArtifact, map_merge_verdict +from agents_shipgate.schemas.verifier import ( + VerifierArtifact, + VerifierDiffStatus, + map_merge_verdict, +) from agents_shipgate.schemas.verify_run import VerifyRunOutcome, build_verify_run_artifact ROOT = Path(__file__).resolve().parent.parent @@ -72,6 +76,7 @@ def _release_decision(decision: str) -> dict[str, object]: def _passed_verifier() -> VerifierArtifact: return VerifierArtifact( workspace="/tmp/repo", + diff_status=VerifierDiffStatus(), config="shipgate.yaml", execution="succeeded", head_status="succeeded", @@ -98,6 +103,7 @@ def _authorized_verifier() -> VerifierArtifact: ) return VerifierArtifact( workspace="/tmp/repo", + diff_status=VerifierDiffStatus(), config="shipgate.yaml", execution="succeeded", head_status="succeeded", @@ -177,7 +183,7 @@ def test_handoff_rejects_tampered_current_verify_run_outcome() -> None: @pytest.mark.parametrize( ("schema_path", "control_path"), [ - ("docs/verifier-schema.v0.6.json", ("control",)), + ("docs/verifier-schema.v0.7.json", ("control",)), ("docs/agent-handoff-schema.v6.json", ("control",)), ("docs/verify-run-schema.v3.json", ("outcome", "control")), ], @@ -190,7 +196,7 @@ def test_generated_public_schemas_reject_contradictory_control( run = _passed_run(verifier) handoff = build_agent_handoff(verifier=verifier, verify_run=run) payload_by_schema = { - "docs/verifier-schema.v0.6.json": verifier.model_dump(mode="json"), + "docs/verifier-schema.v0.7.json": verifier.model_dump(mode="json"), "docs/agent-handoff-schema.v6.json": handoff.model_dump(mode="json"), "docs/verify-run-schema.v3.json": run.model_dump(mode="json"), } @@ -206,7 +212,7 @@ def test_generated_public_schemas_reject_contradictory_control( @pytest.mark.parametrize( "schema_path", - ["docs/verifier-schema.v0.6.json", "docs/agent-handoff-schema.v6.json"], + ["docs/verifier-schema.v0.7.json", "docs/agent-handoff-schema.v6.json"], ) def test_generated_schemas_reject_accepted_authorization_on_passed_gate( schema_path: str, @@ -215,7 +221,7 @@ def test_generated_schemas_reject_accepted_authorization_on_passed_gate( run = _passed_run(verifier) handoff = build_agent_handoff(verifier=verifier, verify_run=run) payload_by_schema = { - "docs/verifier-schema.v0.6.json": verifier.model_dump(mode="json"), + "docs/verifier-schema.v0.7.json": verifier.model_dump(mode="json"), "docs/agent-handoff-schema.v6.json": handoff.model_dump(mode="json"), } payload = deepcopy(payload_by_schema[schema_path]) @@ -243,7 +249,7 @@ def test_verifier_schema_requires_complete_authorized_projection(field: str) -> payload = _authorized_verifier().model_dump(mode="json") payload.pop(field) - schema = json.loads((ROOT / "docs/verifier-schema.v0.6.json").read_text(encoding="utf-8")) + schema = json.loads((ROOT / "docs/verifier-schema.v0.7.json").read_text(encoding="utf-8")) assert list(Draft202012Validator(schema).iter_errors(payload)) @@ -261,6 +267,7 @@ def test_nonpassing_release_decision_cannot_claim_merge_authority(decision: str) with pytest.raises(ValidationError): VerifierArtifact( workspace="/tmp/repo", + diff_status=VerifierDiffStatus(), config="shipgate.yaml", execution="succeeded", head_status="succeeded", @@ -301,6 +308,7 @@ def test_accepted_authorization_rejects_control_mismatch(mismatch: str) -> None: with pytest.raises(ValidationError): VerifierArtifact( workspace="/tmp/repo", + diff_status=VerifierDiffStatus(), config="shipgate.yaml", execution="succeeded", head_status="succeeded", @@ -359,5 +367,5 @@ def test_passed_wrapper_contradictions_fail_pydantic_and_generated_schema( mutate(payload) with pytest.raises(ValidationError): VerifierArtifact.model_validate(payload) - schema = json.loads((ROOT / "docs/verifier-schema.v0.6.json").read_text()) + schema = json.loads((ROOT / "docs/verifier-schema.v0.7.json").read_text()) assert list(Draft202012Validator(schema).iter_errors(payload)) diff --git a/tests/test_verifier_scenarios.py b/tests/test_verifier_scenarios.py index f144fb13..82d2fb2d 100644 --- a/tests/test_verifier_scenarios.py +++ b/tests/test_verifier_scenarios.py @@ -306,7 +306,11 @@ def test_scenario_docs_only_no_shipgate_fails_closed(tmp_path: Path) -> None: assert result.exit_code == 2, result.output payload = json.loads(result.output) - assert payload["trigger"]["should_run"] is False + # verify stopped at the missing manifest before reading any diff, so the + # trigger has no change set to judge and must not claim "no rules matched". + assert payload["trigger"]["evaluation_status"] == "not_evaluated" + assert payload["trigger"]["should_run"] is None + assert payload["trigger"]["skip_reason"] is None assert payload["head_status"] == "failed" assert payload["merge_verdict"] == "unknown" assert payload["applicability"] == "failed" diff --git a/tests/test_verify.py b/tests/test_verify.py index 78bb64af..5207711a 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -59,6 +59,7 @@ from agents_shipgate.schemas.verifier import ( VerifierArtifact, VerifierCapabilityReview, + VerifierDiffStatus, VerifierFixTask, VerifierRepair, ) @@ -852,6 +853,7 @@ def test_verify_real_base_scan_enables_head_diff(tmp_path: Path) -> None: def test_pr_comment_keeps_code_span_values_unescaped() -> None: verifier = VerifierArtifact( workspace="/tmp/work", + diff_status=VerifierDiffStatus(), config="shipgate.yaml", authorization=AuthorizationEvaluationV1.not_requested(), base_ref="origin/main", @@ -950,6 +952,7 @@ def test_capability_review_pr_comment_leads_with_top_changes_and_trust_root() -> ) verifier = VerifierArtifact( workspace="/tmp/work", + diff_status=VerifierDiffStatus(), config="shipgate.yaml", authorization=AuthorizationEvaluationV1.not_requested(), trigger={"rationale": "1 run_shipgate rule(s) matched."}, @@ -1019,6 +1022,7 @@ def test_capability_review_pr_comment_preserves_valid_agent_json_when_compacted( ] verifier = VerifierArtifact( workspace="/tmp/work", + diff_status=VerifierDiffStatus(), config="shipgate.yaml", authorization=AuthorizationEvaluationV1.not_requested(), trigger={"rationale": "1 run_shipgate rule(s) matched."}, @@ -1062,6 +1066,7 @@ def test_capability_review_pr_comment_uses_merge_verdict_vocabulary() -> None: report = _report(decision="review_required", exit_code=0) verifier = VerifierArtifact( workspace="/tmp/work", + diff_status=VerifierDiffStatus(), config="shipgate.yaml", authorization=AuthorizationEvaluationV1.not_requested(), trigger={"rationale": "1 run_shipgate rule(s) matched."}, @@ -1088,6 +1093,7 @@ def test_capability_review_pr_comment_does_not_double_blank_without_headline() - report = _report(decision="review_required", exit_code=0) verifier = VerifierArtifact( workspace="/tmp/work", + diff_status=VerifierDiffStatus(), config="shipgate.yaml", authorization=AuthorizationEvaluationV1.not_requested(), trigger={"rationale": "1 run_shipgate rule(s) matched."}, @@ -1111,6 +1117,7 @@ def test_capability_review_pr_comment_does_not_double_blank_without_headline() - def test_capability_review_pr_comment_unknown_when_head_scan_failed() -> None: verifier = VerifierArtifact( workspace="/tmp/work", + diff_status=VerifierDiffStatus(), config="shipgate.yaml", authorization=AuthorizationEvaluationV1.not_requested(), trigger={"rationale": "1 run_shipgate rule(s) matched."}, @@ -2315,7 +2322,17 @@ def test_verify_preview_docs_only_diff_does_not_recommend_init(tmp_path: Path) - ) -def test_verify_preview_missing_base_without_manifest_recommends_init(tmp_path: Path) -> None: +def test_verify_preview_missing_base_without_manifest_reports_the_missing_ref( + tmp_path: Path, +) -> None: + """An unreadable diff outranks the adoption route, manifest or not. + + A shallow or blobless clone of an un-adopted repository is the normal + shape of first contact, so this is exactly the case where routing to + "Shipgate is not configured here" would hide the fact that the PR was + never inspected. + """ + repo = _init_repo(tmp_path) (repo / "README.md").write_text("base\n", encoding="utf-8") _commit_all(repo, "base") @@ -2343,11 +2360,15 @@ def test_verify_preview_missing_base_without_manifest_recommends_init(tmp_path: assert payload["mode"] == "preview" assert payload["config"] == "shipgate.yaml" assert payload["control"]["state"] == "agent_action_required" - assert payload["control"]["next_action"]["kind"] == "initialize" - assert payload["control"]["next_action"]["command"] == ( - f"shipgate init --workspace {repo} --write --json" - ) + assert payload["control"]["next_action"]["kind"] == "fetch_base" + assert payload["diff_status"]["completeness"] == "unavailable" + assert payload["diff_status"]["reason"] == "refs_missing" + assert payload["trigger"]["evaluation_status"] == "not_evaluated" + assert payload["trigger"]["should_run"] is None + assert payload["trigger"]["skip_reason"] is None assert payload["base_notes"] + assert payload["merge_verdict"] == "unknown" + assert payload["can_merge_without_human"] is False def test_verify_preview_configured_repo_preserves_exact_verify_args(tmp_path: Path) -> None: @@ -2468,8 +2489,10 @@ def test_verify_preview_configured_repo_missing_base_fetches_base(tmp_path: Path assert payload["mode"] == "preview" assert payload["control"]["state"] == "agent_action_required" assert payload["control"]["next_action"]["kind"] == "fetch_base" - assert "could not inspect" in payload["control"]["next_action"]["why"] - assert payload["control"]["next_action"]["expects"] == "origin/main" + why = payload["control"]["next_action"]["why"] + assert "refs_missing" in why + assert "git fetch" in why + assert payload["control"]["next_action"]["expects"] == "origin/main...HEAD" assert payload["base_notes"]