From 78ec1e9a3fa122f9106a4d12e0b16f816daf03ee Mon Sep 17 00:00:00 2001 From: Pengfei Hu Date: Sun, 9 Aug 2026 13:40:04 -0700 Subject: [PATCH 1/3] Fix framework-aware evidence remediation --- src/agents_shipgate/ci/github_summary.py | 9 ++- src/agents_shipgate/cli/_helpers.py | 11 ++-- src/agents_shipgate/report/summary_text.py | 26 ++++++++ tests/test_ci.py | 73 ++++++++++++++++++++++ 4 files changed, 110 insertions(+), 9 deletions(-) diff --git a/src/agents_shipgate/ci/github_summary.py b/src/agents_shipgate/ci/github_summary.py index 3f646c80..7adcc7c1 100644 --- a/src/agents_shipgate/ci/github_summary.py +++ b/src/agents_shipgate/ci/github_summary.py @@ -6,7 +6,10 @@ from agents_shipgate.core.disclaimers import STATIC_VERDICT_DISCLAIMER from agents_shipgate.core.privacy import sanitize_report from agents_shipgate.report.markdown import _safe_markdown_text -from agents_shipgate.report.summary_text import evidence_coverage_text +from agents_shipgate.report.summary_text import ( + evidence_coverage_text, + primary_evidence_remediation_text, +) from agents_shipgate.schemas.report import ReadinessReport @@ -41,8 +44,8 @@ def write_github_step_summary(report: ReadinessReport) -> None: ) if decision.decision == "insufficient_evidence": lines.append( - "Improve evidence: provide MCP export, OpenAPI spec, explicit local " - "tool inventory, or a broader OpenAI SDK source path; then rerun scan." + "Improve evidence: " + f"{_safe_markdown_text(primary_evidence_remediation_text(decision.evidence_coverage))}" ) if agent_summary and agent_summary.first_recommended_action: lines.append(_agent_next_action_line(agent_summary.first_recommended_action)) diff --git a/src/agents_shipgate/cli/_helpers.py b/src/agents_shipgate/cli/_helpers.py index d67498c0..90f83b47 100644 --- a/src/agents_shipgate/cli/_helpers.py +++ b/src/agents_shipgate/cli/_helpers.py @@ -18,7 +18,10 @@ from agents_shipgate.core.disclaimers import STATIC_VERDICT_DISCLAIMER from agents_shipgate.core.errors import AgentsShipgateError, ConfigError, InputParseError from agents_shipgate.core.findings.constants import SEVERITY_ORDER -from agents_shipgate.report.summary_text import evidence_coverage_text +from agents_shipgate.report.summary_text import ( + evidence_coverage_text, + primary_evidence_remediation_text, +) logger = logging.getLogger(__name__) @@ -445,11 +448,7 @@ def _print_cli_summary(report, ci_mode: str, exit_code: int, *, verbose: bool = ev = decision.evidence_coverage typer.echo(f"Evidence coverage: {evidence_coverage_text(ev)}") if decision.decision == "insufficient_evidence": - typer.echo( - "Improve evidence: provide MCP export, OpenAPI spec, explicit " - "local tool inventory, or a broader OpenAI SDK source path; " - "then rerun scan." - ) + typer.echo(f"Improve evidence: {primary_evidence_remediation_text(ev)}") if report.agent_summary and report.agent_summary.first_recommended_action: action = report.agent_summary.first_recommended_action if action.command: diff --git a/src/agents_shipgate/report/summary_text.py b/src/agents_shipgate/report/summary_text.py index 9aa8ff54..aa583027 100644 --- a/src/agents_shipgate/report/summary_text.py +++ b/src/agents_shipgate/report/summary_text.py @@ -2,6 +2,11 @@ from agents_shipgate.schemas.report import EvidenceCoverageDecision +_GENERIC_EVIDENCE_REMEDIATION = ( + "Provide a complete MCP export, OpenAPI spec, explicit local tool inventory, " + "or a broader supported source path; then rerun the scan." +) + def evidence_coverage_text(evidence: EvidenceCoverageDecision) -> str: extras: list[str] = [] @@ -32,3 +37,24 @@ def evidence_coverage_text(evidence: EvidenceCoverageDecision) -> str: extras.append("human review recommended") suffix = f" ({'; '.join(extras)})" if extras else "" return f"{evidence.level}{suffix}" + + +def primary_evidence_remediation_text(evidence: EvidenceCoverageDecision) -> str: + """Render the decision engine's rank-1 evidence-gap action. + + Short-form surfaces must project ``evidence_gaps[0].next_action`` instead + of replacing framework-specific guidance with a generic source list. The + fallback exists only for older reports that predate structured gap rows. + """ + + if not evidence.evidence_gaps: + return _GENERIC_EVIDENCE_REMEDIATION + + action = evidence.evidence_gaps[0].next_action + if action.command: + text = f"Run: {action.command}. {action.expects}" + else: + text = action.expects + if action.path and action.path not in text: + text = f"{text.rstrip('.')} Target: {action.path}." + return text diff --git a/tests/test_ci.py b/tests/test_ci.py index 2b515179..b4647559 100644 --- a/tests/test_ci.py +++ b/tests/test_ci.py @@ -1,7 +1,17 @@ from pathlib import Path from agents_shipgate.ci.github_summary import write_github_step_summary +from agents_shipgate.ci.release_decision import build_release_decision +from agents_shipgate.cli._helpers import _print_cli_summary from agents_shipgate.cli.scan import run_scan +from agents_shipgate.core.domain import ( + AuthInfo, + AuthoritySemanticAssessment, + EffectSemanticAssessment, + Tool, + ToolSemanticAssessment, +) +from agents_shipgate.schemas.bindings import AgentBindingGraphAssessment from agents_shipgate.schemas.report import ( ReadinessReport, ReportSummary, @@ -14,6 +24,48 @@ ) +def _google_adk_insufficient_evidence_report() -> ReadinessReport: + tool = Tool( + id="tool-lookup-case", + name="lookup_case", + source_type="google_adk_function", + source_location="agent.py:14", + auth=AuthInfo(mode="none", explicit=True), + extraction_confidence="medium", + semantic_assessment=ToolSemanticAssessment( + conservative_effect="read", + effect=EffectSemanticAssessment(status="declared", confidence="high"), + authority=AuthoritySemanticAssessment(status="declared", mode="none"), + pass_eligible=True, + ), + ) + report = ReadinessReport( + run_id="test", + project={"name": "project"}, + agent={"name": "agent"}, + environment={"target": "local"}, + summary=ReportSummary( + status="warnings_detected", + human_review_recommended=True, + ), + tool_surface=ToolSurfaceSummary(total_tools=1, high_risk_tools=0), + binding_surface_facts=AgentBindingGraphAssessment( + root_agent_id="agent", + status="structural", + reachable_tool_ids=[tool.id], + pass_eligible=True, + ), + ) + report.release_decision = build_release_decision( + report=report, + tools=[tool], + ci_mode="advisory", + fail_on=None, + new_findings_only=False, + ) + return report + + def test_github_step_summary_is_written(monkeypatch, tmp_path): summary_path = tmp_path / "summary.md" monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary_path)) @@ -40,6 +92,27 @@ def test_github_step_summary_is_written(monkeypatch, tmp_path): assert "did not execute the agent or prove runtime behavior" in summary +def test_short_summaries_project_framework_specific_evidence_action( + monkeypatch, tmp_path, capsys +): + report = _google_adk_insufficient_evidence_report() + + _print_cli_summary(report, "advisory", 0) + console = capsys.readouterr().out + + summary_path = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary_path)) + write_github_step_summary(report) + github = summary_path.read_text(encoding="utf-8") + + assert "google_adk.tool_inventories" in console + assert "google\\_adk.tool\\_inventories" in github + for output in (console, github): + assert "skeleton written next to report.json" in output + assert "suggested-inventory.json" in output + assert "broader OpenAI SDK source path" not in output + + def test_github_step_summary_escapes_diff_highlights(monkeypatch, tmp_path): summary_path = tmp_path / "summary.md" monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary_path)) From b27d87b1eb7e6f01a0dea5fde7b5595523397b3c Mon Sep 17 00:00:00 2001 From: Pengfei Hu Date: Sun, 9 Aug 2026 14:39:49 -0700 Subject: [PATCH 2/3] Address framework remediation review --- CHANGELOG.md | 11 ++ README.md | 2 +- STABILITY.md | 13 ++- docs/quickstart.md | 2 +- src/agents_shipgate/ci/release_decision.py | 39 +++++-- src/agents_shipgate/cli/verify/command.py | 9 ++ src/agents_shipgate/report/summary_text.py | 6 +- tests/test_ci.py | 124 +++++++++++++-------- tests/test_release_decision.py | 4 + tests/test_verify.py | 60 ++++++++++ 10 files changed, 206 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e429e0e0..bbedf241 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ ## Unreleased +- **Insufficient-evidence remediation now stays framework-aware from the + decision engine through every primary short-form surface.** Semantic + `incomplete_surface` gaps for frameworks with explicit inventory support now + lead with the generated `suggested-inventory.json` artifact and the exact + `.tool_inventories` manifest key instead of an unreachable generic + MCP/OpenAPI route. Console scan output, the GitHub step summary, and text-mode + `verify` all project that same rank-1 action; unsupported source shapes retain + the compatibility fallback. The regression runs a real Google ADK workspace + through static extraction and semantic assessment, so it cannot manufacture + a pass-ineligible medium-confidence tool state. ([#318](https://github.com/ThreeMoonsLab/agents-shipgate/issues/318)) + - **Human review now blocks merge and completion, not publication of the evidence a human needs in order to review.** A human route was one universal stop: `control.state: "human_review_required"` with `must_stop: true` and diff --git a/README.md b/README.md index 50b86a88..9c02ba8d 100644 --- a/README.md +++ b/README.md @@ -251,7 +251,7 @@ Then read `report.json.release_decision.decision`, the source-of-truth gate: | Decision | Meaning | Next step | |---|---|---| | `blocked` | Active, unaccepted blockers exist. | Fix the blockers or remove the risky tool surface. | -| `insufficient_evidence` | The scan cannot confidently gate release from the available static evidence. This does not prove the agent is unsafe. | Provide clearer sources such as an MCP export, OpenAPI spec, explicit local tool inventory, or broader OpenAI SDK source path, then rerun. | +| `insufficient_evidence` | The scan cannot confidently gate release from the available static evidence. This does not prove the agent is unsafe. | Follow the first structured evidence-gap action. Supported frameworks name the generated local inventory and exact manifest route; unidentified source shapes receive the generic source guidance. Then rerun. | | `review_required` | Human review is needed, often for accepted debt or evidence gaps below the blocked threshold. | Review the listed items before promotion. | | `passed` | No active blocker or review signal was found. | Keep the report artifact with the PR/release record. | diff --git a/STABILITY.md b/STABILITY.md index eedbd98c..2c37324c 100644 --- a/STABILITY.md +++ b/STABILITY.md @@ -1101,11 +1101,14 @@ takes precedence over both. The precedence is therefore: → `review_required` (other) → `passed`. The intended recovery for a degraded-evidence case — whichever of the two -verdicts it lands on — is to provide clearer local evidence — for example an MCP -export, OpenAPI spec, explicit local tool inventory, broader OpenAI Agents SDK -source path, or validation trace — and rerun the scan. When the decision is -`review_required` because of an active high/critical finding, also resolve that -finding. `agents-shipgate verify` keeps both cases human-routed +verdicts it lands on — is the first structured action in +`release_decision.evidence_coverage.evidence_gaps[]`. For supported frameworks, +that action names the generated local inventory artifact and the exact +`.tool_inventories` manifest route. Only unidentified or unsupported +source shapes receive generic MCP/OpenAPI/inventory guidance. Apply the reviewed +evidence route and rerun the scan. When the decision is `review_required` +because of an active high/critical finding, also resolve that finding. +`agents-shipgate verify` keeps both cases human-routed (`fix_task.actor = "human"`): a degraded-evidence case never opens an automated coding-agent fix path, regardless of which verdict it carries. diff --git a/docs/quickstart.md b/docs/quickstart.md index 4a4225f9..c539353d 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -163,7 +163,7 @@ Then read `report.json.release_decision.decision`, the source-of-truth gate: | Decision | Meaning | Next action | | --- | --- | --- | | `blocked` | Active, unaccepted blockers exist. | Fix blockers or remove the risky tool surface. | -| `insufficient_evidence` | The scan cannot confidently gate release from the available static evidence; this does not prove the agent is unsafe. | Provide an MCP export, OpenAPI spec, explicit local tool inventory, or broader OpenAI SDK source path, then rerun. | +| `insufficient_evidence` | The scan cannot confidently gate release from the available static evidence; this does not prove the agent is unsafe. | Follow the first structured evidence-gap action. Supported frameworks name the generated local inventory and exact manifest route; unidentified source shapes receive the generic source guidance. Then rerun. | | `review_required` | Human review is needed for accepted debt or evidence gaps below the blocked threshold. | Review the listed items before promotion. | | `passed` | No active blocker or review signal was found. | Keep the report artifact with the PR/release record. | diff --git a/src/agents_shipgate/ci/release_decision.py b/src/agents_shipgate/ci/release_decision.py index 119386b4..db9867f5 100644 --- a/src/agents_shipgate/ci/release_decision.py +++ b/src/agents_shipgate/ci/release_decision.py @@ -723,17 +723,32 @@ def _semantic_gap( action_why = "Conflicting binding evidence cannot be auto-resolved." expects = "Reconcile positive structural evidence and reviewed declarations, then rerun verification." elif kind == "incomplete_surface": - action_kind = "provide_complete_inventory" - accepted_values = [ - "complete_mcp_export", - "openapi_spec", - "reviewed_explicit_inventory", - ] - action_why = "The complete statically-bound tool surface must be enumerable." - expects = ( - "Provide a complete MCP export, OpenAPI spec, or reviewed explicit " - "tool inventory, then rerun verification." - ) + manifest_key = _inventory_manifest_key(tool.source_type) + if manifest_key is not None: + action_kind = "declare_tool_inventory" + accepted_values = ["reviewed_explicit_inventory"] + action_why = ( + f"{tool.source_type} extraction is static-only; an explicit " + "local tool inventory is the supported way to make the full " + "surface enumerable." + ) + expects = ( + "Review the skeleton written next to report.json, save it in " + f"your repo, reference it from `{manifest_key}` in " + "shipgate.yaml, then rerun verification." + ) + else: + action_kind = "provide_complete_inventory" + accepted_values = [ + "complete_mcp_export", + "openapi_spec", + "reviewed_explicit_inventory", + ] + action_why = "The complete statically-bound tool surface must be enumerable." + expects = ( + "Provide a complete MCP export, OpenAPI spec, or reviewed explicit " + "tool inventory, then rerun verification." + ) elif kind in { "missing_effect_evidence", "inferred_effect_only", @@ -854,6 +869,8 @@ def _semantic_gap_path(kind: str, tool: Tool) -> str: action_row = f"shipgate.yaml#action_surface.actions[tool={tool.name!r}]" if kind == "incomplete_surface": + if _inventory_manifest_key(tool.source_type) is not None: + return SUGGESTED_INVENTORY_FILENAME return "shipgate.yaml#tool_sources" if kind in _SELECTOR_KINDS: return action_row diff --git a/src/agents_shipgate/cli/verify/command.py b/src/agents_shipgate/cli/verify/command.py index 717251ec..6383157f 100644 --- a/src/agents_shipgate/cli/verify/command.py +++ b/src/agents_shipgate/cli/verify/command.py @@ -19,6 +19,7 @@ from agents_shipgate.core.disclaimers import STATIC_VERDICT_DISCLAIMER from agents_shipgate.core.errors import AgentsShipgateError, ConfigError, InputParseError from agents_shipgate.core.logging import configure_logging +from agents_shipgate.report.summary_text import primary_evidence_remediation_text from agents_shipgate.schemas.diagnostics import NextAction from .git import ensure_git_workspace, staged_paths_under @@ -427,6 +428,14 @@ def verify( typer.echo(f"Trigger: {verifier.trigger.get('rationale')}") typer.echo(f"Base status: {verifier.base_status}") typer.echo(f"Exit code: {exit_code}") + if ( + verifier.release_decision is not None + and verifier.release_decision.decision == "insufficient_evidence" + ): + typer.echo( + "Improve evidence: " + f"{primary_evidence_remediation_text(verifier.release_decision.evidence_coverage)}" + ) typer.echo(f"Static-verdict boundary: {STATIC_VERDICT_DISCLAIMER}") raise typer.Exit(exit_code) diff --git a/src/agents_shipgate/report/summary_text.py b/src/agents_shipgate/report/summary_text.py index aa583027..93629291 100644 --- a/src/agents_shipgate/report/summary_text.py +++ b/src/agents_shipgate/report/summary_text.py @@ -47,6 +47,8 @@ def primary_evidence_remediation_text(evidence: EvidenceCoverageDecision) -> str fallback exists only for older reports that predate structured gap rows. """ + # Current decisions always carry a structured gap here. Only compatibility + # reports from before evidence_gaps existed can reach this fallback. if not evidence.evidence_gaps: return _GENERIC_EVIDENCE_REMEDIATION @@ -56,5 +58,7 @@ def primary_evidence_remediation_text(evidence: EvidenceCoverageDecision) -> str else: text = action.expects if action.path and action.path not in text: - text = f"{text.rstrip('.')} Target: {action.path}." + if not text.endswith((".", "!", "?")): + text = f"{text}." + text = f"{text} Target: {action.path}." return text diff --git a/tests/test_ci.py b/tests/test_ci.py index b4647559..327eea56 100644 --- a/tests/test_ci.py +++ b/tests/test_ci.py @@ -1,18 +1,13 @@ from pathlib import Path from agents_shipgate.ci.github_summary import write_github_step_summary -from agents_shipgate.ci.release_decision import build_release_decision from agents_shipgate.cli._helpers import _print_cli_summary from agents_shipgate.cli.scan import run_scan -from agents_shipgate.core.domain import ( - AuthInfo, - AuthoritySemanticAssessment, - EffectSemanticAssessment, - Tool, - ToolSemanticAssessment, -) -from agents_shipgate.schemas.bindings import AgentBindingGraphAssessment +from agents_shipgate.report.summary_text import primary_evidence_remediation_text from agents_shipgate.schemas.report import ( + EvidenceCoverageDecision, + EvidenceGap, + EvidenceGapAction, ReadinessReport, ReportSummary, ToolSurfaceSummary, @@ -24,44 +19,48 @@ ) -def _google_adk_insufficient_evidence_report() -> ReadinessReport: - tool = Tool( - id="tool-lookup-case", - name="lookup_case", - source_type="google_adk_function", - source_location="agent.py:14", - auth=AuthInfo(mode="none", explicit=True), - extraction_confidence="medium", - semantic_assessment=ToolSemanticAssessment( - conservative_effect="read", - effect=EffectSemanticAssessment(status="declared", confidence="high"), - authority=AuthoritySemanticAssessment(status="declared", mode="none"), - pass_eligible=True, - ), +def _scan_google_adk_insufficient_evidence_project(tmp_path) -> ReadinessReport: + project = tmp_path / "google-adk-project" + project.mkdir() + (project / "agent.py").write_text( + ''' +from google.adk.agents import LlmAgent +from google.adk.tools import FunctionTool + + +def lookup_case(case_id: str) -> dict: + """Look up read-only support case metadata.""" + return {"case_id": case_id} + + +lookup_tool = FunctionTool(func=lookup_case) +root_agent = LlmAgent(name="support_reader", tools=[lookup_tool]) +'''.lstrip(), + encoding="utf-8", ) - report = ReadinessReport( - run_id="test", - project={"name": "project"}, - agent={"name": "agent"}, - environment={"target": "local"}, - summary=ReportSummary( - status="warnings_detected", - human_review_recommended=True, - ), - tool_surface=ToolSurfaceSummary(total_tools=1, high_risk_tools=0), - binding_surface_facts=AgentBindingGraphAssessment( - root_agent_id="agent", - status="structural", - reachable_tool_ids=[tool.id], - pass_eligible=True, - ), + (project / "shipgate.yaml").write_text( + ''' +version: "0.1" +project: + name: google-adk-remediation +agent: + name: support-reader + declared_purpose: [read support case metadata] +environment: + target: local +tool_sources: + - id: adk + type: google_adk + path: agent.py +'''.lstrip(), + encoding="utf-8", ) - report.release_decision = build_release_decision( - report=report, - tools=[tool], + report, _ = run_scan( + config_path=project / "shipgate.yaml", + output_dir=tmp_path / "reports", + formats=["json"], ci_mode="advisory", - fail_on=None, - new_findings_only=False, + packet_enabled=False, ) return report @@ -95,7 +94,15 @@ def test_github_step_summary_is_written(monkeypatch, tmp_path): def test_short_summaries_project_framework_specific_evidence_action( monkeypatch, tmp_path, capsys ): - report = _google_adk_insufficient_evidence_report() + report = _scan_google_adk_insufficient_evidence_project(tmp_path) + assert report.release_decision is not None + assert report.release_decision.decision == "insufficient_evidence" + first_gap = report.release_decision.evidence_coverage.evidence_gaps[0] + assert first_gap.kind == "incomplete_surface" + assert first_gap.next_action.kind == "declare_tool_inventory" + assert first_gap.next_action.path == "suggested-inventory.json" + assert "google_adk.tool_inventories" in first_gap.next_action.expects + assert (tmp_path / "reports" / "suggested-inventory.json").is_file() _print_cli_summary(report, "advisory", 0) console = capsys.readouterr().out @@ -110,9 +117,36 @@ def test_short_summaries_project_framework_specific_evidence_action( for output in (console, github): assert "skeleton written next to report.json" in output assert "suggested-inventory.json" in output + assert "verification. Target: suggested-inventory.json." in output assert "broader OpenAI SDK source path" not in output +def test_primary_evidence_remediation_preserves_terminal_ellipsis(): + evidence = EvidenceCoverageDecision( + level="partial", + human_review_recommended=True, + source_warning_count=1, + low_confidence_tool_count=0, + evidence_gaps=[ + EvidenceGap( + kind="source_warning", + subject="legacy source", + why="The source requires review.", + next_action=EvidenceGapAction( + kind="review_warning", + path="source-notes.json", + why="The source warning must be resolved.", + expects="See source notes...", + ), + ) + ], + ) + + assert primary_evidence_remediation_text(evidence) == ( + "See source notes... Target: source-notes.json." + ) + + def test_github_step_summary_escapes_diff_highlights(monkeypatch, tmp_path): summary_path = tmp_path / "summary.md" monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary_path)) diff --git a/tests/test_release_decision.py b/tests/test_release_decision.py index 06b27f62..5d632225 100644 --- a/tests/test_release_decision.py +++ b/tests/test_release_decision.py @@ -965,6 +965,10 @@ def test_evidence_gaps_low_confidence_tool_points_at_inventory(): "incomplete_surface", "low_confidence_tool", ] + first_gap = gaps[0] + assert first_gap.next_action.kind == "declare_tool_inventory" + assert first_gap.next_action.path == "suggested-inventory.json" + assert "langchain.tool_inventories" in first_gap.next_action.expects gap = gaps[1] assert gap.kind == "low_confidence_tool" assert gap.subject.startswith("lookup_case [") diff --git a/tests/test_verify.py b/tests/test_verify.py index 5207711a..c70388bf 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -850,6 +850,66 @@ def test_verify_real_base_scan_enables_head_diff(tmp_path: Path) -> None: assert report_payload["release_decision"]["decision"] == "insufficient_evidence" +def test_verify_text_projects_google_adk_primary_evidence_action(tmp_path: Path) -> None: + repo = _init_repo(tmp_path) + (repo / "agent.py").write_text( + ''' +from google.adk.agents import LlmAgent +from google.adk.tools import FunctionTool + + +def lookup_case(case_id: str) -> dict: + """Look up read-only support case metadata.""" + return {"case_id": case_id} + + +lookup_tool = FunctionTool(func=lookup_case) +root_agent = LlmAgent(name="support_reader", tools=[lookup_tool]) +'''.lstrip(), + encoding="utf-8", + ) + (repo / "shipgate.yaml").write_text( + ''' +version: "0.1" +project: + name: google-adk-remediation +agent: + name: support-reader + declared_purpose: [read support case metadata] +environment: + target: local +tool_sources: + - id: adk + type: google_adk + path: agent.py +'''.lstrip(), + encoding="utf-8", + ) + _commit_all(repo, "add google adk agent") + + result = runner.invoke( + app, + [ + "verify", + "--workspace", + str(repo), + "--config", + "shipgate.yaml", + "--no-base", + "--format", + "text", + ], + ) + + assert result.exit_code == 0, result.output + assert result.output.startswith("Agents Shipgate verify: insufficient_evidence") + assert "Improve evidence: Run: agents-shipgate verify" in result.output + assert "google_adk.tool_inventories" in result.output + assert "suggested-inventory.json" in result.output + assert "verification. Target: suggested-inventory.json." in result.output + assert "broader OpenAI SDK source path" not in result.output + + def test_pr_comment_keeps_code_span_values_unescaped() -> None: verifier = VerifierArtifact( workspace="/tmp/work", From 73d0757bd5acb0495aeb4b19d04d30a13258be7d Mon Sep 17 00:00:00 2001 From: Pengfei Hu Date: Sun, 9 Aug 2026 18:30:01 -0700 Subject: [PATCH 3/3] Polish framework remediation follow-up --- CHANGELOG.md | 9 ++-- docs/agent-contract-current.md | 2 +- llms-full.txt | 2 +- src/agents_shipgate/cli/verify/fix_task.py | 5 +- src/agents_shipgate/report/summary_text.py | 7 ++- tests/test_ci.py | 1 + tests/test_fix_task_contract.py | 55 ++++++++++++++++++++++ tests/test_verify.py | 4 +- 8 files changed, 74 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbedf241..3b5b7270 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,12 @@ `.tool_inventories` manifest key instead of an unreachable generic MCP/OpenAPI route. Console scan output, the GitHub step summary, and text-mode `verify` all project that same rank-1 action; unsupported source shapes retain - the compatibility fallback. The regression runs a real Google ADK workspace - through static extraction and semantic assessment, so it cannot manufacture - a pass-ineligible medium-confidence tool state. ([#318](https://github.com/ThreeMoonsLab/agents-shipgate/issues/318)) + the compatibility fallback. Human work now precedes the exact rerun command + in text output, and verifier fix tasks collapse the duplicate semantic and + extraction inventory remedies into one instruction. The regression runs a + real Google ADK workspace through static extraction and semantic assessment, + so it cannot manufacture a pass-ineligible medium-confidence tool state. + ([#318](https://github.com/ThreeMoonsLab/agents-shipgate/issues/318)) - **Human review now blocks merge and completion, not publication of the evidence a human needs in order to review.** A human route was one universal diff --git a/docs/agent-contract-current.md b/docs/agent-contract-current.md index 300ebcd1..a52107a1 100644 --- a/docs/agent-contract-current.md +++ b/docs/agent-contract-current.md @@ -291,7 +291,7 @@ In `agents-shipgate-reports/report.json`: - `release_decision.evidence_coverage.semantic_coverage` (v0.29+) — `{total_actions, pass_eligible_actions, gap_count, review_concern_count, reason_counts}`. A non-zero semantic `gap_count` prevents `passed`; a non-zero `review_concern_count` prevents an automatic pass and routes known unscoped/ambient authority to review. Semantic gaps are not Findings and cannot be suppressed, baselined, severity-overridden, waived by `--no-heuristics`, or satisfied by `human_ack`. - `release_decision.evidence_coverage.policy_gap_count` and top-level `policy_evidence_gaps[]` (v0.33+) — policy applicability that is heuristic-only, mixed, unknown, or conflicting. These rows are outside Findings and cannot be suppressed, baselined, severity-overridden, acknowledged, or removed by `--no-heuristics`; any row prevents `passed`. - `release_decision.evidence_coverage.identity_coverage` (v0.30+) — `{total_observations, canonical_tools, bound_tools, pass_eligible_tools, ambiguous_name_count, gap_count, reason_counts}`. Provider-scoped observations remain separate unless an exact reviewed `tool_identity.bindings[]` entry joins them. Any ambiguous selector, invalid binding, or conflicting identity prevents `passed`. -- `release_decision.evidence_coverage.evidence_gaps[]` (v0.26+; semantic kinds added v0.29) — one structured row per measurable gap: `{kind, subject, source_type, source_ref, why, next_action}`. In addition to `low_confidence_tool` and `source_warning`, v0.29 adds `incomplete_surface`, `missing_effect_evidence`, `inferred_effect_only`, `conflicting_effect_evidence`, `missing_authority_evidence`, `partial_authority_evidence`, `conflicting_authority_evidence`, and `invalid_semantic_annotation`. Semantic next actions use `declare_action_effect`, `declare_action_authority`, `provide_complete_inventory`, or `resolve_semantic_conflict`, include accepted values and exact source/manifest pointers, and are always human-routed. Their declaration placeholders carry `suggested_patch_kind="manual"`, `auto_apply=false`, and `requires_human_review=true`; they are not Patch objects. Work the rows in order instead of guessing; Agents Shipgate never auto-asserts effect or authority. +- `release_decision.evidence_coverage.evidence_gaps[]` (v0.26+; semantic kinds added v0.29) — one structured row per measurable gap: `{kind, subject, source_type, source_ref, why, next_action}`. In addition to `low_confidence_tool` and `source_warning`, v0.29 adds `incomplete_surface`, `missing_effect_evidence`, `inferred_effect_only`, `conflicting_effect_evidence`, `missing_authority_evidence`, `partial_authority_evidence`, `conflicting_authority_evidence`, and `invalid_semantic_annotation`. Semantic next actions use `declare_action_effect`, `declare_action_authority`, `declare_tool_inventory`, `provide_complete_inventory`, or `resolve_semantic_conflict`, include accepted values and exact source/manifest pointers, and are always human-routed. Their declaration placeholders carry `suggested_patch_kind="manual"`, `auto_apply=false`, and `requires_human_review=true`; they are not Patch objects. Work the rows in order instead of guessing; Agents Shipgate never auto-asserts effect or authority. - `loaded_policy_packs[].{source,sha256,sha256_status,owner}` (v0.27+) — policy-pack distribution and ownership metadata for organization audit. `sha256_status` is `"verified"` only when the manifest pin matched; otherwise it is `"unpinned"`. This is report metadata; normal pack matching and release gating still come from deterministic rules and `release_decision.decision`. - `findings[].support` (v0.33+) — typed predicate support with status, effective confidence, policy/block eligibility, claim IDs, evidence bases, predicate rows, and `support_hash`. Rule confidence and `block: true` are ceilings/requests; they cannot upgrade the support. Baseline matching for supported findings requires the same support hash. - `findings[].policy_routing` (v0.28+) — optional policy-pack owner, reviewers, and approval-routing metadata. This is non-enforcing reviewer/audit metadata, not `Finding.evidence`; it does not affect fingerprints, suppressions, baselines, `blocks_release`, or `release_decision`. diff --git a/llms-full.txt b/llms-full.txt index 8082bc6b..5a018b2e 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -1350,7 +1350,7 @@ In `agents-shipgate-reports/report.json`: - `release_decision.evidence_coverage.semantic_coverage` (v0.29+) — `{total_actions, pass_eligible_actions, gap_count, review_concern_count, reason_counts}`. A non-zero semantic `gap_count` prevents `passed`; a non-zero `review_concern_count` prevents an automatic pass and routes known unscoped/ambient authority to review. Semantic gaps are not Findings and cannot be suppressed, baselined, severity-overridden, waived by `--no-heuristics`, or satisfied by `human_ack`. - `release_decision.evidence_coverage.policy_gap_count` and top-level `policy_evidence_gaps[]` (v0.33+) — policy applicability that is heuristic-only, mixed, unknown, or conflicting. These rows are outside Findings and cannot be suppressed, baselined, severity-overridden, acknowledged, or removed by `--no-heuristics`; any row prevents `passed`. - `release_decision.evidence_coverage.identity_coverage` (v0.30+) — `{total_observations, canonical_tools, bound_tools, pass_eligible_tools, ambiguous_name_count, gap_count, reason_counts}`. Provider-scoped observations remain separate unless an exact reviewed `tool_identity.bindings[]` entry joins them. Any ambiguous selector, invalid binding, or conflicting identity prevents `passed`. -- `release_decision.evidence_coverage.evidence_gaps[]` (v0.26+; semantic kinds added v0.29) — one structured row per measurable gap: `{kind, subject, source_type, source_ref, why, next_action}`. In addition to `low_confidence_tool` and `source_warning`, v0.29 adds `incomplete_surface`, `missing_effect_evidence`, `inferred_effect_only`, `conflicting_effect_evidence`, `missing_authority_evidence`, `partial_authority_evidence`, `conflicting_authority_evidence`, and `invalid_semantic_annotation`. Semantic next actions use `declare_action_effect`, `declare_action_authority`, `provide_complete_inventory`, or `resolve_semantic_conflict`, include accepted values and exact source/manifest pointers, and are always human-routed. Their declaration placeholders carry `suggested_patch_kind="manual"`, `auto_apply=false`, and `requires_human_review=true`; they are not Patch objects. Work the rows in order instead of guessing; Agents Shipgate never auto-asserts effect or authority. +- `release_decision.evidence_coverage.evidence_gaps[]` (v0.26+; semantic kinds added v0.29) — one structured row per measurable gap: `{kind, subject, source_type, source_ref, why, next_action}`. In addition to `low_confidence_tool` and `source_warning`, v0.29 adds `incomplete_surface`, `missing_effect_evidence`, `inferred_effect_only`, `conflicting_effect_evidence`, `missing_authority_evidence`, `partial_authority_evidence`, `conflicting_authority_evidence`, and `invalid_semantic_annotation`. Semantic next actions use `declare_action_effect`, `declare_action_authority`, `declare_tool_inventory`, `provide_complete_inventory`, or `resolve_semantic_conflict`, include accepted values and exact source/manifest pointers, and are always human-routed. Their declaration placeholders carry `suggested_patch_kind="manual"`, `auto_apply=false`, and `requires_human_review=true`; they are not Patch objects. Work the rows in order instead of guessing; Agents Shipgate never auto-asserts effect or authority. - `loaded_policy_packs[].{source,sha256,sha256_status,owner}` (v0.27+) — policy-pack distribution and ownership metadata for organization audit. `sha256_status` is `"verified"` only when the manifest pin matched; otherwise it is `"unpinned"`. This is report metadata; normal pack matching and release gating still come from deterministic rules and `release_decision.decision`. - `findings[].support` (v0.33+) — typed predicate support with status, effective confidence, policy/block eligibility, claim IDs, evidence bases, predicate rows, and `support_hash`. Rule confidence and `block: true` are ceilings/requests; they cannot upgrade the support. Baseline matching for supported findings requires the same support hash. - `findings[].policy_routing` (v0.28+) — optional policy-pack owner, reviewers, and approval-routing metadata. This is non-enforcing reviewer/audit metadata, not `Finding.evidence`; it does not affect fingerprints, suppressions, baselines, `blocks_release`, or `release_decision`. diff --git a/src/agents_shipgate/cli/verify/fix_task.py b/src/agents_shipgate/cli/verify/fix_task.py index b0831f39..b2a8d43d 100644 --- a/src/agents_shipgate/cli/verify/fix_task.py +++ b/src/agents_shipgate/cli/verify/fix_task.py @@ -443,7 +443,10 @@ def _insufficient_evidence_remedies(report: ReadinessReport) -> list[str]: decision = report.release_decision assert decision is not None for gap in decision.evidence_coverage.evidence_gaps: - if gap.kind in {"low_confidence_tool", "source_warning"}: + if gap.kind in {"low_confidence_tool", "source_warning"} or ( + gap.kind == "incomplete_surface" + and gap.next_action.kind == "declare_tool_inventory" + ): continue action = gap.next_action accepted = ( diff --git a/src/agents_shipgate/report/summary_text.py b/src/agents_shipgate/report/summary_text.py index 93629291..112efc90 100644 --- a/src/agents_shipgate/report/summary_text.py +++ b/src/agents_shipgate/report/summary_text.py @@ -53,12 +53,11 @@ def primary_evidence_remediation_text(evidence: EvidenceCoverageDecision) -> str return _GENERIC_EVIDENCE_REMEDIATION action = evidence.evidence_gaps[0].next_action - if action.command: - text = f"Run: {action.command}. {action.expects}" - else: - text = action.expects + text = action.expects if action.path and action.path not in text: if not text.endswith((".", "!", "?")): text = f"{text}." text = f"{text} Target: {action.path}." + if action.command: + text = f"{text}\nRun: {action.command}" return text diff --git a/tests/test_ci.py b/tests/test_ci.py index 327eea56..75ce2bc1 100644 --- a/tests/test_ci.py +++ b/tests/test_ci.py @@ -118,6 +118,7 @@ def test_short_summaries_project_framework_specific_evidence_action( assert "skeleton written next to report.json" in output assert "suggested-inventory.json" in output assert "verification. Target: suggested-inventory.json." in output + assert output.index("Review the skeleton") < output.index("Run:") assert "broader OpenAI SDK source path" not in output diff --git a/tests/test_fix_task_contract.py b/tests/test_fix_task_contract.py index a275b91b..ff60b0d7 100644 --- a/tests/test_fix_task_contract.py +++ b/tests/test_fix_task_contract.py @@ -786,6 +786,61 @@ def test_insufficient_evidence_names_low_confidence_sources() -> None: assert "mcp-tools.json" not in joined +def test_inventory_semantic_gap_does_not_duplicate_low_confidence_remedy() -> None: + inventory_action = EvidenceGapAction( + kind="declare_tool_inventory", + path="suggested-inventory.json", + why="The complete tool surface must be enumerable.", + expects=( + "Review the skeleton and reference it from " + "`langchain.tool_inventories`." + ), + ) + report = _report( + decision="insufficient_evidence", + findings=[], + low_confidence_tool_count=1, + semantic_gap_count=1, + tool_inventory=[ + { + "name": "lookup_case", + "source_type": "langchain_function", + "source_ref": "agent.py", + "confidence": "medium", + } + ], + evidence_gaps=[ + EvidenceGap( + kind="incomplete_surface", + subject="lookup_case [langchain]", + source_type="langchain_function", + source_ref="agent.py", + why="Static extraction did not prove the complete surface.", + next_action=inventory_action, + ), + EvidenceGap( + kind="low_confidence_tool", + subject="lookup_case [langchain]", + source_type="langchain_function", + source_ref="agent.py", + why="extraction_confidence=medium", + next_action=inventory_action, + ), + ], + ) + + task = _fix_task(report) + + assert task is not None and task.actor == "human" + inventory_instructions = [ + instruction + for instruction in task.instructions + if "langchain.tool_inventories" in instruction + ] + assert len(inventory_instructions) == 1 + assert not any("Review the skeleton" in item for item in task.instructions) + + def test_insufficient_evidence_without_inventory_gives_generic_remedy() -> None: f = _with_applicable_patch( _finding("F1", requires_human_review=False, autofix_safe=True) diff --git a/tests/test_verify.py b/tests/test_verify.py index c70388bf..cca0dd78 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -903,7 +903,9 @@ def lookup_case(case_id: str) -> dict: assert result.exit_code == 0, result.output assert result.output.startswith("Agents Shipgate verify: insufficient_evidence") - assert "Improve evidence: Run: agents-shipgate verify" in result.output + assert "Improve evidence: Review the skeleton" in result.output + assert "\nRun: agents-shipgate verify" in result.output + assert result.output.index("Review the skeleton") < result.output.index("Run:") assert "google_adk.tool_inventories" in result.output assert "suggested-inventory.json" in result.output assert "verification. Target: suggested-inventory.json." in result.output