From 5123f8cc0fefd6119d0c4ff8e31c9f8f3082a3bc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 15:17:50 +0900 Subject: [PATCH 01/80] docs: define reusable repository README quality standard --- docs/repository-readme-quality-standard.md | 211 +++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 docs/repository-readme-quality-standard.md diff --git a/docs/repository-readme-quality-standard.md b/docs/repository-readme-quality-standard.md new file mode 100644 index 0000000000..3d3d5895ea --- /dev/null +++ b/docs/repository-readme-quality-standard.md @@ -0,0 +1,211 @@ +# ContextualWisdomLab repository README quality standard + +## Purpose + +Every ContextualWisdomLab repository owns its own product README because the README must be reviewed against that repository's actual code, PRD, ADRs, release state, security boundary, and license provenance. The organization control plane may define a reusable quality standard, but it must not manufacture product claims or overwrite repository-specific language. + +This document is the shared review pattern for repository landing pages. It is intentionally a **quality contract, not a copy-and-paste template**. A good README should feel consistent across the organization while still making the product's bounded context, terminology, operating reality, and obligations obvious. + +## Reader jobs + +A root README should help four readers reach a safe next action quickly: + +1. **Prospective user or buyer** — understand what problem the product solves, what it does today, and what it deliberately does not claim. +2. **Integrator** — understand how to install or consume it, the public integration boundary, and which neighboring product owns adjacent authority. +3. **Maintainer** — find the verification, architecture, contribution, security, and release evidence without exposing internal automation procedure as customer copy. +4. **Diligence reviewer** — distinguish source metadata from released artifacts, first-party licensing from dependency licensing, and implemented capability from roadmap or active-PR evidence. + +The first screen should answer "what is this, why would I use it, and what can I do next?" before explaining implementation internals. + +## Recommended information architecture + +Use the sections that are relevant to the repository. Do not add empty headings merely for visual consistency. + +### 1. Product name and one-line promise + +Start with the exact repository/product casing and a one- or two-sentence value proposition written in domain language. Prefer the user outcome over an internal technology inventory. + +Good: + +> Generate browsable static directory indexes without running a dynamic listing service. + +Weak: + +> Kotlin 1.3 CLI using Clikt, Gradle, and JaCoCo. + +Technology belongs later unless the technology itself is the product. + +### 2. Product boundary and non-goals + +State what the repository owns and, where confusion is likely, what it does not own. Keep adjacent ContextualWisdomLab products behind explicit integration boundaries rather than making a leaf repository sound like the whole platform. + +Useful boundary language includes: + +- source system remains authoritative; +- this library computes X but does not decide Y; +- this adapter consumes a released/versioned contract but does not own the foreign product's database; +- this documentation/source version is not release or deployment evidence. + +Do not expose private table names, secret names, internal incident procedures, raw infrastructure topology, or maintainer-only automation unless a customer genuinely needs them to use the product safely. + +### 3. Install or quick start + +Provide the shortest **truthful, code-current** path to a useful result. Verify every command against current package metadata, lockfiles, Makefiles, build files, Compose files, CLI help, or tests. + +Rules: + +- do not advertise a package registry installation when only source checkout is supported; +- do not claim a hosted service exists because local Compose exists; +- do not use private sibling checkouts as a public installation contract; +- include real runtime prerequisites and version floors when they are enforced; +- keep irreversible or privileged actions out of the default quick start unless the product inherently requires them and the safety boundary is explicit. + +If the repository is architecture-only or pre-runtime, say so instead of inventing an installation section. Give the reader the correct next action, such as reading the contract or running repository validation. + +### 4. Common usage or public API + +Show the stable user/integrator surface, not a tour of internal modules. Prefer one representative example plus links to complete reference material. + +For libraries, name exported/public symbols and their responsibility. For services, name supported public endpoints only when they are current code truth. For CLI products, show the primary task-oriented commands and keep exhaustive flags in generated help/reference docs. + +### 5. Architecture and integration context + +Explain enough architecture to prevent misuse: + +- core responsibility; +- main data/evidence flow; +- authority boundaries; +- optional versus required integrations; +- local versus external processing where relevant; +- security or privacy boundary that changes how a user should operate the product. + +Link to ADRs, PRD/TRD, architecture diagrams, API schemas, or operator runbooks for detail. The README should navigate to technical authority rather than duplicate it until the two inevitably drift. + +### 6. Status and quality signals + +Status claims must be exact and durable. + +Prefer: + +- "package metadata declares version 0.5-9"; +- "this is a pre-release architecture foundation"; +- "the repository contains CI/SAST/security workflows; inspect the exact revision's results". + +Avoid: + +- treating a package version as proof of a published release; +- copying mutable PR head SHAs or run IDs into the root README; +- badges for workflows that do not exist or no longer represent the protected branch; +- unsupported benchmark, customer, certification, adoption, availability, or production-readiness claims. + +Mutable exact-head integration evidence belongs in PR descriptions, gap ledgers, or generated evidence—not evergreen customer copy. + +### 7. Documentation map and support + +Give readers a compact map to the canonical sources that actually exist. Typical links include: + +- documentation home; +- architecture / ADR index; +- API or schema reference; +- security policy / private vulnerability reporting; +- contribution guidance; +- changelog and releases; +- advanced operator reference when detailed runbooks were intentionally moved out of the README. + +Do not link to planned files or Pages sites that are not published. A `docs/index.md` source is not proof that GitHub Pages is live. + +### 8. Contribution guidance + +State the smallest useful contributor contract: how to verify changes, what public boundary must remain stable, and where deeper maintainer instructions live. Avoid turning the customer README into an hourly-agent or PR-automation manual. + +### 9. License and commercial-use boundary + +A README license section is evidence-backed, not ceremonial. Before writing it, perform repository-level due diligence. + +Distinguish all of the following: + +1. **license of repository-authored source/documentation**; +2. **inherited/copied/derived source obligations**; +3. **third-party runtime/build/test dependency licenses**; +4. **vendored assets, models, datasets, fonts, standards-derived material, container bases, or binaries**; +5. **external service/provider terms**. + +Do not say "MIT" or "Apache-2.0" merely because the organization prefers those licenses. Preserve valid existing MIT/Apache lineage. If the repository is wholly ContextualWisdomLab-authored and provenance establishes the necessary rights, add the appropriate permissive root license and matching package metadata in the authoritative PR. + +If rights are inherited or uncertain, do not silently override them. Examples: + +- a package whose metadata declares GPL and names upstream/external copyright holders remains GPL unless sufficient rights for relicensing are established; +- a repository MIT license does not relicense an LGPL/GPL dependency; +- a dependency's permissive license does not grant a license to otherwise unlicensed repository source; +- absence of a root `LICENSE` is a diligence prompt, not a reason to omit the question. + +Under current ContextualWisdomLab commercial-intake policy, GPL/LGPL/AGPL-family source or dependencies are not an approved default inbound baseline. GPL can permit commercial use under its terms; the policy issue is copyleft/distribution compatibility, not a claim that GPL is a noncommercial license. Record the exact component and obligation, then remove/replace it when safe or preserve a precise provenance blocker and closure evidence. + +## README anti-patterns + +Treat these as review findings when they materially reduce clarity or accuracy: + +- implementation inventory before product value; +- internal PR/agent instructions in customer copy; +- mutable check/PR status persisted as evergreen product truth; +- stale personal/upstream installation URLs after repository ownership changes; +- giant runbooks that bury the supported public workflow; +- unverified badges, customer logos, certifications, benchmarks, or release claims; +- copy-pasted architecture that contradicts the current PRD/ADR/code; +- claiming one service owns data or authorization that actually belongs to another bounded context; +- saying "commercial friendly" while a known GPL-family inbound blocker remains; +- adding a new permissive LICENSE solely because none existed, without provenance review; +- hiding third-party license obligations behind the repository's first-party license; +- creating a duplicate README PR when an existing writable product/documentation branch already owns the file. + +When a README has valuable but overly detailed operator content, prefer moving that content intact to a durable advanced/reference document and linking to it from a concise landing page rather than deleting knowledge. + +## Evidence checklist before editing + +Read the smallest authoritative set necessary to verify claims. Depending on repository type, inspect: + +- current protected/default branch and open README/documentation/license PRs; +- root README and documentation index; +- PRD/TRD/product-planning documents; +- accepted ADRs and architecture docs; +- package/build metadata (`pyproject.toml`, `package.json`, `Cargo.toml`, `DESCRIPTION`, Gradle files, etc.); +- lockfiles and dependency manifests; +- CLI/API/source symbols used by quick-start examples; +- CI/workflow commands used for verification; +- root LICENSE, NOTICE, THIRD_PARTY_NOTICES, file headers and package license metadata; +- git/upstream/fork provenance when ownership or relicensing is not obvious; +- vendored/copied/generated assets and submodules; +- live GitHub Releases/Pages state only when the README makes a release/publication claim. + +Do not use dependency licenses as a shortcut for source-license analysis. + +## Integration loop + +README work is complete only when the branch is integrated or a real external blocker remains. + +1. Search for overlapping README/documentation/license PRs before creating a new lane. +2. Update the most authoritative writable existing lane when coherent with its scope. +3. Make the smallest safe concrete improvement in the same run that confirms the defect. +4. Re-read current reviews and inline threads. +5. Inspect exact-head workflow/check results; predecessor evidence never transfers after a push. +6. Root-cause repository-owned failures and fix them rather than documenting around them. +7. When the root cause is central, move to the owning control-plane/library repository rather than adding a leaf workaround. +8. Merge through the normal protected path as soon as the unchanged exact head satisfies all applicable checks, review/thread requirements, mergeability, licensing/provenance gates, and current governance. +9. If one PR is waiting, continue another safe README/documentation lane; waiting is not completion. +10. After merge, continue to the next highest-leverage repository. + +Do not bypass substantive failing tests, unresolved security findings, meaningful review objections, conflicts, required governance, or genuine provenance blockers. + +## Quality bar + +A README is good enough when a new reader can answer, without reading source code first: + +- What problem does this product solve? +- What is the repository responsible for—and not responsible for? +- What can I safely run or integrate today? +- Where is the deeper technical authority? +- What evidence should I use to judge current quality/release state? +- How do I report a security problem or contribute? +- What license does the repository actually grant, and what important third-party/provenance limits remain? + +Consistency across ContextualWisdomLab comes from answering those questions with the same evidence discipline, not from making every README sound identical. From 7f4c5e3e0efb7bfe29f33b60d4264858effd2996 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 23:17:15 +0900 Subject: [PATCH 02/80] fix(scheduler): hold pre-review branch updates while current-head checks are in flight (#1937) A scheduler run that executes after a 2h+ queue wait finds its scanned PR behind main and merges main into the head before any review is dispatched (inspect_pr, pre-review path). That push cancels every queued check on the old head (22/28 on #1926, 21/30 on #1484) and requeues the PR at the back, so under a saturated queue no head ever finishes its checks: 76 of the 77 PRs merged since 2026-09-04 had 0/12 required contexts satisfied at merge. has_in_flight_check_runs() reuses latest_check_runs()/running_check_state(); the pre-review path now decides "wait" while any newest current-head check run is still queued or running. No age cap on purpose: a cap would restart the loop. The post-approval update path is unchanged (main is strict=true). CLAUDE.md described only the post-approval update; it now names both paths. Refs #1935 Co-authored-by: Claude Fable 5.1 --- CHANGELOG.md | 4 ++ CLAUDE.md | 7 +-- scripts/ci/pr_review_merge_scheduler_core.py | 18 ++++++++ tests/test_pr_review_merge_scheduler.py | 47 ++++++++++++++++++++ 4 files changed, 73 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b040db1b0..ca81dcea1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Scheduler holds pre-review branch updates while checks are in flight + +- `inspect_pr` now decides `wait` instead of `update_branch` when a behind, unreviewed head still has queued or running check runs (`has_in_flight_check_runs`, built on the existing `latest_check_runs`/`running_check_state`). Under a saturated runner queue each PR's own delayed `pull_request_target` scheduler run merged `main` into the head before review dispatch, cancelling every queued check on the old head (22/28 on #1926, 21/30 on #1484) and requeueing the PR at the back, so no head ever completed its checks: 76 of the 77 PRs merged into this repository since 2026-09-04 had 0/12 required contexts satisfied at merge time. The hold has no age cap on purpose -- a check that never finishes keeps the head in place instead of restarting that loop, and the update resumes once every newest check run is terminal. `CLAUDE.md` now describes both update paths. Tracked in #1935. + ### CodeQL scan dispatch matrix serialisation - Serialised the dispatched CodeQL matrix with `toJSON()` in `codeql-scan-dispatch.yml`. `codeql-pr.yml` sends `client_payload.matrix` as an array and the handler assigned it straight into `env:`, where a value must be a scalar, so GitHub rejected the step with "A sequence was not expected" and the dispatched scan never ran -- 0 successes against 136 failures since the handler was added in #1776. The validate step already consumes the value through `jq`, so JSON text is the shape it was written for and no consumer changes. Added a string contract test, because neither `yaml.safe_load` nor `actionlint` 1.7.12 flags this: it is an Actions template rule, so only GitHub's own validator rejects it and no local gate catches the class. diff --git a/CLAUDE.md b/CLAUDE.md index f1b2cbb482..30db1fc23b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,9 +48,10 @@ an actually-executed PoC via `scripts/ci/sandboxed_verify.py` or `scripts/ci/san split `Developer experience:` / `User experience:` sections). Deterministic code may repair only trusted `path:line` bindings on LLM probes that already carry an independent proof and source-line digest; it never invents observed -results. The scheduler updates a PR branch only -when the latest review is approved, no current-head check has failed, and GitHub reports the PR as -behind. The mechanical merge scheduler itself never synthesizes a fix: it gives `DIRTY`/`CONFLICTING` +results. The scheduler updates a PR branch in two cases: after approval, when no current-head check +has failed and GitHub reports the PR as behind; and before review dispatch, when the PR is behind and +no current-head check is still queued or running (an in-flight check is evidence the update would +discard; see #1935). The mechanical merge scheduler itself never synthesizes a fix: it gives `DIRTY`/`CONFLICTING` PRs repair guidance. A separate edit-capable autofix flow (`scripts/ci/pr_review_fix_scheduler.py` → `.github/workflows/pr-review-autofix.yml`) may, for an approved same-repository-head PR, merge the base into the head and resolve the conflict markers; the diff --git a/scripts/ci/pr_review_merge_scheduler_core.py b/scripts/ci/pr_review_merge_scheduler_core.py index e97b41074a..c4e9d28ebd 100644 --- a/scripts/ci/pr_review_merge_scheduler_core.py +++ b/scripts/ci/pr_review_merge_scheduler_core.py @@ -1904,6 +1904,11 @@ def opencode_in_progress(pr: dict[str, Any], *, stale_after_minutes: int | None return opencode_progress_state(pr, stale_after_minutes=stale_after) == "running" +def has_in_flight_check_runs(pr: dict[str, Any]) -> bool: + """Return whether any newest current-head check run is still queued or running.""" + return any(running_check_state(node) == "running" for node in latest_check_runs(pr)) + + _STRIX_SUCCESS_CONCLUSIONS = {"SUCCESS"} @@ -4781,6 +4786,19 @@ def request_branch_update(freshness_reason: str, *, suffix: str = "") -> Decisio f"current head has no OpenCode approval; branch is outdated before review dispatch, " f"but head repo {head_repo} is not writable by the scheduler credential", ) + if has_in_flight_check_runs(pr): + # Updating now would cancel every queued or running check on the + # current head and requeue the pull request behind them. Under a + # saturated runner queue the PR's own delayed scheduler run does + # this on every execution, so no head ever finishes its checks + # (#1935). Deliberately no age cap: a check that never finishes + # keeps the head where it is instead of restarting that loop. + return decide( + "wait", + "current head has no OpenCode approval; branch is outdated before review dispatch, " + "but current-head checks are still queued or running; holding the update so their " + "evidence is not discarded", + ) if merge_state == "BEHIND": freshness_reason = "current head has no OpenCode approval; branch is outdated before review dispatch" else: diff --git a/tests/test_pr_review_merge_scheduler.py b/tests/test_pr_review_merge_scheduler.py index 1e5848aac3..2cbda7f85b 100644 --- a/tests/test_pr_review_merge_scheduler.py +++ b/tests/test_pr_review_merge_scheduler.py @@ -10648,3 +10648,50 @@ def test_reconcile_releases_strix_lease_when_no_run_was_created(tmp_path): record = next(iter(load_state_file(gate.state_path).records.values())) assert record.status == "stale" + + +def test_inspect_pr_holds_pre_review_update_while_current_head_checks_run(): + """A behind, unreviewed head keeps its queued checks instead of being updated (#1935). + + Under a saturated queue the PR's own delayed scheduler run used to merge + ``main`` into the head before review dispatch, cancelling every queued + check on the old head and requeueing the PR behind them. The hold has no + age cap on purpose: a check that never finishes keeps the head in place + rather than restarting that loop, and the update resumes as soon as every + newest check run has a terminal status. + """ + + def behind_with(nodes): + return make_pr( + mergeStateStatus="BEHIND", + statusCheckRollup={"contexts": {"nodes": nodes}}, + ) + + held = inspect( + behind_with( + [ + {"__typename": "CheckRun", "name": "trivy-fs", "status": "QUEUED", "conclusion": None}, + {"__typename": "CheckRun", "name": "scan-pr-queue", "status": "IN_PROGRESS", "conclusion": None}, + {"__typename": "CheckRun", "name": "osv-scan", "status": "COMPLETED", "conclusion": "SUCCESS"}, + ] + ) + ) + assert held.action == "wait" + assert "branch is outdated before review dispatch" in held.reason + assert "checks are still queued or running" in held.reason + + resumed = inspect( + behind_with( + [ + {"__typename": "CheckRun", "name": "trivy-fs", "status": "COMPLETED", "conclusion": "SUCCESS"}, + {"__typename": "CheckRun", "name": "scan-pr-queue", "status": "COMPLETED", "conclusion": "SKIPPED"}, + ] + ) + ) + assert resumed.action == "update_branch" + assert resumed.reason.startswith( + "current head has no OpenCode approval; branch is outdated before review dispatch" + ) + assert "checks are still queued or running" not in resumed.reason + + assert sched.has_in_flight_check_runs(behind_with([])) is False From 7c32d20648a46d1a778c4c2411fe705dc8dc33e7 Mon Sep 17 00:00:00 2001 From: seonghobae Date: Sat, 5 Sep 2026 14:40:24 +0000 Subject: [PATCH 03/80] fix(strix): coalesce push scans per protected branch instead of one group per run id The workflow-level concurrency key fell back to github.run_id for every non-PR event, so each main push was its own group and no newer main head ever retired an older, superseded scan. Measured 2026-09-05T14:27Z in .github: nine push/main Strix runs outstanding at once (five running, one past two hours; four queued) against a 10-30 minute normal scan, each holding a slot under the shared 60-job ceiling. Scope push events as push- with the existing cancel-in-progress: true, so a newer head of the same protected branch supersedes the older scan exactly as a newer PR head does. A push scan covers the whole tree and publishes no strix commit status, so the newest head subsumes every older one; schedule and PR-less repository_dispatch keep a unique run id and the pr_number admission output is unchanged. Contract: queue-contract docstring records the measurement and a new assertion pins the push-{0} clause; the quick gate asserts it too; the 2026-09-04 doctoring record carries a dated amendment. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01BV96rXhqoR3tYZ9AeAVur4 --- .github/workflows/strix.yml | 14 +++++++++++++- ...rtup-failure-and-strix-concurrency-20260904.md | 14 ++++++++++++++ scripts/ci/test_strix_quick_gate.sh | 1 + tests/test_required_workflow_queue_contract.py | 15 +++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 58ed3dab8d..d8d8c2b35b 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -77,12 +77,24 @@ on: concurrency: # Workflow-level admission is required: job-level groups are never evaluated # while the whole run is queued behind the organization job ceiling. + # Push scans coalesce per protected branch: a newer head of the same branch + # supersedes the older scan exactly as a newer PR head does. A push scan + # covers the whole tree (STRIX_TARGET_PATH is './' outside PR scope) and + # publishes no 'strix' commit status, so the newest head subsumes every + # older one. With the run-id fallback every main push was its own group and + # nothing ever retired a superseded main scan: on 2026-09-05 nine push/main + # runs were outstanding at once in this repository (five running for up to + # two hours, four queued) against a 10-30 minute normal scan, each holding a + # slot under the shared 60-job ceiling. schedule and repository_dispatch + # without a PR number keep a unique run id. group: >- strix-security-scan-${{ github.event.pull_request.base.repo.full_name || github.event.client_payload.target_repository || github.repository }}-${{ github.event.pull_request.number || - github.event.client_payload.pr_number || github.run_id }} + github.event.client_payload.pr_number || + (github.event_name == 'push' && format('push-{0}', github.ref_name)) || + github.run_id }} cancel-in-progress: true # Scorecard Token-Permissions (alert #43): keep the workflow-level token diff --git a/docs/doctoring/startup-failure-and-strix-concurrency-20260904.md b/docs/doctoring/startup-failure-and-strix-concurrency-20260904.md index e710eb5d1f..a905213e04 100644 --- a/docs/doctoring/startup-failure-and-strix-concurrency-20260904.md +++ b/docs/doctoring/startup-failure-and-strix-concurrency-20260904.md @@ -43,6 +43,20 @@ another. Workflow-level concurrency was deliberately not used because GitHub applies it before any live-head admission job can run and does not guarantee concurrency ordering. +**Amendment (2026-09-05).** "nor one another" no longer holds for `push` +events on the same branch. Measured at 14:27Z in `.github`: nine `push`/`main` +Strix runs were outstanding at once (five running, jobs started 12:31-14:25Z, +one already past two hours; four queued), each holding one slot under the +shared 60-job ceiling, against a 10-30 minute normal scan. The run-id fallback +in the workflow-level group made every main push its own group, so no newer +main head ever retired an older scan. The workflow-level group now scopes +`push` events as `push-`: a newer head of the same protected branch +supersedes the older scan exactly as a newer PR head does. This loses nothing +the gate consumes: a push scan covers the whole tree (`STRIX_TARGET_PATH` is +`./` outside PR scope) and publishes no `strix` commit status. `schedule` and +PR-less `repository_dispatch` runs still receive a unique run id. The +`pr_number=${GITHUB_RUN_ID}` admission output is unchanged. + ## Verification - `python -m pytest -q tests/test_pr_review_merge_scheduler.py -k 'startup_failures or startup_failure'` diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index b9b1c43de3..6ea00c099f 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -203,6 +203,7 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "needs: [changed-scope, admit-current-head]" "strix provider queue waits for live-head admission" assert_file_contains "$workflow_file" 'strix-security-scan-${{' "strix workflow coalesces by repository and PR before job admission" assert_file_not_contains "$workflow_file" 'strix-security-scan-${{ needs.admit-current-head.outputs.target_repository }}-${{' "strix concurrency is not delayed until job admission" + assert_file_contains "$workflow_file" "format('push-{0}', github.ref_name)" "strix push scans coalesce per protected branch instead of one group per run id" assert_file_contains "$workflow_file" "cancel-superseded-pr-runs:" "strix workflow runs superseded-head cleanup outside the provider scan queue" assert_file_not_contains "$workflow_file" "format('closed-pr-{0}-{1}'" "strix cleanup does not need a second concurrency queue" assert_file_contains "$workflow_file" 'echo "pr_number=${GITHUB_RUN_ID}"' "strix workflow preserves independent push and schedule evidence" diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 803d43ab59..0c05ca6f6f 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -382,6 +382,17 @@ def test_strix_serializes_provider_evidence_per_repository_and_pr() -> None: superseded runs before runner admission, including runs still blocked by the organization-wide job ceiling. Native and dispatched evidence share one group; non-PR events use a unique run id. + + 2026-09-05: push events are scoped per protected branch (``push-``) + instead of a unique run id. Measured that morning in this repository: + nine ``push``/``main`` Strix runs were outstanding at once (five running + for up to two hours, four queued) against a 10-30 minute normal scan, + because the run-id fallback made every main push its own group and + nothing ever retired a superseded main scan. A push scan covers the whole + tree (``STRIX_TARGET_PATH`` is ``./`` outside PR scope) and publishes no + ``strix`` commit status, so the newest branch head subsumes every older + one. ``schedule`` and ``repository_dispatch`` without a PR number keep a + unique run id. """ workflow = workflow_text("strix.yml") concurrency_contract = workflow.split("concurrency:", 1)[1].split( @@ -398,6 +409,10 @@ def test_strix_serializes_provider_evidence_per_repository_and_pr() -> None: assert "github.event.pull_request.number" in concurrency_contract assert "github.event.client_payload.pr_number" in concurrency_contract assert "github.run_id" in concurrency_contract + assert ( + "(github.event_name == 'push' && format('push-{0}', github.ref_name)) ||" + in concurrency_contract + ) assert "github.event.pull_request.head.sha" not in concurrency_contract assert "github.event.client_payload.pr_head_sha" not in concurrency_contract assert "cancel-in-progress: true" in concurrency_contract From 1ed074c37e6e13fd3092fed21a677d04f5905bc0 Mon Sep 17 00:00:00 2001 From: seonghobae Date: Sat, 5 Sep 2026 16:19:28 +0000 Subject: [PATCH 04/80] docs(strix): state the merge-burst tradeoff of per-branch push coalescing Requested in review on #1938: during a merge burst each new main head cancels the previous scan, so the latest main is scanned once merging pauses for one scan duration; the weekly schedule scan is the floor. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01BV96rXhqoR3tYZ9AeAVur4 --- .../startup-failure-and-strix-concurrency-20260904.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/doctoring/startup-failure-and-strix-concurrency-20260904.md b/docs/doctoring/startup-failure-and-strix-concurrency-20260904.md index a905213e04..7c380ce7b5 100644 --- a/docs/doctoring/startup-failure-and-strix-concurrency-20260904.md +++ b/docs/doctoring/startup-failure-and-strix-concurrency-20260904.md @@ -57,6 +57,14 @@ the gate consumes: a push scan covers the whole tree (`STRIX_TARGET_PATH` is PR-less `repository_dispatch` runs still receive a unique run id. The `pr_number=${GITHUB_RUN_ID}` admission output is unchanged. +Tradeoff, stated so a later reader of the security dashboard is not +surprised: with `main` moving roughly every 30 minutes against a 10-30 minute +scan, "main is scanned after every merge" becomes "the latest `main` is +scanned once merging pauses for at least one scan duration". During a merge +burst each new head cancels the previous scan; the burst's final head is +scanned, and the weekly full-tree `schedule` scan (unique run id, never +cancelled) is the floor under a sustained burst. + ## Verification - `python -m pytest -q tests/test_pr_review_merge_scheduler.py -k 'startup_failures or startup_failure'` From 0dc71046a3117b706c39c855352388f0f788a882 Mon Sep 17 00:00:00 2001 From: seonghobae Date: Sat, 5 Sep 2026 16:20:38 +0000 Subject: [PATCH 05/80] docs(strix): record the coverage half of the push-coalescing trade in the workflow comment Peer measurement on #1938: 50 main pushes in 24 h, median gap 8.4 min, mean 26.5 min, so only the final head of each merge burst completes a scan. Stated next to the group key so a later coverage measurement does not read the gaps as a defect. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01BV96rXhqoR3tYZ9AeAVur4 --- .github/workflows/strix.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index d8d8c2b35b..b301e2318a 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -87,6 +87,12 @@ concurrency: # two hours, four queued) against a 10-30 minute normal scan, each holding a # slot under the shared 60-job ceiling. schedule and repository_dispatch # without a PR number keep a unique run id. + # Coverage trade, measured the same day: main moved 50 times in 24 hours + # (median gap 8.4 min, mean 26.5 min, bursty), so with cancel-in-progress + # only the final head of each merge burst completes a scan -- one completed + # main scan per quiet window, not one per push. That is the intended + # exchange: a cancelled push scan loses no evidence a later whole-tree scan + # does not re-cover, and the weekly schedule scan is the floor. group: >- strix-security-scan-${{ github.event.pull_request.base.repo.full_name || From f2f91b806122ed233e3a0e2a325246077c2e15e4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 02:25:33 +0900 Subject: [PATCH 06/80] fix(review): round-robin catalog fill across accounts within a tier (#1939) build_zdr_prioritized_catalog sorted eligible routes by (cost, ZDR, provider, model) and filled the bounded catalog in that order, taking up to account_cap per account. With the sidecar's ORCHESTRATOR_CATALOG_ ACCOUNT_CAP=8 and ORCHESTRATOR_CATALOG_LIMIT=12 the fill took 8 nvidia_nim + 4 nvidia_nim_sub and stopped before the alphabetically last account: noema-review run 33969842312 admitted 62 free routes across three accounts (free_account_diversity 3) and served 12 NVIDIA routes, of which runtime preflight kept 2, so a stalled NVIDIA endpoint had no other account to fail over to (contextual-orchestrator#1045). Keep the sort; group the sorted rows by (cost, ZDR) tier and fill each tier round-robin across provider accounts until limit, honouring account_cap. Tier order, cap, limit and discovery-order independence are unchanged; the same input now yields 4 + 4 + 4. The launcher's evidence_only filter (#1476) is not the cause on the current pin (2e414d15 includes contextual-orchestrator#949), so that PR stays a complementary hardening. Tests: three new cases (interleave within tier; ZDR tier still first; exhausted accounts hand turns over) were RED against the old loop (3 failed) and are GREEN now. Gate on this tree: 2896 passed, coverage 100%, interrogate 100%. Co-authored-by: Claude Fable 5.1 --- CHANGELOG.md | 4 + .../contextual_orchestrator_review_policy.py | 53 ++++++++--- ...t_contextual_orchestrator_review_policy.py | 89 +++++++++++++++++++ 3 files changed, 132 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca81dcea1d..46d599a320 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Review sidecar catalog interleaves credential accounts + +- `build_zdr_prioritized_catalog` now fills each free/ZDR tier round-robin across independently credentialed accounts instead of in provider-name order. The sidecar exports `ORCHESTRATOR_CATALOG_ACCOUNT_CAP=8` with `ORCHESTRATOR_CATALOG_LIMIT=12`, and the sorted fill took 8 `nvidia_nim` routes and 4 `nvidia_nim_sub` routes before any `openrouter` route was reached, so a review that admitted 62 free routes across three accounts served a NVIDIA-only catalog (`noema-review` run 33969842312: `free_pool_admitted_routes` 62, `free_selected_count` 12, runtime preflight `ready_count` 2 of 12) and the failover loop had no other account to leave a stalled NVIDIA endpoint for -- the `noema-review` 502 class tracked in contextual-orchestrator#1045. Tier order (free before priced, ZDR before non-ZDR), the account cap, the limit, and the discovery-order independence contract are unchanged; the same input now yields 4 + 4 + 4. Contrasts with #1476, which hardens `_routable_discovered_models` against a pin that regresses the OpenRouter `evidence_only` flag: on the current pin (`2e414d15`, includes contextual-orchestrator#949) OpenRouter rows already reach the catalog builder, and the selection was what dropped them. + ### Scheduler holds pre-review branch updates while checks are in flight - `inspect_pr` now decides `wait` instead of `update_branch` when a behind, unreviewed head still has queued or running check runs (`has_in_flight_check_runs`, built on the existing `latest_check_runs`/`running_check_state`). Under a saturated runner queue each PR's own delayed `pull_request_target` scheduler run merged `main` into the head before review dispatch, cancelling every queued check on the old head (22/28 on #1926, 21/30 on #1484) and requeueing the PR at the back, so no head ever completed its checks: 76 of the 77 PRs merged into this repository since 2026-09-04 had 0/12 required contexts satisfied at merge time. The hold has no age cap on purpose -- a check that never finishes keeps the head in place instead of restarting that loop, and the update resumes once every newest check run is terminal. `CLAUDE.md` now describes both update paths. Tracked in #1935. diff --git a/scripts/ci/contextual_orchestrator_review_policy.py b/scripts/ci/contextual_orchestrator_review_policy.py index 910b8da3a9..e609e67ff3 100644 --- a/scripts/ci/contextual_orchestrator_review_policy.py +++ b/scripts/ci/contextual_orchestrator_review_policy.py @@ -13,6 +13,7 @@ from __future__ import annotations import argparse +import itertools import json import math import re @@ -273,6 +274,21 @@ def _free_pool_source_admitted(row: Mapping[str, Any]) -> bool: ) +def _route_tier(row: Mapping[str, Any], zdr_endpoints: frozenset[str]) -> tuple[int, int]: + """Return the ``(cost rank, ZDR rank)`` tier a route is selected within. + + Free routes rank before priced ones and ZDR-attested routes before + unattested ones; the tier is what the catalog fill must never reorder, + while accounts inside one tier may be interleaved freely. + """ + attested = is_zdr_model( + str(row["provider"]), + model=str(row["model"]), + zdr_endpoints=zdr_endpoints, + ) + return (_COST_EVIDENCE_RANK[_cost_evidence(row)], 0 if attested else 1) + + def build_zdr_prioritized_catalog( rows: Iterable[Mapping[str, Any]], *, @@ -317,27 +333,36 @@ def build_zdr_prioritized_catalog( ] eligible_rows.sort( key=lambda row: ( - _COST_EVIDENCE_RANK[_cost_evidence(row)], - 0 - if is_zdr_model( - str(row["provider"]), - model=str(row["model"]), - zdr_endpoints=zdr_endpoints, - ) - else 1, + *_route_tier(row, zdr_endpoints), str(row["provider"]), str(row["model"]), ) ) + # Fill each (cost, ZDR) tier round-robin across independently credentialed + # accounts. A plain sorted fill let the alphabetically first account take + # its whole cap before the next account saw a slot: on 2026-09-05 the review + # sidecar admitted 62 free routes across three accounts and served + # 8 nvidia_nim + 4 nvidia_nim_sub + 0 openrouter (limit 12, cap 8), so a + # stalled NVIDIA endpoint had no other account to fail over to + # (ContextualWisdomLab/.github#1476, contextual-orchestrator#1045). per_account: Counter[str] = Counter() picked: list[Mapping[str, Any]] = [] - for row in eligible_rows: - account = provider_account(str(row["provider"])) - if per_account[account] >= account_cap: - continue - per_account[account] += 1 - picked.append(row) + for _tier, tier_rows in itertools.groupby( + eligible_rows, key=lambda row: _route_tier(row, zdr_endpoints) + ): + queues: dict[str, list[Mapping[str, Any]]] = {} + for row in tier_rows: + queues.setdefault(provider_account(str(row["provider"])), []).append(row) + while queues and len(picked) < limit: + for account in list(queues): + if per_account[account] >= account_cap or not queues[account]: + del queues[account] + continue + picked.append(queues[account].pop(0)) + per_account[account] += 1 + if len(picked) >= limit: + break if len(picked) >= limit: break diff --git a/tests/test_contextual_orchestrator_review_policy.py b/tests/test_contextual_orchestrator_review_policy.py index 4cda949897..41c1ea40b2 100644 --- a/tests/test_contextual_orchestrator_review_policy.py +++ b/tests/test_contextual_orchestrator_review_policy.py @@ -563,3 +563,92 @@ def test_private_catalog_fails_closed_without_attested_zdr_route() -> None: account_cap=4, require_zdr=True, ) + + +def _free_rows(provider: str, count: int, prefix: str) -> list[dict[str, object]]: + """Return ``count`` free discovery rows for one credential account.""" + return [ + { + "provider": provider, + "model": f"{prefix}{i}", + "agent_id": f"{prefix}_{i}", + "is_free": True, + **FREE_PRICE, + } + for i in range(count) + ] + + +def test_build_catalog_interleaves_accounts_within_a_tier() -> None: + """A bounded catalog spreads across admitted accounts instead of filling alphabetically. + + Measured on 2026-09-05 (``noema-review`` run 33969842312): 62 admitted free + routes across three accounts, limit 12, account cap 8, served as + 8 ``nvidia_nim`` + 4 ``nvidia_nim_sub`` + 0 ``openrouter`` because the + sorted fill reached the limit before the alphabetically last account got a + slot -- so a stalled NVIDIA endpoint had no other account to fail over to. + """ + report = { + "models": _free_rows("nvidia_nim", 8, "a") + + _free_rows("nvidia_nim_sub", 8, "b") + + _free_rows("openrouter", 8, "o") + } + result = policy.build_zdr_prioritized_catalog( + policy.parse_discovery_report(report), limit=12, account_cap=8 + ) + providers = [agent["provider_name"] for agent in result["agents"]] + assert providers[:3] == ["nvidia_nim", "nvidia_nim_sub", "openrouter"] + assert providers.count("nvidia_nim") == 4 + assert providers.count("nvidia_nim_sub") == 4 + assert providers.count("openrouter") == 4 + + +def test_build_catalog_interleaving_keeps_zdr_tier_first() -> None: + """Account interleaving never lifts a non-ZDR route above an attested one.""" + report = { + "models": _free_rows("nvidia_nim", 3, "a") + + [ + { + "provider": "openrouter", + "model": "deepseek/deepseek-r1:free", + "agent_id": "or_zdr", + "is_free": True, + **FREE_PRICE, + } + ] + + _free_rows("openrouter", 3, "o") + } + result = policy.build_zdr_prioritized_catalog( + policy.parse_discovery_report(report), + limit=4, + account_cap=8, + zdr_endpoints=ZDR_FEED, + ) + assert result["agents"][0]["model"] == "deepseek/deepseek-r1:free" + assert [agent["provider_name"] for agent in result["agents"]][1:] == [ + "nvidia_nim", + "openrouter", + "nvidia_nim", + ] + + +def test_build_catalog_interleaving_skips_exhausted_accounts() -> None: + """An account with fewer routes than its share hands its turns to the others.""" + report = { + "models": _free_rows("nvidia_nim", 5, "a") + + _free_rows("nvidia_nim_sub", 1, "b") + + _free_rows("openrouter", 2, "o") + } + result = policy.build_zdr_prioritized_catalog( + policy.parse_discovery_report(report), limit=12, account_cap=8 + ) + assert [agent["provider_name"] for agent in result["agents"]] == [ + "nvidia_nim", + "nvidia_nim_sub", + "openrouter", + "nvidia_nim", + "openrouter", + "nvidia_nim", + "nvidia_nim", + "nvidia_nim", + ] From d9eb9f79b6ce66c1225c26be385ae814d87d9aca Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 06:59:11 +0900 Subject: [PATCH 07/80] fix(sidecar): record the orchestrator's per-attempt trace in the review sidecar's stderr (#1943) The review sidecar never configured its orchestrator process's logging, so it ran at Python's default WARNING while contextual_orchestrator logs every provider attempt, classified failure, backoff and circuit event at DEBUG. A 3122 s noema-review 502 on 2026-09-05 (.github#1940) could only be attributed by reading source: six preflight-ready routes, two retry layers (TaskOrchestrator.tool_retry_attempts=1 over ModelClient max_retries=2 at a 90 s per-recv timeout), about 548 s per hop. The launcher now calls the vendored debug_logging.configure_logging before serving, DEBUG by default and overridable through ORCHESTRATOR_SIDECAR_LOG_LEVEL, and gives every root handler a timestamped format so per-hop durations can be read off the trace. At the vendored pin no DEBUG site logs a prompt, payload or response body; the only free-text field (provider_attempt_failed's error_message) is kept out of CI evidence by the sidecar's allow-list sanitizer, which a companion change extends to admit these templates and upload the file. Tests: level default, override, level-and-format application, invalid level -> SystemExit, and a source-order contract that logging is configured before credential work; all five fail against main's launcher. Gate on this tree: 2901 passed, coverage 100%, interrogate 100% (the final commit differs from the gated tree by a docstring only; the touched test module and interrogate were re-run on the final text). Co-authored-by: Claude Fable 5.1 --- CHANGELOG.md | 4 ++ ...contextual_orchestrator_review_launcher.py | 61 +++++++++++++++++- ...l_orchestrator_review_runtime_preflight.py | 63 +++++++++++++++++++ 3 files changed, 127 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46d599a320..efad99e30b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Review sidecar records the orchestrator's per-attempt trace + +- `contextual_orchestrator_review_launcher.py` now configures the orchestrator process's logging before serving (`_configure_sidecar_logging`, calling the vendored `contextual_orchestrator.debug_logging.configure_logging`), defaulting to `DEBUG` with a timestamped format and overridable through `ORCHESTRATOR_SIDECAR_LOG_LEVEL`. The orchestrator logs every provider attempt, its classified failure, backoff, and circuit event at `DEBUG` and only `provider_exhausted`/`circuit_opened` at the default `WARNING`, so a failed review left no way to see which routes were tried or how long each took: a 3122 s `noema-review` 502 on 2026-09-05 could only be attributed to "six ready routes, two retry layers, about 548 s per hop" by reading source, not the log. None of the `DEBUG` sites at the vendored pin carries prompt or response content, and the sidecar already pipes this stderr through the redacting sanitizer before it is written to `strix_runs/contextual-orchestrator-sidecar.stderr.log`; a companion change uploads that file as a failure artifact. + ### Review sidecar catalog interleaves credential accounts - `build_zdr_prioritized_catalog` now fills each free/ZDR tier round-robin across independently credentialed accounts instead of in provider-name order. The sidecar exports `ORCHESTRATOR_CATALOG_ACCOUNT_CAP=8` with `ORCHESTRATOR_CATALOG_LIMIT=12`, and the sorted fill took 8 `nvidia_nim` routes and 4 `nvidia_nim_sub` routes before any `openrouter` route was reached, so a review that admitted 62 free routes across three accounts served a NVIDIA-only catalog (`noema-review` run 33969842312: `free_pool_admitted_routes` 62, `free_selected_count` 12, runtime preflight `ready_count` 2 of 12) and the failover loop had no other account to leave a stalled NVIDIA endpoint for -- the `noema-review` 502 class tracked in contextual-orchestrator#1045. Tier order (free before priced, ZDR before non-ZDR), the account cap, the limit, and the discovery-order independence contract are unchanged; the same input now yields 4 + 4 + 4. Contrasts with #1476, which hardens `_routable_discovered_models` against a pin that regresses the OpenRouter `evidence_only` flag: on the current pin (`2e414d15`, includes contextual-orchestrator#949) OpenRouter rows already reach the catalog builder, and the selection was what dropped them. diff --git a/scripts/ci/contextual_orchestrator_review_launcher.py b/scripts/ci/contextual_orchestrator_review_launcher.py index 2e56809639..502843c994 100644 --- a/scripts/ci/contextual_orchestrator_review_launcher.py +++ b/scripts/ci/contextual_orchestrator_review_launcher.py @@ -23,11 +23,12 @@ import argparse import json +import logging import os import re import sys from pathlib import Path -from typing import Any +from typing import Any, Callable from scripts.ci.contextual_orchestrator_review_policy import FREE_POOL_CREDENTIAL_NAMES @@ -673,6 +674,62 @@ def _catalog_account_cap(default: int) -> int: return int(os.environ.get("ORCHESTRATOR_CATALOG_ACCOUNT_CAP", str(default))) +DEFAULT_SIDECAR_LOG_LEVEL = "DEBUG" +SIDECAR_LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s %(message)s" + + +def _sidecar_log_level() -> str: + """Return the log level the review sidecar configures for its orchestrator process. + + Defaults to ``DEBUG`` because that is where ``contextual_orchestrator`` + records the per-request trace a failed review needs afterwards: every + provider attempt (``provider_attempt``), its classified failure + (``provider_attempt_failed`` with error type and transient flag), backoff, + and circuit events are ``_LOGGER.debug`` calls, while the default + ``WARNING`` level keeps only ``provider_exhausted``/``circuit_opened``. At + the vendored pin none of those DEBUG sites logs a prompt, payload, or + response body; the one free-text field is ``provider_attempt_failed``'s + ``error_message`` (the exception text, which can quote an upstream error + body), and the sidecar pipes this process's stderr through the allow-list + sanitizer before it reaches disk, so only lines the sanitizer recognises + -- and only their structured fields -- become CI evidence. On + 2026-09-05 a 3122 s ``noema-review`` failure could not be attributed to + "six ready routes, two retry layers, 548 s per hop" from the job log alone + because this trace was never emitted. Override with + ``ORCHESTRATOR_SIDECAR_LOG_LEVEL``. + """ + return os.environ.get("ORCHESTRATOR_SIDECAR_LOG_LEVEL", DEFAULT_SIDECAR_LOG_LEVEL) + + +def _configure_sidecar_logging(configure_logging: Callable[[str], None]) -> str: + """Configure the orchestrator process's logging for CI evidence. + + ``configure_logging`` is ``contextual_orchestrator.debug_logging.configure_logging`` + (injected so this module stays importable without the vendored package): + it installs the root level with ``basicConfig(force=True)``. Its default + formatter carries no timestamp, and a per-attempt trace without + timestamps cannot yield per-hop durations, so every root handler is then + given :data:`SIDECAR_LOG_FORMAT`. + + Returns: + The level name that was applied. + + Raises: + SystemExit: If ``ORCHESTRATOR_SIDECAR_LOG_LEVEL`` is not a level name + the orchestrator accepts; a misspelt level must not silently leave + the process at ``WARNING``. + """ + level = _sidecar_log_level() + try: + configure_logging(level) + except ValueError as exc: + raise SystemExit(f"ORCHESTRATOR_SIDECAR_LOG_LEVEL is invalid: {exc}") from None + formatter = logging.Formatter(SIDECAR_LOG_FORMAT) + for handler in logging.getLogger().handlers: + handler.setFormatter(formatter) + return level + + def _with_discovery_counts( report: dict[str, object], rows: list[dict[str, Any]], @@ -802,7 +859,9 @@ def main(argv: list[str] | None = None) -> int: parse_discovery_report, provider_account, ) + from contextual_orchestrator.debug_logging import configure_logging + _configure_sidecar_logging(configure_logging) registered = register_review_credentials(os.environ) auth_token = args.auth_token or get_credential(REVIEW_AUTH_CREDENTIAL_NAME) if not auth_token: diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index 559c2d1e99..d38cd19c43 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -1768,3 +1768,66 @@ def test_sidecar_stream_sanitizer_omits_no_summary_for_fully_safe_input( assert main() == 0 assert output.getvalue() == "client_disconnected\n" + + +def test_sidecar_log_level_defaults_to_debug(monkeypatch: pytest.MonkeyPatch) -> None: + """The sidecar asks for DEBUG so provider attempts and circuit events are recorded.""" + monkeypatch.delenv("ORCHESTRATOR_SIDECAR_LOG_LEVEL", raising=False) + namespace = _load_launcher() + assert namespace["_sidecar_log_level"]() == "DEBUG" + assert namespace["DEFAULT_SIDECAR_LOG_LEVEL"] == "DEBUG" + + +def test_sidecar_log_level_honors_an_explicit_override(monkeypatch: pytest.MonkeyPatch) -> None: + """An operator-set ``ORCHESTRATOR_SIDECAR_LOG_LEVEL`` is passed through untouched.""" + monkeypatch.setenv("ORCHESTRATOR_SIDECAR_LOG_LEVEL", "INFO") + namespace = _load_launcher() + assert namespace["_sidecar_log_level"]() == "INFO" + + +def test_configure_sidecar_logging_applies_level_and_timestamped_format( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The injected configurator receives the level and every root handler gets timestamps.""" + import logging + + monkeypatch.delenv("ORCHESTRATOR_SIDECAR_LOG_LEVEL", raising=False) + namespace = _load_launcher() + received: list[str] = [] + + def fake_configure_logging(level_name: str) -> None: + received.append(level_name) + logging.basicConfig(level=getattr(logging, level_name), force=True) + + try: + applied = namespace["_configure_sidecar_logging"](fake_configure_logging) + assert applied == "DEBUG" + assert received == ["DEBUG"] + handlers = logging.getLogger().handlers + assert handlers, "basicConfig(force=True) must have installed a root handler" + for handler in handlers: + assert handler.formatter is not None + assert "%(asctime)s" in handler.formatter._fmt # noqa: SLF001 - formatter has no public getter + finally: + logging.basicConfig(level=logging.WARNING, force=True) + + +def test_configure_sidecar_logging_rejects_an_invalid_level(monkeypatch: pytest.MonkeyPatch) -> None: + """A misspelt level fails the launch instead of silently staying at WARNING.""" + monkeypatch.setenv("ORCHESTRATOR_SIDECAR_LOG_LEVEL", "LOUD") + namespace = _load_launcher() + + def strict_configure_logging(level_name: str) -> None: + raise ValueError(f"unknown log level {level_name!r}") + + with pytest.raises(SystemExit, match="ORCHESTRATOR_SIDECAR_LOG_LEVEL is invalid: unknown log level 'LOUD'"): + namespace["_configure_sidecar_logging"](strict_configure_logging) + + +def test_main_configures_sidecar_logging_before_touching_credentials() -> None: + """``main()`` wires the orchestrator's own ``configure_logging`` in before any credential work.""" + source = _LAUNCHER.read_text(encoding="utf-8") + configure_at = source.index("_configure_sidecar_logging(configure_logging)") + credentials_at = source.index("registered = register_review_credentials(os.environ)") + assert configure_at < credentials_at + assert "from contextual_orchestrator.debug_logging import configure_logging" in source From 972b74be2b44d354ef5ad06f051cf7ee7d7225ce Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 07:08:59 +0900 Subject: [PATCH 08/80] fix(sidecar): let the stream sanitizer pass orchestrator route and circuit events (#1945) * fix(sidecar): let the stream sanitizer pass orchestrator route and circuit events Every provider_*/circuit_* line from the orchestrator was folded into omitted_unstructured_lines, so even the provider_exhausted WARNING that fires today never reached an artifact. Admit those templates field by field against bounded charsets, cut provider_attempt_failed before its free-text error_message, and accept both the default and the sidecar formatter log prefixes (keeping the timestamp for durations). Companion to #1943 and #1944. Refs #1935, #1939 Co-Authored-By: Claude Fable 5.1 * fix(sidecar): accept float circuit counters and pin the real formatter output The orchestrator's circuit counters are floats (failures 0.0 += 1.0, circuit_reset_seconds 30.0), so the lines that reach stderr say failures=2.0 / reset_seconds=30.0; the integer-only pattern rejected both circuit_failure and circuit_opened. Found by rendering the templates through the sidecar's logging.Formatter, which the new test now does for all ten. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- CHANGELOG.md | 3 + ..._contextual_orchestrator_sidecar_stream.py | 58 +++++++++ ...l_orchestrator_review_runtime_preflight.py | 112 ++++++++++++++++++ 3 files changed, 173 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index efad99e30b..15d9e6e188 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +### Sidecar sanitizer admits orchestrator route and circuit events + +- `scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py` now passes the orchestrator's own `provider_attempt`, `provider_attempt_failed` (cut before the free-text `error_message=`), `provider_backoff`, `provider_exhausted`, `provider_rejected_permanent`, `provider_no_retry_budget` and `circuit_failure|opened|reset|cleared` lines (whose `failures`/`reset_seconds` are floats at runtime, `2.0`/`30.0`), matched field by field against bounded identifier and number charsets, with either Python's default `LEVEL:name:` prefix or the sidecar formatter's `asctime LEVEL name` prefix (the timestamp is kept so per-route durations can be read as differences). Until now every one of these lines was folded into `omitted_unstructured_lines`, so the `provider_exhausted` WARNING that already fires today after a route's retry budget is spent never reached an artifact, and a 3122 s walk across six ready routes (run `33981136873`) had no per-route trace. Companion to #1943 (sidecar DEBUG logging) and #1944 (Noema uploads the file on failure). Refs #1935, #1939. ### Review sidecar records the orchestrator's per-attempt trace - `contextual_orchestrator_review_launcher.py` now configures the orchestrator process's logging before serving (`_configure_sidecar_logging`, calling the vendored `contextual_orchestrator.debug_logging.configure_logging`), defaulting to `DEBUG` with a timestamped format and overridable through `ORCHESTRATOR_SIDECAR_LOG_LEVEL`. The orchestrator logs every provider attempt, its classified failure, backoff, and circuit event at `DEBUG` and only `provider_exhausted`/`circuit_opened` at the default `WARNING`, so a failed review left no way to see which routes were tried or how long each took: a 3122 s `noema-review` 502 on 2026-09-05 could only be attributed to "six ready routes, two retry layers, about 548 s per hop" by reading source, not the log. None of the `DEBUG` sites at the vendored pin carries prompt or response content, and the sidecar already pipes this stderr through the redacting sanitizer before it is written to `strix_runs/contextual-orchestrator-sidecar.stderr.log`; a companion change uploads that file as a failure artifact. diff --git a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py index 18bc11b667..51b9a5df27 100644 --- a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py +++ b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py @@ -20,6 +20,38 @@ r"error_type=(?P[A-Za-z_][A-Za-z0-9_]{0,63})" r"(?: http_status=(?P[1-5][0-9]{2}))?" ) +_LOG_PREFIX = re.compile( + r"^(?:(?P\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}) )?" + r"(?:DEBUG|INFO|WARNING|ERROR)[: ][A-Za-z0-9_.]+[: ]" +) +_AGENT_ID = r"[a-z][a-z0-9_]*" +_MODEL_ID = r"[A-Za-z0-9_./:-]+" +_ERROR_TYPE = r"[A-Za-z_][A-Za-z0-9_.]*" +_NUMBER = r"\d+(?:\.\d+)?" +# contextual_orchestrator/orchestrator.py templates at the vendored pin. Every +# field is a bounded identifier or number; ``error_message`` is free text and is +# deliberately excluded from the match so it can never be re-emitted. +# ``failures`` and ``reset_seconds`` are floats at runtime (``0.0 += 1.0``, ``30.0``), +# so they take the number charset; ``threshold`` is an int. +_ORCHESTRATOR_EVENTS = tuple( + re.compile(pattern) + for pattern in ( + rf"^provider_attempt agent_id={_AGENT_ID} model={_MODEL_ID} attempt=\d+/\d+$", + rf"^provider_attempt_failed agent_id={_AGENT_ID} model={_MODEL_ID} attempt=\d+ " + rf"error_type={_ERROR_TYPE} transient=(?:True|False)(?= error_message=)", + rf"^provider_backoff agent_id={_AGENT_ID} attempt=\d+ delay_seconds={_NUMBER}$", + rf"^provider_exhausted agent_id={_AGENT_ID} model={_MODEL_ID} attempts=\d+ " + rf"final_error_type={_ERROR_TYPE}$", + rf"^provider_rejected_permanent agent_id={_AGENT_ID} model={_MODEL_ID} attempts=\d+ " + rf"final_error_type={_ERROR_TYPE}$", + rf"^provider_no_retry_budget agent_id={_AGENT_ID} model={_MODEL_ID} attempts=\d+ " + rf"final_error_type={_ERROR_TYPE} transient=(?:True|False)$", + rf"^circuit_failure agent_id={_AGENT_ID} failures={_NUMBER} threshold=\d+$", + rf"^circuit_opened agent_id={_AGENT_ID} failures={_NUMBER} threshold=\d+ reset_seconds={_NUMBER}$", + rf"^circuit_reset agent_id={_AGENT_ID}$", + rf"^circuit_cleared agent_id={_AGENT_ID}$", + ) +) _PREFIX_SUMMARIES = ( ("review sidecar preflight failed:", "review sidecar preflight failed"), ("review sidecar discovery failed:", "review sidecar discovery failed"), @@ -43,6 +75,29 @@ ) +def _sanitize_orchestrator_event(stripped: str) -> str | None: + """Return an orchestrator route or circuit event reduced to its bounded fields. + + Accepts the bare message, Python's default ``LEVEL:name:message`` prefix, and + the sidecar formatter's ``asctime LEVEL name message`` prefix; the timestamp + is kept (digits and punctuation only) so per-route durations can be read as + differences. ``provider_attempt_failed`` is cut before ``error_message=``, + which carries upstream text. + """ + prefix = _LOG_PREFIX.match(stripped) + message = stripped[prefix.end():] if prefix is not None else stripped + for pattern in _ORCHESTRATOR_EVENTS: + match = pattern.match(message) + if match is None: + continue + summary = match.group(0) + if message.startswith("provider_attempt_failed "): + summary += " error_message=" + asctime = prefix.group("asctime") if prefix is not None else None + return f"{asctime} {summary}" if asctime else summary + return None + + def sanitize_line(line: str) -> str | None: """Return one allowlisted diagnostic summary or ``None`` for raw content.""" stripped = line.strip() @@ -68,6 +123,9 @@ def sanitize_line(line: str) -> str | None: if http_status is not None: summary += f" http_status={http_status}" return summary + orchestrator_event = _sanitize_orchestrator_event(stripped) + if orchestrator_event is not None: + return orchestrator_event if stripped in ("client_disconnected", "discovery_diagnostics_complete"): return stripped for prefix, summary in _PREFIX_SUMMARIES: diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index d38cd19c43..d0ace81e8b 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -1719,6 +1719,118 @@ def test_sidecar_stream_sanitizer_allowlists_only_bounded_diagnostics() -> None: assert sanitize_line("provider response sk-secret") is None +def test_sidecar_stream_sanitizer_admits_orchestrator_route_events() -> None: + """Per-route attempt, retry-budget, and circuit events survive with bounded fields only. + + Before this, every orchestrator ``provider_*``/``circuit_*`` line was folded + into ``omitted_unstructured_lines``, so a 3122 s walk across six routes left + no per-route trace in the artifact (#1935 / #1939). Both log prefixes are + accepted so runs before and after the sidecar formatter read the same way. + """ + namespace = _load_sanitizer() + sanitize_line = namespace["sanitize_line"] + secret = "sk-secret-must-not-enter-artifact" + + assert sanitize_line( + "provider_attempt agent_id=nvidia_nim_deepseek model=deepseek-ai/deepseek-v4-flash-0731 attempt=1/3" + ) == "provider_attempt agent_id=nvidia_nim_deepseek model=deepseek-ai/deepseek-v4-flash-0731 attempt=1/3" + assert sanitize_line( + "WARNING:contextual_orchestrator.orchestrator:provider_exhausted agent_id=nvidia_nim_x " + "model=deepseek-ai/deepseek-v4-flash-0731 attempts=3 final_error_type=TimeoutError" + ) == ( + "provider_exhausted agent_id=nvidia_nim_x model=deepseek-ai/deepseek-v4-flash-0731 " + "attempts=3 final_error_type=TimeoutError" + ) + failed = sanitize_line( + "2026-09-05 21:40:00,123 DEBUG contextual_orchestrator.orchestrator provider_attempt_failed " + f"agent_id=openrouter_gemma model=google/gemma-3-12b-it:free attempt=2 error_type=HTTPError " + f"transient=True error_message=upstream said {secret}" + ) + assert failed == ( + "2026-09-05 21:40:00,123 provider_attempt_failed agent_id=openrouter_gemma " + "model=google/gemma-3-12b-it:free attempt=2 error_type=HTTPError transient=True " + "error_message=" + ) + assert secret not in failed + assert sanitize_line( + "provider_backoff agent_id=nvidia_nim_x attempt=1 delay_seconds=0.500" + ) == "provider_backoff agent_id=nvidia_nim_x attempt=1 delay_seconds=0.500" + assert sanitize_line( + "INFO:contextual_orchestrator.orchestrator:provider_no_retry_budget agent_id=bytez_a " + "model=m/x attempts=1 final_error_type=InvalidChatResponse transient=False" + ) == ( + "provider_no_retry_budget agent_id=bytez_a model=m/x attempts=1 " + "final_error_type=InvalidChatResponse transient=False" + ) + assert sanitize_line( + "provider_rejected_permanent agent_id=bytez_a model=m/x attempts=1 final_error_type=ValueError" + ) == "provider_rejected_permanent agent_id=bytez_a model=m/x attempts=1 final_error_type=ValueError" + assert sanitize_line( + "2026-09-05 21:41:02,000 WARNING contextual_orchestrator.orchestrator circuit_opened " + "agent_id=nvidia_nim_x failures=3.0 threshold=3 reset_seconds=30.0" + ) == "2026-09-05 21:41:02,000 circuit_opened agent_id=nvidia_nim_x failures=3.0 threshold=3 reset_seconds=30.0" + assert sanitize_line("circuit_failure agent_id=nvidia_nim_x failures=2.0 threshold=3") == ( + "circuit_failure agent_id=nvidia_nim_x failures=2.0 threshold=3" + ) + assert sanitize_line("circuit_reset agent_id=nvidia_nim_x") == "circuit_reset agent_id=nvidia_nim_x" + assert sanitize_line("circuit_cleared agent_id=nvidia_nim_x") == "circuit_cleared agent_id=nvidia_nim_x" + + # Tampered or free-text variants stay out: an uppercase agent id, trailing text + # after a complete template, a failed-attempt line that lacks the error_message + # boundary, and a prefix with no known template. + assert sanitize_line("provider_attempt agent_id=NVIDIA model=m/x attempt=1/3") is None + assert sanitize_line(f"provider_attempt agent_id=nvidia_nim_x model=m/x attempt=1/3 {secret}") is None + assert sanitize_line( + "provider_attempt_failed agent_id=nvidia_nim_x model=m/x attempt=1 error_type=E transient=False" + ) is None + assert sanitize_line(f"DEBUG:contextual_orchestrator.orchestrator:{secret}") is None + + +def test_sidecar_stream_sanitizer_matches_real_formatter_output() -> None: + """Fixtures typed from a template miss runtime value types; render the real records. + + The circuit counters are floats in the orchestrator (``failures`` starts at + ``0.0`` and is incremented by ``1.0``; ``circuit_reset_seconds`` is ``30.0``), so + the lines that actually reach stderr say ``failures=2.0``, not ``failures=2``. + Render each template through ``logging.Formatter`` with the sidecar format + and the runtime value types, and require every one to pass. + """ + import logging + + namespace = _load_sanitizer() + sanitize_line = namespace["sanitize_line"] + formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s") + records = ( + (logging.DEBUG, "provider_attempt agent_id=%s model=%s attempt=%d/%d", ("nvidia_nim_x", "deepseek-ai/deepseek-v4-flash-0731", 1, 3)), + (logging.DEBUG, "provider_attempt_failed agent_id=%s model=%s attempt=%d error_type=%s transient=%s error_message=%s", ("nvidia_nim_x", "deepseek-ai/deepseek-v4-flash-0731", 1, "TimeoutError", True, "Bearer sk-secret in body")), + (logging.DEBUG, "provider_backoff agent_id=%s attempt=%d delay_seconds=%.3f", ("nvidia_nim_x", 1, 0.5)), + (logging.WARNING, "provider_exhausted agent_id=%s model=%s attempts=%s final_error_type=%s", ("nvidia_nim_x", "deepseek-ai/deepseek-v4-flash-0731", 3, "TimeoutError")), + (logging.WARNING, "provider_rejected_permanent agent_id=%s model=%s attempts=%s final_error_type=%s", ("bytez_a", "m/x", 1, "ValueError")), + (logging.WARNING, "provider_no_retry_budget agent_id=%s model=%s attempts=%s final_error_type=%s transient=%s", ("bytez_a", "m/x", 1, "InvalidChatResponse", False)), + (logging.DEBUG, "circuit_failure agent_id=%s failures=%s threshold=%s", ("nvidia_nim_x", 2.0, 3)), + (logging.WARNING, "circuit_opened agent_id=%s failures=%s threshold=%s reset_seconds=%s", ("nvidia_nim_x", 3.0, 3, 30.0)), + (logging.DEBUG, "circuit_reset agent_id=%s", ("nvidia_nim_x",)), + (logging.DEBUG, "circuit_cleared agent_id=%s", ("nvidia_nim_x",)), + ) + for level, template, args in records: + record = logging.LogRecord( + "contextual_orchestrator.orchestrator", level, __file__, 0, template, args, None + ) + rendered = formatter.format(record) + sanitized = sanitize_line(rendered) + assert sanitized is not None, rendered + assert "sk-secret" not in sanitized + assert sanitized.split(" ", 2)[2].split(" ")[0] == template.split(" ")[0] + assert sanitize_line( + formatter.format( + logging.LogRecord( + "contextual_orchestrator.orchestrator", logging.DEBUG, __file__, 0, + "circuit_failure agent_id=%s failures=%s threshold=%s", ("nvidia_nim_x", 2.0, 3), None, + ) + ) + ).endswith("circuit_failure agent_id=nvidia_nim_x failures=2.0 threshold=3") + + def test_sidecar_stream_sanitizer_summarizes_unstructured_and_traceback_lines( monkeypatch: pytest.MonkeyPatch, ) -> None: From fe827e133e7d867015d088777553e22736344c55 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 07:15:08 +0900 Subject: [PATCH 09/80] fix(noema): upload the sidecar stderr and preflight report when the verdict phase fails (#1944) A failed noema-review run left artifacts=0, so a 3122 s walk across six ready routes ending in HTTP 502 (run 33981136873) was diagnosable only from the caller's one-line summary. Ship the sanitized sidecar stderr and the preflight report on failure, using the same pinned upload-artifact and the same file Strix already publishes in strix-reports. Refs #1935, #1939 Co-authored-by: Claude Fable 5.1 --- .github/workflows/noema-review.yml | 11 +++++++ CHANGELOG.md | 3 ++ ...st_noema_orchestrator_workflow_contract.py | 31 +++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index 21ea967201..f8ab55c896 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -688,6 +688,17 @@ jobs: echo "::notice::Noema model phase produced no publishable envelope; publication is skipped." fi + - name: Upload contextual-orchestrator sidecar evidence on failure + if: failure() && env.PR_NUMBER != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: noema-sidecar-evidence + path: | + strix_runs/contextual-orchestrator-sidecar.stderr.log + strix_runs/contextual-orchestrator-preflight.json + if-no-files-found: ignore + retention-days: 5 + - name: Refresh repository-scoped Noema GitHub App token for publication if: env.PR_NUMBER != '' && steps.noema_prepare.outputs.prepared == 'true' && steps.noema_credential.outputs.source == 'github-app' id: noema_github_app_publication_token diff --git a/CHANGELOG.md b/CHANGELOG.md index 15d9e6e188..55a2a2f211 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +### Noema review ships sidecar evidence on failure + +- `noema-review.yml` now uploads `strix_runs/contextual-orchestrator-sidecar.stderr.log` and `strix_runs/contextual-orchestrator-preflight.json` as the `noema-sidecar-evidence` artifact when the verdict phase fails (`if: failure()`, the same pinned `actions/upload-artifact` Strix uses, `if-no-files-found: ignore`, 5-day retention). Until now a failed Noema run left `artifacts=0` -- run `33981136873` spent 3122 s walking six ready routes twice each and ended in HTTP 502 with no per-route trace anywhere but the sidecar's stderr -- so the only diagnosis available was the caller's one-line summary. The stderr file is the sanitizer's bounded allowlist output (`sanitize_contextual_orchestrator_sidecar_stream.py`), the same file Strix already publishes in `strix-reports`; per-attempt route outcomes still need an allowlisted structured line from the orchestrator to appear in it. Refs #1935, #1939. ### Sidecar sanitizer admits orchestrator route and circuit events - `scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py` now passes the orchestrator's own `provider_attempt`, `provider_attempt_failed` (cut before the free-text `error_message=`), `provider_backoff`, `provider_exhausted`, `provider_rejected_permanent`, `provider_no_retry_budget` and `circuit_failure|opened|reset|cleared` lines (whose `failures`/`reset_seconds` are floats at runtime, `2.0`/`30.0`), matched field by field against bounded identifier and number charsets, with either Python's default `LEVEL:name:` prefix or the sidecar formatter's `asctime LEVEL name` prefix (the timestamp is kept so per-route durations can be read as differences). Until now every one of these lines was folded into `omitted_unstructured_lines`, so the `provider_exhausted` WARNING that already fires today after a route's retry budget is spent never reached an artifact, and a 3122 s walk across six ready routes (run `33981136873`) had no per-route trace. Companion to #1943 (sidecar DEBUG logging) and #1944 (Noema uploads the file on failure). Refs #1935, #1939. diff --git a/tests/test_noema_orchestrator_workflow_contract.py b/tests/test_noema_orchestrator_workflow_contract.py index 937cf6fe97..628fa3cbc1 100644 --- a/tests/test_noema_orchestrator_workflow_contract.py +++ b/tests/test_noema_orchestrator_workflow_contract.py @@ -485,3 +485,34 @@ def test_noema_review_job_has_no_job_level_timeout() -> None: encoding="utf-8" ) ), "the two-hour-per-model allowance this bound relies on must still be documented" + + +def test_noema_review_uploads_sidecar_evidence_on_failure() -> None: + """A failed verdict phase ships the sanitized sidecar stderr and preflight report. + + Before this step a failed Noema run left ``artifacts=0`` (run 33981136873: + 3122 s, then HTTP 502, no per-route trace in the job log). The stderr file + is the sidecar sanitizer's bounded allowlist output -- the same file Strix + already publishes in ``strix-reports`` -- so shipping it on failure adds + diagnosis without adding exposure (#1935 follow-up). + """ + workflow = workflow_text("noema-review.yml") + name = "Upload contextual-orchestrator sidecar evidence on failure" + step = workflow_step(workflow, name) + assert "if: failure() && env.PR_NUMBER != ''" in step + strix_pin = re.search( + r"actions/upload-artifact@([0-9a-f]{40})", workflow_text("strix.yml") + ).group(1) + assert f"actions/upload-artifact@{strix_pin}" in step + assert "name: noema-sidecar-evidence" in step + assert "strix_runs/contextual-orchestrator-sidecar.stderr.log" in step + assert "strix_runs/contextual-orchestrator-preflight.json" in step + assert "if-no-files-found: ignore" in step + assert "retention-days: 5" in step + prepare = workflow.index(" - name: Prepare Noema model verdict\n") + upload = workflow.index(f" - name: {name}\n") + refresh = workflow.index( + " - name: Refresh repository-scoped Noema GitHub App token for publication\n" + ) + assert prepare < upload < refresh + assert workflow.count("actions/upload-artifact@") == 1 From 0e42fbcaf341532616ef0a83404dce8d0fd5b42a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 09:18:00 +0900 Subject: [PATCH 10/80] fix(sidecar): keep transient-rejected preflight routes as deferred failover The review sidecar's preflight discarded a route the moment its 16-token probe raised, including on 429 and 5xx, while the serving gateway treats exactly those statuses as transient: it retries the route and fails over across it (provider_errors.PROVIDER_STATUS_SURFACES marks 429 retryable; orchestrator.TRANSIENT_HTTP_STATUS). Under concurrent CI load the probes spend the per-key budgets themselves: noema-review run 33993637015 (.github#1687, 2026-09-05) rejected 11 of 12 routes -- six with 429, three of them on NVIDIA keys whose sibling routes were ready -- served the single ready route for 542 s and returned 502. Routes whose probe answered with a status in the gateway's transient set are now kept as deferred, ranked after every ready route by a catalog priority penalty, so failover has somewhere to go. A probe that timed out records no http_status and stays rejected, so the silent route whose request costs the full two-layer retry budget is never admitted on that evidence. ready_count is unchanged, deferred_count is reported, and with no ready route the stage still fails, so ADR-0005's priced-catalog fallback contract is untouched. The escalation path is not touched. The stream sanitizer admits preflight_route_deferred next to preflight_route_rejected. Tests: deferral order and priorities, all-transient still fails, frozen and plain agent demotion, deferred log line, sanitizer pass/drop/scrub; all five fail against main's launcher and sanitizer. Gate on this tree: 2909 passed, coverage 100%, interrogate 100% (final commit differs from the gated tree by one comment; the touched module and interrogate re-run). Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 4 + ...contextual_orchestrator_review_launcher.py | 68 +++++++++- ..._contextual_orchestrator_sidecar_stream.py | 5 +- ...l_orchestrator_review_runtime_preflight.py | 121 ++++++++++++++++++ 4 files changed, 191 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55a2a2f211..600728538b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Review sidecar preflight keeps transient-rejected routes as deferred failover + +- `_preflight_review_agents` no longer discards a route whose 16-token probe answered with a status the serving gateway itself retries and fails over across (`408 409 425 429 500 502 503 504 529`, the vendored orchestrator's `TRANSIENT_HTTP_STATUS`). Such routes are kept as **deferred**, ranked after every ready route by a catalog-priority penalty, so a stalled or rate-limited ready route has somewhere to fail over to; `ready_count` is unchanged, a new `deferred_count` is reported, and `rejected_count` covers only routes the gateway would not retry either (404, auth failures, invalid responses). With no ready route the stage still fails as before, so ADR-0005's priced-catalog fallback contract is untouched. Motivation: `noema-review` run 33993637015 (2026-09-05) rejected 11 of 12 routes -- six with 429, three of them on NVIDIA keys whose sibling routes were ready -- served the single ready route for 542 s and returned 502; under this rule the same run would have served 1 ready + 6 deferred. The sanitized stream gains a `preflight_route_deferred` line alongside `preflight_route_rejected`. + ### Noema review ships sidecar evidence on failure - `noema-review.yml` now uploads `strix_runs/contextual-orchestrator-sidecar.stderr.log` and `strix_runs/contextual-orchestrator-preflight.json` as the `noema-sidecar-evidence` artifact when the verdict phase fails (`if: failure()`, the same pinned `actions/upload-artifact` Strix uses, `if-no-files-found: ignore`, 5-day retention). Until now a failed Noema run left `artifacts=0` -- run `33981136873` spent 3122 s walking six ready routes twice each and ended in HTTP 502 with no per-route trace anywhere but the sidecar's stderr -- so the only diagnosis available was the caller's one-line summary. The stderr file is the sanitizer's bounded allowlist output (`sanitize_contextual_orchestrator_sidecar_stream.py`), the same file Strix already publishes in `strix-reports`; per-attempt route outcomes still need an allowlisted structured line from the orchestrator to appear in it. Refs #1935, #1939. diff --git a/scripts/ci/contextual_orchestrator_review_launcher.py b/scripts/ci/contextual_orchestrator_review_launcher.py index 502843c994..85f0fbc4b1 100644 --- a/scripts/ci/contextual_orchestrator_review_launcher.py +++ b/scripts/ci/contextual_orchestrator_review_launcher.py @@ -22,6 +22,8 @@ from __future__ import annotations import argparse +import copy +import dataclasses import json import logging import os @@ -63,6 +65,28 @@ # Shared cap on how many candidates in one preflight run may use the # escalation retry above. It bounds request count, never model response time. REVIEW_PREFLIGHT_MAX_ESCALATIONS = 4 +# Probe outcomes the serving gateway itself treats as transient -- it retries +# the same route and then fails over across exactly these statuses +# (contextual_orchestrator.orchestrator.TRANSIENT_HTTP_STATUS at the vendored +# pin; provider_errors.PROVIDER_STATUS_SURFACES marks 429 retryable). A route +# that answered one of them to the 16-token probe is not known to be dead; it +# was rate-limited or unlucky in the second the probe ran, very often because +# the probe itself spent the per-key budget. Discarding it left the serving +# set with nothing to fail over to: on 2026-09-05 a noema-review preflight +# rejected 11 of 12 routes -- six of them with 429 -- served the one ready +# route for 542 s and returned 502. Such routes are kept as *deferred*, ranked +# after every ready route, so failover has somewhere to go. Only a route that +# *answered* with one of these statuses qualifies (a probe that timed out +# records no http_status and stays rejected), so deferral never admits, on the +# strength of a probe that already showed it, the silent route whose request +# costs the full two-layer retry budget. Keep this set in sync with the +# vendored orchestrator's; a status the gateway would not retry must not be +# deferred. +REVIEW_PREFLIGHT_DEFERRABLE_HTTP_STATUS = frozenset({408, 409, 425, 429, 500, 502, 503, 504, 529}) +# Subtracted from a deferred route's catalog priority so the orchestrator's +# ranking (higher priority first; catalog priorities are 0..-11) never places a +# deferred route ahead of a ready one. +REVIEW_PREFLIGHT_DEFERRED_PRIORITY_PENALTY = 1000 class ReviewPreflightError(RuntimeError): @@ -280,6 +304,22 @@ def _record_provider_exception(row: dict[str, object], exc: Exception) -> None: row.pop("reasoning_without_content", None) +def _demote_agent(agent: object, penalty: int) -> object: + """Return a copy of ``agent`` whose ``priority`` is lowered by ``penalty``. + + Serving agents are frozen ``ModelAgent`` dataclasses, so the copy goes + through :func:`dataclasses.replace`; the plain objects tests use are + shallow-copied and assigned. A missing ``priority`` counts as 0, matching + the dataclass default. + """ + priority = int(getattr(agent, "priority", 0)) - penalty + if dataclasses.is_dataclass(agent) and not isinstance(agent, type): + return dataclasses.replace(agent, priority=priority) + demoted = copy.copy(agent) + demoted.priority = priority + return demoted + + def _response_has_reasoning_without_content(response: object) -> bool: """Return whether a response matches the vendored "reasoning, no content" signature. @@ -508,11 +548,28 @@ def _preflight_review_agents( ) routes.append(row) + # Deferral pass: a route rejected with a status the serving gateway would + # retry and fail over across is kept behind the ready routes instead of + # being discarded -- but only once at least one route is ready. With no + # ready route the run still fails this stage exactly as before, so + # _preflight_with_fallback's "priced catalog only after every primary + # route rejects" contract (ADR-0005) is unchanged. ``routes`` holds one + # row per agent in ``agents`` order (every branch above appends once). + deferred: list[object] = [] + if viable: + for agent, row in zip(agents, routes): + if ( + row.get("status") == "rejected" + and row.get("http_status") in REVIEW_PREFLIGHT_DEFERRABLE_HTTP_STATUS + ): + row["status"] = "deferred" + deferred.append(_demote_agent(agent, REVIEW_PREFLIGHT_DEFERRED_PRIORITY_PENALTY)) report: dict[str, object] = { "contract": "strix-plain-chat-preflight-v2", "probed_count": len(agents), "ready_count": len(viable), - "rejected_count": len(agents) - len(viable), + "deferred_count": len(deferred), + "rejected_count": len(agents) - len(viable) - len(deferred), "escalations_used": escalations_used, "escalation_budget": REVIEW_PREFLIGHT_MAX_ESCALATIONS, "routes": routes, @@ -521,7 +578,7 @@ def _preflight_review_agents( raise ReviewPreflightError( "no provider route passed the Strix plain-chat preflight", report ) - return viable, report + return [*viable, *deferred], report def _preflight_with_fallback( @@ -581,8 +638,9 @@ def _log_preflight_rejections(report: dict[str, object]) -> None: if not isinstance(routes, list): return for row in routes: - if not isinstance(row, dict) or row.get("status") != "rejected": + if not isinstance(row, dict) or row.get("status") not in ("rejected", "deferred"): continue + event = f"preflight_route_{row['status']}" # Re-validate rather than trust the caller's own sanitization: this # print reaches the sidecar's sanitized stderr stream unchanged, so an # out-of-contract value here (not a plain identifier) must degrade to @@ -602,13 +660,13 @@ def _log_preflight_rejections(report: dict[str, object]) -> None: http_status = row.get("http_status") if isinstance(http_status, int) and not isinstance(http_status, bool) and 100 <= http_status <= 599: print( - f"preflight_route_rejected provider={provider} " + f"{event} provider={provider} " f"error_type={error_type} http_status={http_status}", file=sys.stderr, ) else: print( - f"preflight_route_rejected provider={provider} error_type={error_type}", + f"{event} provider={provider} error_type={error_type}", file=sys.stderr, ) diff --git a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py index 51b9a5df27..a3227f2e32 100644 --- a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py +++ b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py @@ -16,7 +16,7 @@ r"code=(?P[A-Za-z0-9_.-]{1,64})" ) _PREFLIGHT_ROUTE_REJECTED = re.compile( - r"preflight_route_rejected provider=(?P[a-z][a-z0-9_]{0,63}) " + r"preflight_route_(?Prejected|deferred) provider=(?P[a-z][a-z0-9_]{0,63}) " r"error_type=(?P[A-Za-z_][A-Za-z0-9_]{0,63})" r"(?: http_status=(?P[1-5][0-9]{2}))?" ) @@ -116,7 +116,8 @@ def sanitize_line(line: str) -> str | None: preflight_route_rejected = _PREFLIGHT_ROUTE_REJECTED.search(stripped) if preflight_route_rejected is not None: summary = ( - f"preflight_route_rejected provider={preflight_route_rejected.group('provider')} " + f"preflight_route_{preflight_route_rejected.group('event')} " + f"provider={preflight_route_rejected.group('provider')} " f"error_type={preflight_route_rejected.group('error_type')}" ) http_status = preflight_route_rejected.group("http_status") diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index d0ace81e8b..902237bdad 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -1943,3 +1943,124 @@ def test_main_configures_sidecar_logging_before_touching_credentials() -> None: credentials_at = source.index("registered = register_review_credentials(os.environ)") assert configure_at < credentials_at assert "from contextual_orchestrator.debug_logging import configure_logging" in source + + +def _preflight_agents(*ids: str) -> list[SimpleNamespace]: + """Return catalog-shaped agents with descending priorities, one per id.""" + return [ + SimpleNamespace(id=agent_id, provider_name=agent_id.split("_")[0], model=f"{agent_id}/m", priority=-index) + for index, agent_id in enumerate(ids) + ] + + +class _StatusError(Exception): + """Exception with a ``code`` attribute, the shape ``_safe_http_status`` reads.""" + + def __init__(self, code: int) -> None: + super().__init__(f"HTTP Error {code}") + self.code = code + + +def test_preflight_defers_transient_probe_statuses_behind_ready_routes() -> None: + """A 429/5xx probe answer keeps the route, ranked after every ready route. + + noema-review run 33993637015 (2026-09-05) rejected 11 of 12 routes -- six + with 429 -- served the single ready route for 542 s and returned 502. The + serving gateway retries and fails over across exactly these statuses, so + discarding them at preflight left it nowhere to go. + """ + namespace = _load_launcher() + agents = _preflight_agents("nvidia_ready", "openrouter_limited", "nvidia_missing", "nvidia_down") + client = _ProbeClient( + { + "nvidia_ready": {"choices": [{"message": {"content": "OK"}, "finish_reason": "stop"}]}, + "openrouter_limited": _StatusError(429), + "nvidia_missing": _StatusError(404), + "nvidia_down": _StatusError(503), + } + ) + served, report = namespace["_preflight_review_agents"](agents, client=client) + + assert [agent.id for agent in served] == ["nvidia_ready", "openrouter_limited", "nvidia_down"] + assert served[0].priority == 0 + penalty = namespace["REVIEW_PREFLIGHT_DEFERRED_PRIORITY_PENALTY"] + assert served[1].priority == -1 - penalty + assert served[2].priority == -3 - penalty + assert agents[1].priority == -1, "deferral must not mutate the caller's agent" + assert report["ready_count"] == 1 + assert report["deferred_count"] == 2 + assert report["rejected_count"] == 1 + statuses = {row["agent_id"]: row["status"] for row in report["routes"]} + assert statuses == { + "nvidia_ready": "ready", + "openrouter_limited": "deferred", + "nvidia_missing": "rejected", + "nvidia_down": "deferred", + } + + +def test_preflight_still_fails_when_no_route_is_ready() -> None: + """All-transient rejections keep failing the stage so the priced fallback still runs.""" + namespace = _load_launcher() + agents = _preflight_agents("openrouter_a", "nvidia_b") + client = _ProbeClient({"openrouter_a": _StatusError(429), "nvidia_b": _StatusError(429)}) + with pytest.raises(namespace["ReviewPreflightError"]) as excinfo: + namespace["_preflight_review_agents"](agents, client=client) + report = excinfo.value.report + assert report["ready_count"] == 0 + assert report["deferred_count"] == 0 + assert report["rejected_count"] == 2 + assert {row["status"] for row in report["routes"]} == {"rejected"} + + +def test_demote_agent_handles_frozen_dataclasses_and_plain_objects() -> None: + """The serving ``ModelAgent`` is a frozen dataclass; test doubles are plain objects.""" + import dataclasses + + namespace = _load_launcher() + + @dataclasses.dataclass(frozen=True) + class _Frozen: + id: str + priority: int = 0 + + frozen = _Frozen(id="a", priority=-2) + demoted = namespace["_demote_agent"](frozen, 1000) + assert demoted.priority == -1002 and frozen.priority == -2 + plain = SimpleNamespace(id="b") + demoted_plain = namespace["_demote_agent"](plain, 1000) + assert demoted_plain.priority == -1000 and not hasattr(plain, "priority") + + +def test_log_preflight_rejections_reports_deferred_routes(capsys: pytest.CaptureFixture[str]) -> None: + """Deferred routes get their own bounded line so the stream tells them apart.""" + namespace = _load_launcher() + namespace["_log_preflight_rejections"]( + { + "routes": [ + {"provider": "openrouter", "status": "deferred", "error_type": "HTTPError", "http_status": 429}, + {"provider": "nvidia_nim", "status": "rejected", "error_type": "HTTPError", "http_status": 404}, + {"provider": "nvidia_nim_sub", "status": "ready"}, + ] + } + ) + err = capsys.readouterr().err + assert "preflight_route_deferred provider=openrouter error_type=HTTPError http_status=429" in err + assert "preflight_route_rejected provider=nvidia_nim error_type=HTTPError http_status=404" in err + assert "nvidia_nim_sub" not in err + + +def test_sidecar_stream_sanitizer_passes_deferred_preflight_lines() -> None: + """``preflight_route_deferred`` reaches the artifact with the same bounded fields as rejected.""" + sanitizer = _load_sanitizer() + sanitize_line = sanitizer["sanitize_line"] + deferred = "preflight_route_deferred provider=openrouter error_type=HTTPError http_status=429" + rejected = "preflight_route_rejected provider=nvidia_nim error_type=HTTPError http_status=404" + assert sanitize_line(deferred) == deferred + assert sanitize_line(rejected) == rejected + assert sanitize_line("preflight_route_deferred provider=openrouter error_type=HTTPError") == ( + "preflight_route_deferred provider=openrouter error_type=HTTPError" + ) + assert sanitize_line("preflight_route_paused provider=openrouter error_type=HTTPError http_status=429") is None + assert sanitize_line(deferred + " token=sk-secret") is not None + assert "sk-secret" not in sanitize_line(deferred + " token=sk-secret") From 53c0a87b7fa50a7d1d3f402198922d4cd3ad76a7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 10:23:23 +0900 Subject: [PATCH 11/80] docs(sidecar): describe the deferral bound in passthrough terms The review request is served by the passthrough path (one attempt per candidate via proxy_send_once, stream=False), so a silent route costs one socket-silence timeout, not a multi-layer retry budget. Comment-only. Co-Authored-By: Claude Fable 5.1 --- scripts/ci/contextual_orchestrator_review_launcher.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/ci/contextual_orchestrator_review_launcher.py b/scripts/ci/contextual_orchestrator_review_launcher.py index 85f0fbc4b1..7cd94e2d77 100644 --- a/scripts/ci/contextual_orchestrator_review_launcher.py +++ b/scripts/ci/contextual_orchestrator_review_launcher.py @@ -78,10 +78,10 @@ # after every ready route, so failover has somewhere to go. Only a route that # *answered* with one of these statuses qualifies (a probe that timed out # records no http_status and stays rejected), so deferral never admits, on the -# strength of a probe that already showed it, the silent route whose request -# costs the full two-layer retry budget. Keep this set in sync with the -# vendored orchestrator's; a status the gateway would not retry must not be -# deferred. +# strength of a probe that already showed it, the silent route whose single +# passthrough attempt would sit through the socket-silence timeout. Keep this +# set in sync with the vendored orchestrator's; a status the gateway would not +# retry must not be deferred. REVIEW_PREFLIGHT_DEFERRABLE_HTTP_STATUS = frozenset({408, 409, 425, 429, 500, 502, 503, 504, 529}) # Subtracted from a deferred route's catalog priority so the orchestrator's # ranking (higher priority first; catalog priorities are 0..-11) never places a From 37a1129aeaa27ebc8b3d8999c83b060008d0123d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 10:29:47 +0900 Subject: [PATCH 12/80] docs(sidecar): state the deferral bound in terms of the gateway retry budget The first noema-sidecar-evidence trace (.github#1661 run 33995553859) shows the review path is _invoke: a silent route costs two rounds of three 90 s timeouts. Word the constant's comment on that basis instead of the single-attempt passthrough wording of the previous commit. Comment-only. Co-Authored-By: Claude Fable 5.1 --- scripts/ci/contextual_orchestrator_review_launcher.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/ci/contextual_orchestrator_review_launcher.py b/scripts/ci/contextual_orchestrator_review_launcher.py index 7cd94e2d77..27917eba4e 100644 --- a/scripts/ci/contextual_orchestrator_review_launcher.py +++ b/scripts/ci/contextual_orchestrator_review_launcher.py @@ -78,10 +78,10 @@ # after every ready route, so failover has somewhere to go. Only a route that # *answered* with one of these statuses qualifies (a probe that timed out # records no http_status and stays rejected), so deferral never admits, on the -# strength of a probe that already showed it, the silent route whose single -# passthrough attempt would sit through the socket-silence timeout. Keep this -# set in sync with the vendored orchestrator's; a status the gateway would not -# retry must not be deferred. +# strength of a probe that already showed it, the silent route whose serving +# request would spend the gateway's full retry budget in 90 s timeouts. Keep +# this set in sync with the vendored orchestrator's; a status the gateway +# would not retry must not be deferred. REVIEW_PREFLIGHT_DEFERRABLE_HTTP_STATUS = frozenset({408, 409, 425, 429, 500, 502, 503, 504, 529}) # Subtracted from a deferred route's catalog priority so the orchestrator's # ranking (higher priority first; catalog priorities are 0..-11) never places a From 6d897c990e473a9d3a34bd1b6547f39b9bc344fb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 10:45:04 +0900 Subject: [PATCH 13/80] fix(sidecar): fill the preflight served set lazily to a readiness target The catalog becomes a candidate list (up to 24, per-account cap 8, the tier-then-round-robin order from #1939) and _preflight_review_agents probes it in order, stopping at REVIEW_PREFLIGHT_TARGET_READY = 8 ready routes or REVIEW_PREFLIGHT_MAX_PROBES = 16 probes, whichever comes first. A permanently dead candidate (NIM lists gemma-3-12b/4b and answers 404 on every run) costs one probe instead of a served slot; a healthy pool stops early. Why: after #1939's four-per-account slice, each NVIDIA key's slots were its first four models alphabetically -- deepseek-v4-flash, deepseek-v4-pro and the two gemma-3 404s -- so each key served two contended routes. Preflight ready fell from 6/12 to 1-3/12 and noema-review in .github went from 7 successes / 14 failures to 0 / 22 (lane jan's census on #1948, verdict-step conclusions). - REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES 12 -> 24, PRIMARY_ROUTE_LIMIT 8 -> 16; sidecar ORCHESTRATOR_CATALOG_LIMIT default and the composite action's catalog_limit default 12 -> 24. - Report gains candidate_count, target_ready, probe_budget; probed_count now counts probes sent; rejected_count = probed - ready - deferred. #1947's deferral pass pairs exactly the probed prefix. - ADR-0029 (new; 0028 is claimed by #1668), ADR-0003 section 2 stage-budget sentence amended, CHANGELOG. Tests: four new lazy-fill tests plus the updated stage-limit invariant; all five fail on #1947's tree (negative control). Gate: 2913 passed, 1 skipped, coverage 100% (13150 statements, 0 missed), interrogate 100%. Refs #1939, #1947, #1948. Co-Authored-By: Claude Fable 5.1 --- .../orchestrator-free-sidecar/action.yml | 4 +- CHANGELOG.md | 4 + ...ntextual-orchestrator-vendored-free-zdr.md | 9 +- docs/adr/0029-sidecar-preflight-lazy-fill.md | 47 ++++++++ ...contextual_orchestrator_review_launcher.py | 41 +++++-- .../contextual_orchestrator_review_sidecar.sh | 7 +- ...l_orchestrator_review_runtime_preflight.py | 111 +++++++++++++++++- 7 files changed, 201 insertions(+), 22 deletions(-) create mode 100644 docs/adr/0029-sidecar-preflight-lazy-fill.md diff --git a/.github/actions/orchestrator-free-sidecar/action.yml b/.github/actions/orchestrator-free-sidecar/action.yml index 196c86b0f6..edddfe1bc3 100644 --- a/.github/actions/orchestrator-free-sidecar/action.yml +++ b/.github/actions/orchestrator-free-sidecar/action.yml @@ -6,9 +6,9 @@ inputs: required: false default: "false" catalog_limit: - description: Maximum discovered route catalog size for the sidecar preflight. + description: Maximum discovered route catalog size for the sidecar preflight (a candidate list probed lazily to a readiness target, ADR-0029). required: false - default: "12" + default: "24" catalog_account_cap: description: Maximum routes admitted from one credential account. required: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 600728538b..cdb7632af3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Review sidecar preflight fills the served set lazily to a readiness target + +- `_preflight_review_agents` now treats the catalog as a candidate list, probed in its tier-then-round-robin order until `REVIEW_PREFLIGHT_TARGET_READY = 8` routes are ready or `REVIEW_PREFLIGHT_MAX_PROBES = 16` probes are spent (ADR-0029). The candidate budget rises from 12 to 24 (`REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES`; auto pool split 16 free / 8 priced; the sidecar's `ORCHESTRATOR_CATALOG_LIMIT` default follows), the per-account cap stays 8. A permanently dead candidate -- NIM lists `gemma-3-12b`/`gemma-3-4b` and answers 404 on every run -- now costs one probe instead of a served slot, and a healthy pool stops early instead of always probing every candidate. Motivation: after #1939's four-per-account slice each NVIDIA key's slots were its first four models alphabetically, two of them those 404s, so preflight readiness fell from 6/12 to 1–3/12 and `noema-review` on this repository went from 7 successes / 14 failures to 0 / 22. The report gains `candidate_count`, `target_ready` and `probe_budget`; `probed_count` counts probes actually sent. ADR-0003's stage-budget sentence is amended. Refs #1939, #1947, #1948. + ### Review sidecar preflight keeps transient-rejected routes as deferred failover - `_preflight_review_agents` no longer discards a route whose 16-token probe answered with a status the serving gateway itself retries and fails over across (`408 409 425 429 500 502 503 504 529`, the vendored orchestrator's `TRANSIENT_HTTP_STATUS`). Such routes are kept as **deferred**, ranked after every ready route by a catalog-priority penalty, so a stalled or rate-limited ready route has somewhere to fail over to; `ready_count` is unchanged, a new `deferred_count` is reported, and `rejected_count` covers only routes the gateway would not retry either (404, auth failures, invalid responses). With no ready route the stage still fails as before, so ADR-0005's priced-catalog fallback contract is untouched. Motivation: `noema-review` run 33993637015 (2026-09-05) rejected 11 of 12 routes -- six with 429, three of them on NVIDIA keys whose sibling routes were ready -- served the single ready route for 542 s and returned 502; under this rule the same run would have served 1 ready + 6 deferred. The sanitized stream gains a `preflight_route_deferred` line alongside `preflight_route_rejected`. diff --git a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md index 04dc04c7a2..a39f0595cf 100644 --- a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md +++ b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md @@ -50,9 +50,12 @@ all five, and auto-optimize routing by cost. route rejects the real runtime request contract does it rebuild once from fully price-attested routes and record the rejected primary attempt. This is evidence-triggered failover, not an arbitrary free/paid mixing ratio. - Both stages share one twelve-route startup budget: no more than eight routes - enter the free primary stage and only its remaining capacity may enter priced - fallback. Full discovery counts remain in policy evidence, and the transient + Both stages share one bounded startup budget of twenty-four candidates: no + more than sixteen enter the free primary stage and only its remaining + capacity may enter priced fallback. Candidates are probed lazily in catalog + order until eight routes are ready or sixteen probes are spent per stage + (ADR-0029), so a dead candidate costs one probe, not a served slot. Full + discovery counts remain in policy evidence, and the transient priced catalog is removed immediately after loading. 3. **ZDR-first within each cost tier**: `scripts/ci/zdr_policy.py` defines ZDR the way OpenRouter does ("a provider will not store your data for any period diff --git a/docs/adr/0029-sidecar-preflight-lazy-fill.md b/docs/adr/0029-sidecar-preflight-lazy-fill.md new file mode 100644 index 0000000000..e5abca67e0 --- /dev/null +++ b/docs/adr/0029-sidecar-preflight-lazy-fill.md @@ -0,0 +1,47 @@ +# ADR-0029: Review sidecar preflight fills the served set lazily to a readiness target + +- **Status:** Proposed +- **Date:** 2026-09-06 +- **Scope:** `scripts/ci/contextual_orchestrator_review_launcher.py` (`_preflight_review_agents`, the stage limits), `scripts/ci/contextual_orchestrator_review_sidecar.sh` (`ORCHESTRATOR_CATALOG_LIMIT` default), ADR-0003 §2's stage budget sentence +- **Amends:** ADR-0003 (the "twelve-route startup budget" clause). ADR-0005's attempt counts are historical and are not restored. + +## Problem + +The review sidecar selected a fixed catalog of twelve routes and probed every one of them, then served whatever was ready. `.github#1939` made the selection diverse (round-robin across credential accounts inside each cost/ZDR tier, four routes per account), which was right, but it exposed a second defect: the per-account slice is filled from an alphabetically sorted model list, and for both NVIDIA NIM keys the first four models are `deepseek-v4-flash`, `deepseek-v4-pro`, `gemma-3-12b`, `gemma-3-4b`. NIM lists the two `gemma-3` models but answers `404` to every chat request on every run observed. Each NVIDIA key therefore served two working routes, both the most contended models, while the pre-#1939 eight-slot fill had reached `meta/llama-3.2-11b`, `llama-3.2-90b` and `meta/muse-glimmer-30b`, which were ready in every Strix artifact of that afternoon. + +Measured on `ContextualWisdomLab/.github` (lane jan's census on `#1948`, verdict-step conclusions only, draft skips excluded): + +| window | preflight ready of 12 | `noema-review` success / failure | +|---|---|---| +| before `#1939` (`main@f2f91b80`, 2026-09-05T17:25Z) | 6, 6, 5 (16:37–16:56Z artifacts) | 7 / 14 | +| after | 1–3 (23:47Z onward) | 0 / 22 | + +The evening's rate-limit pressure is a confound; the mechanism is not. A fixed slice from a list with dead entries wastes the slice, and probing every candidate regardless of how many are already ready spends per-key rate budget (`#1948`) for nothing. + +## Constraints + +1. No model name is hard-coded anywhere in the fill; a dead candidate is discovered by its probe, not by a list. +2. Probe spend per sidecar boot stays bounded and is stated as a number, because the probes themselves consume the per-key budgets the served routes need (`#1948`). +3. `#1947`'s deferral (a probed route that answered a transient status is kept behind the ready routes) applies unchanged to whatever was probed. +4. ADR-0003's evidence-triggered priced fallback (only after every free candidate rejects) keeps its shape; the two stages still share one startup budget. +5. `ready_count` keeps its meaning (routes proven ready by a probe) so the peers' post-merge discriminators stay comparable. + +## Decision + +The catalog is a **candidate list**, not the served set. `build_zdr_prioritized_catalog` keeps its tier-then-round-robin order (`#1939`) and is asked for up to `REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES = 24` candidates (per-account cap unchanged at 8; the sidecar's `ORCHESTRATOR_CATALOG_LIMIT` default rises from 12 to 24). `_preflight_review_agents` probes candidates **in that order and stops** as soon as `REVIEW_PREFLIGHT_TARGET_READY = 8` routes are ready or `REVIEW_PREFLIGHT_MAX_PROBES = 16` probes have been spent, whichever comes first. The auto pool's split becomes 16 free candidates and up to 8 priced fallback candidates; each stage's list fits the probe cap, so the worst case remains "every candidate probed once". + +The report gains `candidate_count`, `target_ready` and `probe_budget`; `probed_count` now counts probes actually sent, and `rejected_count` is `probed − ready − deferred`. Unprobed candidates get no `routes` row. + +## Consequences + +- **Good:** a dead candidate costs one probe and yields its place to the next candidate in the same account's list; a healthy hour stops after about eight to twelve probes instead of always twelve; a bad hour is bounded at sixteen probes per stage. +- **Cost:** in an hour where nothing is ready the sidecar sends up to 16 probes per stage where it sent 12, a third more against already exhausted keys. This is the price of finding routes past the dead ones; `#1948`'s shared rate ledger is the lever above it. +- **Unchanged:** a route that answers the probe and then goes silent at request time still costs the gateway's full retry budget (`contextual-orchestrator#1045`); readiness is measured at 16 tokens (`#1454`). +- **Discriminator:** post-merge, `probed_count` versus `candidate_count` per boot and `ready_count` of the served set, read from the `runtime preflight summary` in the job log or the `noema-sidecar-evidence` artifact, compared with the table above. + +## Alternatives considered + +- **Raise the per-account cap back to 8 with a 12-route limit** — restores the pre-#1939 pool but reintroduces the single-account fill that `#1939` fixed; the 404s would still occupy slots. +- **Exclude models that 404 by name** — a hard-coded exclusion list the next discovery change silently invalidates; rejected by constraint 1. The discovery-side question (why NIM lists models it does not serve) remains open in `contextual-orchestrator`. +- **Family-level round-robin before the per-account cap** (jan's fallback proposal) — reduces same-family contention but does not touch dead candidates; can be layered later if the census shows family contention as the residual. +- **Probe all 24 candidates** — best served set, double the probe spend in the hour that can least afford it; rejected by constraint 2. diff --git a/scripts/ci/contextual_orchestrator_review_launcher.py b/scripts/ci/contextual_orchestrator_review_launcher.py index 27917eba4e..cb95df8b74 100644 --- a/scripts/ci/contextual_orchestrator_review_launcher.py +++ b/scripts/ci/contextual_orchestrator_review_launcher.py @@ -45,8 +45,18 @@ # Provider-neutral sampling: several modern endpoints reject non-default # temperatures, while 1.0 is the OpenAI-compatible default. REVIEW_TEMPERATURE = 1.0 -REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES = 12 -REVIEW_PREFLIGHT_PRIMARY_ROUTE_LIMIT = 8 +# Lazy fill (ADR-0029): the catalog is a *candidate* list, probed in its +# tier-then-round-robin order until REVIEW_PREFLIGHT_TARGET_READY routes are +# ready or REVIEW_PREFLIGHT_MAX_PROBES probes are spent, whichever comes first. +# A permanently dead candidate (NIM lists gemma-3-12b/4b but answers 404 on +# every run) then costs one probe instead of a served slot, and a healthy hour +# stops early instead of always probing every candidate. The two stage limits +# bound the candidate lists (auto pool: 16 free, the remainder priced); probe +# spend per stage is bounded by REVIEW_PREFLIGHT_MAX_PROBES, not by them. +REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES = 24 +REVIEW_PREFLIGHT_PRIMARY_ROUTE_LIMIT = 16 +REVIEW_PREFLIGHT_TARGET_READY = 8 +REVIEW_PREFLIGHT_MAX_PROBES = 16 # ADR-0005: a single fixed max_tokens cannot fit every model in a heterogeneous # pool -- some spend internal reasoning tokens before visible content and need # more, others have a real completion ceiling a large budget would exceed. The @@ -409,6 +419,13 @@ def _preflight_review_agents( response, both fields are absent entirely (there is no response to describe) rather than silently retaining the base attempt's values. + Candidates are probed lazily in catalog order (ADR-0029): probing stops + once ``REVIEW_PREFLIGHT_TARGET_READY`` routes are ready or + ``REVIEW_PREFLIGHT_MAX_PROBES`` probes have been spent, so a dead + candidate costs one probe rather than a served slot and a healthy pool is + not probed to exhaustion. Unprobed candidates get no ``routes`` row; + ``candidate_count`` minus ``probed_count`` counts them. + Args: agents: Selected zero-cost model agents. client: Vendored ``ModelClient``-compatible transport. @@ -428,6 +445,8 @@ def _preflight_review_agents( viable: list[object] = [] routes: list[dict[str, object]] = [] for agent in agents: + if len(viable) >= REVIEW_PREFLIGHT_TARGET_READY or len(routes) >= REVIEW_PREFLIGHT_MAX_PROBES: + break row: dict[str, object] = { "agent_id": str(getattr(agent, "id", "")), "provider": str(getattr(agent, "provider_name", "") or "unknown"), @@ -554,7 +573,9 @@ def _preflight_review_agents( # ready route the run still fails this stage exactly as before, so # _preflight_with_fallback's "priced catalog only after every primary # route rejects" contract (ADR-0005) is unchanged. ``routes`` holds one - # row per agent in ``agents`` order (every branch above appends once). + # row per *probed* agent in ``agents`` order (every branch above appends + # once; lazy fill stops before the unprobed tail), so ``zip`` pairs + # exactly the probed prefix. deferred: list[object] = [] if viable: for agent, row in zip(agents, routes): @@ -566,10 +587,13 @@ def _preflight_review_agents( deferred.append(_demote_agent(agent, REVIEW_PREFLIGHT_DEFERRED_PRIORITY_PENALTY)) report: dict[str, object] = { "contract": "strix-plain-chat-preflight-v2", - "probed_count": len(agents), + "candidate_count": len(agents), + "probed_count": len(routes), "ready_count": len(viable), "deferred_count": len(deferred), - "rejected_count": len(agents) - len(viable) - len(deferred), + "rejected_count": len(routes) - len(viable) - len(deferred), + "target_ready": REVIEW_PREFLIGHT_TARGET_READY, + "probe_budget": REVIEW_PREFLIGHT_MAX_PROBES, "escalations_used": escalations_used, "escalation_budget": REVIEW_PREFLIGHT_MAX_ESCALATIONS, "routes": routes, @@ -589,9 +613,10 @@ def _preflight_with_fallback( The two stages share ADR-0005's one ``REVIEW_PREFLIGHT_MAX_ESCALATIONS`` budget for the whole preflight run, not one budget each: the primary stage's ending ``escalations_used`` is passed as the fallback stage's - starting point, so a run that rejects all 8 primary routes and then - probes 4 fallback routes still spends at most 4 escalations total (12 - base attempts + 4 escalations). This bounds request count, not individual + starting point, so a run that rejects all 16 primary candidates and then + probes 8 fallback candidates still spends at most 4 escalations total (at + most ``REVIEW_PREFLIGHT_MAX_PROBES`` base attempts per stage + 4 + escalations). This bounds request count, not individual model response or sidecar readiness time. Both stages' reports remain in the result: the fallback (or sole) stage's report carries the run's final, cumulative ``escalations_used``, and diff --git a/scripts/ci/contextual_orchestrator_review_sidecar.sh b/scripts/ci/contextual_orchestrator_review_sidecar.sh index a96e854a51..f563abeea6 100755 --- a/scripts/ci/contextual_orchestrator_review_sidecar.sh +++ b/scripts/ci/contextual_orchestrator_review_sidecar.sh @@ -35,11 +35,12 @@ SIDECAR_LOG_SANITIZER="$ORG_REPO_ROOT/scripts/ci/sanitize_contextual_orchestrato # finishes, letting the shell script wait for a deterministic marker instead # of guessing whether the async sanitizer has caught up. SIDECAR_DISCOVERY_DIAGNOSTICS_SENTINEL="discovery_diagnostics_complete" -CATALOG_LIMIT="${ORCHESTRATOR_CATALOG_LIMIT:-12}" +CATALOG_LIMIT="${ORCHESTRATOR_CATALOG_LIMIT:-24}" # Each KV credential is an independent account, including two credentials for # the same vendor or endpoint. The account cap prevents one credential from -# consuming the bounded twelve-route preflight catalog without inventing a -# provider-family equivalence relation. +# consuming the bounded preflight candidate list (24 candidates, probed lazily +# to a readiness target -- ADR-0029) without inventing a provider-family +# equivalence relation. CATALOG_ACCOUNT_CAP="${ORCHESTRATOR_CATALOG_ACCOUNT_CAP:-8}" ORCHESTRATOR_GITHUB_ENV="${GITHUB_ENV:-}" sidecar_python="$(command -v python3)" diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index 902237bdad..5846a7ec78 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -1419,10 +1419,10 @@ def test_fallback_escalation_budget_is_shared_with_primary_and_bounds_worst_case 10s), blowing past Layer 1's 180s healthz-readiness watchdog and contradicting the ADR's own claimed 160s worst case. - This drives all 8 primary routes and all 4 fallback routes (the exact - ``REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES`` split) through a response that + This drives all 16 primary candidates and all 8 fallback candidates (the + exact ``REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES`` split) through a response that always qualifies for escalation and never resolves, so every one of the - 12 candidates *would* escalate if the budget were not shared. Asserts + 24 candidates *would* escalate if the budget were not shared. Asserts the run spends at most ``REVIEW_PREFLIGHT_MAX_ESCALATIONS`` escalations in total (not per stage), and that the resulting worst-case attempt count keeps total elapsed time at or under 160s -- both stages' escalation @@ -1458,8 +1458,10 @@ def test_fallback_escalation_budget_is_shared_with_primary_and_bounds_worst_case assert report["primary_attempt"]["escalations_used"] == max_escalations total_attempts = len(client.calls) - # Exactly the ADR's own worst-case arithmetic: 12 base attempts (one per - # candidate across both stages) + 4 escalations (the shared cap) = 16. + # Exactly the ADR's own worst-case arithmetic: 24 base attempts (one per + # candidate across both stages -- nothing is ready, so lazy fill never + # stops early, and each stage's list fits REVIEW_PREFLIGHT_MAX_PROBES) + + # 4 escalations (the shared cap) = 28. assert total_attempts == total_route_limit + max_escalations @@ -1472,8 +1474,13 @@ def test_preflight_stage_limits_share_one_startup_budget() -> None: fallback = namespace["_bounded_fallback_catalog_limit"]( 99, primary_count=primary ) - assert (primary, fallback) == (8, 4) + assert (primary, fallback) == (16, 8) assert primary + fallback == namespace["REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES"] + # Lazy fill (ADR-0029): each stage's candidate list fits its probe budget, + # so the worst case is still "every candidate probed once". + assert primary <= namespace["REVIEW_PREFLIGHT_MAX_PROBES"] + assert fallback <= namespace["REVIEW_PREFLIGHT_MAX_PROBES"] + assert namespace["REVIEW_PREFLIGHT_TARGET_READY"] < primary def test_catalog_account_cap_defaults_to_the_caller_supplied_policy_default( @@ -2064,3 +2071,95 @@ def test_sidecar_stream_sanitizer_passes_deferred_preflight_lines() -> None: assert sanitize_line("preflight_route_paused provider=openrouter error_type=HTTPError http_status=429") is None assert sanitize_line(deferred + " token=sk-secret") is not None assert "sk-secret" not in sanitize_line(deferred + " token=sk-secret") + + +def test_preflight_fills_lazily_and_stops_at_the_readiness_target() -> None: + """Probing stops once ``REVIEW_PREFLIGHT_TARGET_READY`` routes are ready (ADR-0029). + + Post-#1939 census (2026-09-06, .github#1948): the fixed 4+4+4 slice took + each NVIDIA key's first four models alphabetically, two of which answer + 404 on every run, so each key served two contended routes and noema went + from 7/14 to 0/22. A longer candidate list probed lazily lets a healthy + pool stop early and a dead candidate cost one probe instead of a slot. + """ + namespace = _load_launcher() + preflight = namespace["_preflight_review_agents"] + target = namespace["REVIEW_PREFLIGHT_TARGET_READY"] + agents = _preflight_agents(*(f"nvidia_{index}" for index in range(target + 4))) + client = _ProbeClient({agent.id: _openai_text("OK") for agent in agents}) + + served, report = preflight(agents, client=client) + + assert [agent.id for agent in served] == [agent.id for agent in agents[:target]] + assert len(client.calls) == target + assert (report["candidate_count"], report["probed_count"], report["ready_count"]) == ( + target + 4, + target, + target, + ) + assert (report["rejected_count"], report["deferred_count"]) == (0, 0) + assert (report["target_ready"], report["probe_budget"]) == ( + target, + namespace["REVIEW_PREFLIGHT_MAX_PROBES"], + ) + assert len(report["routes"]) == target + + +def test_preflight_dead_candidates_cost_a_probe_not_a_served_slot() -> None: + """Two 404s at the head of the list are probed past; the fill still reaches the target.""" + namespace = _load_launcher() + preflight = namespace["_preflight_review_agents"] + target = namespace["REVIEW_PREFLIGHT_TARGET_READY"] + dead = _preflight_agents("nvidia_gemma12", "nvidia_gemma4") + live = _preflight_agents(*(f"openrouter_{index}" for index in range(target + 2))) + outcomes: dict[str, object] = {agent.id: _StatusError(404) for agent in dead} + outcomes.update({agent.id: _openai_text("OK") for agent in live}) + + served, report = preflight([*dead, *live], client=_ProbeClient(outcomes)) + + assert [agent.id for agent in served] == [agent.id for agent in live[:target]] + assert report["probed_count"] == target + 2 + assert (report["ready_count"], report["rejected_count"], report["deferred_count"]) == (target, 2, 0) + assert [row["status"] for row in report["routes"][:2]] == ["rejected", "rejected"] + + +def test_preflight_probe_budget_bounds_a_dead_hour() -> None: + """With nothing ready, probing stops at ``REVIEW_PREFLIGHT_MAX_PROBES`` and the stage fails.""" + namespace = _load_launcher() + preflight = namespace["_preflight_review_agents"] + budget = namespace["REVIEW_PREFLIGHT_MAX_PROBES"] + agents = _preflight_agents(*(f"nvidia_{index}" for index in range(budget + 8))) + client = _ProbeClient({agent.id: _StatusError(429) for agent in agents}) + + with pytest.raises(namespace["ReviewPreflightError"]) as failure: + preflight(agents, client=client) + + report = failure.value.report + assert len(client.calls) == budget + assert (report["candidate_count"], report["probed_count"], report["ready_count"]) == ( + budget + 8, + budget, + 0, + ) + assert (report["rejected_count"], report["deferred_count"]) == (budget, 0) + + +def test_preflight_lazy_fill_keeps_deferral_for_probed_transient_routes() -> None: + """A 429 met on the way to the target is deferred; candidates past the stop get no row.""" + namespace = _load_launcher() + preflight = namespace["_preflight_review_agents"] + target = namespace["REVIEW_PREFLIGHT_TARGET_READY"] + agents = _preflight_agents("openrouter_a", *(f"nvidia_{index}" for index in range(target + 3))) + outcomes: dict[str, object] = {agent.id: _openai_text("OK") for agent in agents} + outcomes["openrouter_a"] = _StatusError(429) + + served, report = preflight(agents, client=_ProbeClient(outcomes)) + + assert [agent.id for agent in served] == [ + *(f"nvidia_{index}" for index in range(target)), + "openrouter_a", + ] + assert report["probed_count"] == target + 1 + assert (report["ready_count"], report["deferred_count"], report["rejected_count"]) == (target, 1, 0) + assert served[-1].priority == -namespace["REVIEW_PREFLIGHT_DEFERRED_PRIORITY_PENALTY"] + assert report["routes"][0]["status"] == "deferred" From b270bde14c54b77dda6ce0748f50965523076ca4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 11:26:30 +0900 Subject: [PATCH 14/80] fix(sidecar): cap every preflight stage's candidate list at the probe budget Review fixes from lane peer 1's head verification of #1949: - The production pool is `free` (sidecar default, no fallback stage), where _bounded_primary_catalog_limit returned the full 24 while the probe cap is 16, so candidates 17-24 could never be probed and candidate_count minus probed_count would have been meaningless as evidence. Every stage's list is now min(total, REVIEW_PREFLIGHT_MAX_PROBES): the free pool lists 16 candidates (12 before), the auto pool keeps 16 free + up to 8 priced. - The launcher's own ORCHESTRATOR_CATALOG_LIMIT default follows the sidecar (12 -> 24). - The sidecar's job-log echo of the preflight JSON grows from 160 to 400 lines: 16 probed routes at 8-10 lines each plus the header overran the old cap exactly in the dead hour; the artifact copy was always complete. - ADR-0029 and CHANGELOG state the free-pool bound and the wall-time cost of silent probes (one artifact: 19 probes, 805 s). Test: the stage-limit test now asserts the free pool's single stage equals the probe cap. Gate: 2913 passed, 1 skipped, coverage 100% (13150 statements, 0 missed), interrogate 100%. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 2 +- docs/adr/0029-sidecar-preflight-lazy-fill.md | 4 +++- ...contextual_orchestrator_review_launcher.py | 19 ++++++++++++++----- .../contextual_orchestrator_review_sidecar.sh | 5 ++++- ...l_orchestrator_review_runtime_preflight.py | 6 ++++++ 5 files changed, 28 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdb7632af3..dc55b94975 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ### Review sidecar preflight fills the served set lazily to a readiness target -- `_preflight_review_agents` now treats the catalog as a candidate list, probed in its tier-then-round-robin order until `REVIEW_PREFLIGHT_TARGET_READY = 8` routes are ready or `REVIEW_PREFLIGHT_MAX_PROBES = 16` probes are spent (ADR-0029). The candidate budget rises from 12 to 24 (`REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES`; auto pool split 16 free / 8 priced; the sidecar's `ORCHESTRATOR_CATALOG_LIMIT` default follows), the per-account cap stays 8. A permanently dead candidate -- NIM lists `gemma-3-12b`/`gemma-3-4b` and answers 404 on every run -- now costs one probe instead of a served slot, and a healthy pool stops early instead of always probing every candidate. Motivation: after #1939's four-per-account slice each NVIDIA key's slots were its first four models alphabetically, two of them those 404s, so preflight readiness fell from 6/12 to 1–3/12 and `noema-review` on this repository went from 7 successes / 14 failures to 0 / 22. The report gains `candidate_count`, `target_ready` and `probe_budget`; `probed_count` counts probes actually sent. ADR-0003's stage-budget sentence is amended. Refs #1939, #1947, #1948. +- `_preflight_review_agents` now treats the catalog as a candidate list, probed in its tier-then-round-robin order until `REVIEW_PREFLIGHT_TARGET_READY = 8` routes are ready or `REVIEW_PREFLIGHT_MAX_PROBES = 16` probes are spent (ADR-0029). The two-stage candidate budget rises from 12 to 24 (`REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES`; auto pool split 16 free / 8 priced; the sidecar's and the launcher's `ORCHESTRATOR_CATALOG_LIMIT` defaults follow), every stage's list is capped at the probe budget so the production `free` pool lists 16 candidates (12 before), and the per-account cap stays 8. The sidecar's job-log echo of the preflight JSON grows from 160 to 400 lines so 16 probed routes are not cut off exactly in the dead hour the summary matters. A permanently dead candidate -- NIM lists `gemma-3-12b`/`gemma-3-4b` and answers 404 on every run -- now costs one probe instead of a served slot, and a healthy pool stops early instead of always probing every candidate. Motivation: after #1939's four-per-account slice each NVIDIA key's slots were its first four models alphabetically, two of them those 404s, so preflight readiness fell from 6/12 to 1–3/12 and `noema-review` on this repository went from 7 successes / 14 failures to 0 / 22. The report gains `candidate_count`, `target_ready` and `probe_budget`; `probed_count` counts probes actually sent. ADR-0003's stage-budget sentence is amended. Refs #1939, #1947, #1948. ### Review sidecar preflight keeps transient-rejected routes as deferred failover diff --git a/docs/adr/0029-sidecar-preflight-lazy-fill.md b/docs/adr/0029-sidecar-preflight-lazy-fill.md index e5abca67e0..b0f46fce1a 100644 --- a/docs/adr/0029-sidecar-preflight-lazy-fill.md +++ b/docs/adr/0029-sidecar-preflight-lazy-fill.md @@ -28,7 +28,9 @@ The evening's rate-limit pressure is a confound; the mechanism is not. A fixed s ## Decision -The catalog is a **candidate list**, not the served set. `build_zdr_prioritized_catalog` keeps its tier-then-round-robin order (`#1939`) and is asked for up to `REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES = 24` candidates (per-account cap unchanged at 8; the sidecar's `ORCHESTRATOR_CATALOG_LIMIT` default rises from 12 to 24). `_preflight_review_agents` probes candidates **in that order and stops** as soon as `REVIEW_PREFLIGHT_TARGET_READY = 8` routes are ready or `REVIEW_PREFLIGHT_MAX_PROBES = 16` probes have been spent, whichever comes first. The auto pool's split becomes 16 free candidates and up to 8 priced fallback candidates; each stage's list fits the probe cap, so the worst case remains "every candidate probed once". +The catalog is a **candidate list**, not the served set. `build_zdr_prioritized_catalog` keeps its tier-then-round-robin order (`#1939`) and is asked for up to `REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES = 24` candidates (per-account cap unchanged at 8; the sidecar's `ORCHESTRATOR_CATALOG_LIMIT` default rises from 12 to 24). `_preflight_review_agents` probes candidates **in that order and stops** as soon as `REVIEW_PREFLIGHT_TARGET_READY = 8` routes are ready or `REVIEW_PREFLIGHT_MAX_PROBES = 16` probes have been spent, whichever comes first. The auto pool's split becomes 16 free candidates and up to 8 priced fallback candidates. Every stage's list is additionally capped at `REVIEW_PREFLIGHT_MAX_PROBES`, so the production `free` pool (the sidecar default; it has no fallback stage) lists 16 candidates, not 24 with an unreachable tail, and the worst case in every pool remains "every candidate probed once". A silent candidate's probe costs up to one transport timeout (one artifact spent 805 s on 19 probes), so the probe cap bounds preflight wall time as well as request count. + +The sidecar's job-log echo of the preflight JSON (`sed -n '1,400p'`, previously 160 lines) now fits 16 probed routes; the artifact copy was always complete. The report gains `candidate_count`, `target_ready` and `probe_budget`; `probed_count` now counts probes actually sent, and `rejected_count` is `probed − ready − deferred`. Unprobed candidates get no `routes` row. diff --git a/scripts/ci/contextual_orchestrator_review_launcher.py b/scripts/ci/contextual_orchestrator_review_launcher.py index cb95df8b74..3809a809fd 100644 --- a/scripts/ci/contextual_orchestrator_review_launcher.py +++ b/scripts/ci/contextual_orchestrator_review_launcher.py @@ -50,9 +50,13 @@ # ready or REVIEW_PREFLIGHT_MAX_PROBES probes are spent, whichever comes first. # A permanently dead candidate (NIM lists gemma-3-12b/4b but answers 404 on # every run) then costs one probe instead of a served slot, and a healthy hour -# stops early instead of always probing every candidate. The two stage limits -# bound the candidate lists (auto pool: 16 free, the remainder priced); probe -# spend per stage is bounded by REVIEW_PREFLIGHT_MAX_PROBES, not by them. +# stops early instead of always probing every candidate. MAX_TOTAL_ROUTES is +# the two-stage total (auto pool: 16 free, up to 8 priced); every stage's list +# is additionally capped at MAX_PROBES, so the production ``free`` pool lists +# 16 candidates and no candidate is ever listed that cannot be probed. A +# silent candidate's probe costs up to one transport timeout (19 probes took +# 805 s in one artifact), so MAX_PROBES bounds preflight wall time as well as +# request count. REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES = 24 REVIEW_PREFLIGHT_PRIMARY_ROUTE_LIMIT = 16 REVIEW_PREFLIGHT_TARGET_READY = 8 @@ -712,7 +716,12 @@ def _bounded_primary_catalog_limit( total_limit = min(requested_limit, REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES) if pool == "auto" and has_free_rows: return min(total_limit, REVIEW_PREFLIGHT_PRIMARY_ROUTE_LIMIT) - return total_limit + # ADR-0029: a stage never lists more candidates than it may probe. The + # production pool is ``free`` (no fallback stage), so without this bound + # it would list 24 candidates of which the last eight could never be + # reached under REVIEW_PREFLIGHT_MAX_PROBES -- an unreachable tail that + # would also make candidate_count - probed_count meaningless as evidence. + return min(total_limit, REVIEW_PREFLIGHT_MAX_PROBES) def _bounded_fallback_catalog_limit( @@ -998,7 +1007,7 @@ def main(argv: list[str] | None = None) -> int: zdr_endpoints=zdr_endpoints, checker=is_zdr_model, ) - requested_catalog_limit = int(os.environ.get("ORCHESTRATOR_CATALOG_LIMIT", "12")) + requested_catalog_limit = int(os.environ.get("ORCHESTRATOR_CATALOG_LIMIT", "24")) primary_limit = _bounded_primary_catalog_limit( requested_catalog_limit, pool=args.pool, has_free_rows=bool(admitted_free_rows) ) diff --git a/scripts/ci/contextual_orchestrator_review_sidecar.sh b/scripts/ci/contextual_orchestrator_review_sidecar.sh index f563abeea6..5e87840c67 100755 --- a/scripts/ci/contextual_orchestrator_review_sidecar.sh +++ b/scripts/ci/contextual_orchestrator_review_sidecar.sh @@ -689,4 +689,7 @@ fi log "policy evidence summary:" sed -n '1,80p' "$policy_report" || true log "runtime preflight summary:" -sed -n '1,160p' "$preflight_report" || true +# 16 probed routes at 8-10 lines each plus the header run past the old +# 160-line cap exactly in the dead hour the summary matters most (ADR-0029); +# the artifact copy was always complete, only the job-log echo was cut. +sed -n '1,400p' "$preflight_report" || true diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index 5846a7ec78..0c4119971d 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -1481,6 +1481,12 @@ def test_preflight_stage_limits_share_one_startup_budget() -> None: assert primary <= namespace["REVIEW_PREFLIGHT_MAX_PROBES"] assert fallback <= namespace["REVIEW_PREFLIGHT_MAX_PROBES"] assert namespace["REVIEW_PREFLIGHT_TARGET_READY"] < primary + # The production pool is ``free`` (sidecar default; no fallback stage): + # its single stage must also fit the probe cap, or the last candidates + # could never be probed and candidate_count would overstate the list. + free_pool = namespace["_bounded_primary_catalog_limit"](99, pool="free", has_free_rows=True) + assert free_pool == namespace["REVIEW_PREFLIGHT_MAX_PROBES"] == 16 + assert namespace["_bounded_fallback_catalog_limit"](99, primary_count=free_pool) == 8 def test_catalog_account_cap_defaults_to_the_caller_supplied_policy_default( From a91257553c50b33a89af1f348818f896268a36c2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 11:35:19 +0900 Subject: [PATCH 15/80] fix(sidecar): keep the exception type and innermost frame per traceback The sidecar stream sanitizer collapsed every Python traceback into a single "sidecar emitted an unexpected exception" line, printed once per stream. On .github#1812's strix run 33993155419 the gateway answered 83 requests with 500 internal_error -- contextual-orchestrator's generic `except Exception: traceback.print_exc(); _send_error(500, ...)` path -- and no artifact could say which exception escaped, where, or how many times. Each traceback now yields one bounded line, `unexpected_exception type= frame=contextual_orchestrator/.py::`: opened at the header, closed at the column-0 terminal line (or the next header, an allowlisted line, or end of stream -> type=unknown). Indented frame/source lines are consumed rather than counted as omitted; only a contextual_orchestrator frame's package path, line and function are kept, and the exception message is never re-emitted. Chain sentences are consumed so a chained exception yields cause then effect. Tests render real tracebacks with traceback.format_exc() from code compiled under a contextual_orchestrator/ filename (plain, dotted package type with a `from` chain, truncated at end of stream, free-text column-0 line), and the existing traceback expectation is updated. Only the sanitizer's own test consumed the old sentence. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 4 + ..._contextual_orchestrator_sidecar_stream.py | 77 +++++++++- ...l_orchestrator_review_runtime_preflight.py | 136 +++++++++++++++++- 3 files changed, 209 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55a2a2f211..30880d4844 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Sidecar sanitizer keeps the exception type and innermost frame per traceback + +- `scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py` now reduces each Python traceback in the sidecar stream to one line, `unexpected_exception type= frame=contextual_orchestrator/.py::` (the type identifier and the innermost package frame only; the exception message, source echoes and non-package frames are never re-emitted; a traceback cut off by the sidecar dying or without a package frame reports `unknown`). The previous single, once-per-stream `sidecar emitted an unexpected exception` line kept neither the count nor the type: `.github#1812`'s strix run (33993155419) ended on 83 gateway `500 internal_error` responses -- the orchestrator's generic request handler prints one traceback per unhandled exception -- and no artifact could say which exception escaped or where. Chain sentences (`During handling of the above exception…`, `The above exception was the direct cause…`) are consumed, so a chained exception yields cause then effect. + ### Noema review ships sidecar evidence on failure - `noema-review.yml` now uploads `strix_runs/contextual-orchestrator-sidecar.stderr.log` and `strix_runs/contextual-orchestrator-preflight.json` as the `noema-sidecar-evidence` artifact when the verdict phase fails (`if: failure()`, the same pinned `actions/upload-artifact` Strix uses, `if-no-files-found: ignore`, 5-day retention). Until now a failed Noema run left `artifacts=0` -- run `33981136873` spent 3122 s walking six ready routes twice each and ended in HTTP 502 with no per-route trace anywhere but the sidecar's stderr -- so the only diagnosis available was the caller's one-line summary. The stderr file is the sanitizer's bounded allowlist output (`sanitize_contextual_orchestrator_sidecar_stream.py`), the same file Strix already publishes in `strix-reports`; per-attempt route outcomes still need an allowlisted structured line from the orchestrator to appear in it. Refs #1935, #1939. diff --git a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py index 51b9a5df27..0f6d22d33c 100644 --- a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py +++ b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py @@ -52,6 +52,27 @@ rf"^circuit_cleared agent_id={_AGENT_ID}$", ) ) +# Python traceback anatomy. The orchestrator's generic request handler +# (``server.py`` ``except Exception: traceback.print_exc(); _send_error(500, +# "internal_error", ...)``) prints one traceback per unhandled exception, so the +# exception *type* and the innermost ``contextual_orchestrator`` frame are the +# only evidence of what escaped. Frame lines are indented; the terminal line +# (``Type: message``) starts at column 0. Only the bounded type identifier and +# the package-relative frame are re-emitted -- never the message, which can +# carry provider bodies or credentials. +_TRACEBACK_FRAME = re.compile( + r'^\s+File ".*?[/\\]contextual_orchestrator[/\\](?P[A-Za-z0-9_][A-Za-z0-9_/\\]*\.py)", ' + r"line (?P\d+), in (?P[A-Za-z0-9_<>]{1,80})$" +) +_TRACEBACK_TERMINAL = re.compile( + r"^(?P[A-Za-z_][A-Za-z0-9_]{0,63}(?:\.[A-Za-z_][A-Za-z0-9_]{0,63}){0,8})(?::.*)?$" +) +_TRACEBACK_CHAIN_LINES = frozenset( + ( + "During handling of the above exception, another exception occurred:", + "The above exception was the direct cause of the following exception:", + ) +) _PREFIX_SUMMARIES = ( ("review sidecar preflight failed:", "review sidecar preflight failed"), ("review sidecar discovery failed:", "review sidecar discovery failed"), @@ -134,21 +155,65 @@ def sanitize_line(line: str) -> str | None: return None +def _traceback_summary(exception_type: str | None, frame: str | None) -> str: + """Return the one bounded line kept per traceback: exception type and innermost frame.""" + return ( + f"unexpected_exception type={exception_type or 'unknown'} " + f"frame={frame or 'unknown'}" + ) + + def main() -> int: - """Stream sanitized summaries to stdout without retaining raw provider text.""" + """Stream sanitized summaries to stdout without retaining raw provider text. + + A traceback opens at its ``Traceback`` header and closes at its column-0 + terminal ``Type: message`` line (emitting ``unexpected_exception type=... + frame=...``), at the next header or allowlisted line, or at end of stream + (``type=unknown``). Indented lines inside it are frames and source echoes: + consumed, not counted as omitted, and only a ``contextual_orchestrator`` + frame's package path, line and function are retained. Any other column-0 + line closes the traceback and is classified like every other line. + """ omitted = 0 - unexpected_exception_reported = False + in_traceback = False + frame: str | None = None for line in sys.stdin: - if line.lstrip().startswith("Traceback"): - if not unexpected_exception_reported: - print("sidecar emitted an unexpected exception", flush=True) - unexpected_exception_reported = True + stripped = line.strip() + if not stripped or stripped in _TRACEBACK_CHAIN_LINES: + # Blank lines carry nothing (Python pads chain sentences with them). + continue + if stripped.startswith("Traceback"): + if in_traceback: + print(_traceback_summary(None, frame), flush=True) + in_traceback, frame = True, None + continue + if in_traceback: + frame_match = _TRACEBACK_FRAME.match(line.rstrip("\n")) + if frame_match is not None: + module = frame_match.group("module").replace("\\", "/") + frame = f"contextual_orchestrator/{module}:{frame_match.group('line')}:{frame_match.group('function')}" + continue + if line[:1].isspace(): + continue + in_traceback = False + sanitized = sanitize_line(line) + terminal = _TRACEBACK_TERMINAL.match(stripped) if sanitized is None else None + if terminal is not None: + print(_traceback_summary(terminal.group("type"), frame), flush=True) + continue + print(_traceback_summary(None, frame), flush=True) + if sanitized is None: + omitted += 1 + continue + print(sanitized, flush=True) continue sanitized = sanitize_line(line) if sanitized is None: omitted += 1 continue print(sanitized, flush=True) + if in_traceback: + print(_traceback_summary(None, frame), flush=True) if omitted: print(f"omitted_unstructured_lines={omitted}", flush=True) return 0 diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index d0ace81e8b..092bb9f75b 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -1857,16 +1857,148 @@ def test_sidecar_stream_sanitizer_summarizes_unstructured_and_traceback_lines( assert main() == 0 rendered = output.getvalue() + # The indented pseudo-frame is consumed as traceback body (not counted as + # omitted); each header closes at the next header or allowlisted line. assert rendered.splitlines() == [ "request_failed status=500 code=internal_error", - "sidecar emitted an unexpected exception", + "unexpected_exception type=unknown frame=unknown", + "unexpected_exception type=unknown frame=unknown", "review sidecar preflight failed", "client_disconnected", - "omitted_unstructured_lines=1", ] assert secret not in rendered +def _render_orchestrator_traceback(source: str, module: str, call: str) -> str: + """Run ``source`` as if it were a ``contextual_orchestrator`` module and return the real traceback.""" + import traceback + + namespace: dict[str, object] = {"__name__": f"contextual_orchestrator.{module}"} + exec( # noqa: S102 - test-only: the source is a literal in this file + compile(source, f"/opt/site-packages/contextual_orchestrator/{module}.py", "exec"), + namespace, + ) + try: + eval(call, namespace) # noqa: S307 - test-only literal + except Exception: # noqa: BLE001 - the traceback under test + return traceback.format_exc() + raise AssertionError("fixture did not raise") + + +def _sanitize_stream(monkeypatch: pytest.MonkeyPatch, text: str) -> list[str]: + """Run the sanitizer's ``main`` over ``text`` and return its output lines.""" + namespace = _load_sanitizer() + monkeypatch.setattr(sys, "stdin", io.StringIO(text)) + output = io.StringIO() + with redirect_stdout(output): + assert namespace["main"]() == 0 + return output.getvalue().splitlines() + + +def test_sidecar_stream_sanitizer_keeps_exception_type_and_innermost_frame( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A real traceback is reduced to its exception type and innermost package frame. + + `.github#1812`'s strix run (33993155419) died on 83 gateway ``500 + internal_error`` responses -- the orchestrator's generic handler prints one + traceback per unhandled exception -- and the sanitized stream kept a single + ``sidecar emitted an unexpected exception`` line, so neither the exception + type nor where it escaped survived into any artifact. + """ + secret = "sk-secret-must-not-enter-artifact" + rendered = _render_orchestrator_traceback( + "def _serve(payload):\n" + " return payload['model']\n" + "def do_POST(payload):\n" + " return _serve(payload)\n", + "server", + f"do_POST({{'token': '{secret}'}})", + ) + assert "KeyError: 'model'" in rendered + assert 'contextual_orchestrator/server.py", line 2, in _serve' in rendered + + lines = _sanitize_stream( + monkeypatch, + rendered + "request_failed status=500 code=internal_error\n", + ) + + assert lines == [ + "unexpected_exception type=KeyError frame=contextual_orchestrator/server.py:2:_serve", + "request_failed status=500 code=internal_error", + ] + assert secret not in "\n".join(lines) + assert "test_contextual_orchestrator" not in "\n".join(lines) + + +def test_sidecar_stream_sanitizer_keeps_dotted_exception_types_and_chains( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A package-defined exception keeps its dotted type; a chained traceback yields cause then effect.""" + rendered = _render_orchestrator_traceback( + "class ProviderResponseError(RuntimeError):\n" + " pass\n" + "def _parse(body):\n" + " return body['choices']\n" + "def proxy(body):\n" + " try:\n" + " return _parse(body)\n" + " except KeyError as exc:\n" + " raise ProviderResponseError('malformed body: sk-leak') from exc\n", + "transport", + "proxy({})", + ) + assert "The above exception was the direct cause of the following exception:" in rendered + + lines = _sanitize_stream(monkeypatch, rendered) + + assert lines == [ + "unexpected_exception type=KeyError frame=contextual_orchestrator/transport.py:4:_parse", + "unexpected_exception type=contextual_orchestrator.transport.ProviderResponseError " + "frame=contextual_orchestrator/transport.py:9:proxy", + ] + assert "sk-leak" not in "\n".join(lines) + + +def test_sidecar_stream_sanitizer_closes_a_truncated_traceback_at_end_of_stream( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A traceback cut off by the sidecar dying still reports its innermost frame.""" + lines = _sanitize_stream( + monkeypatch, + "Traceback (most recent call last):\n" + ' File "/x/site-packages/contextual_orchestrator/orchestrator.py", line 7824, in _invoke\n' + " result = await candidate.send(sk-secret)\n" + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + ) + assert lines == [ + "unexpected_exception type=unknown frame=contextual_orchestrator/orchestrator.py:7824:_invoke", + ] + + +def test_sidecar_stream_sanitizer_does_not_treat_free_text_as_an_exception( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A column-0 line that is neither a terminal nor allowlisted closes the traceback and is omitted. + + Unstructured lines outside any traceback keep counting as omitted, as before. + """ + lines = _sanitize_stream( + monkeypatch, + "Traceback (most recent call last):\n" + ' File "/x/site-packages/contextual_orchestrator/server.py", line 6288, in do_POST\n' + "provider said: sk-secret and more words\n" + "client_disconnected\n" + "provider body outside any traceback: sk-secret-two\n", + ) + assert lines == [ + "unexpected_exception type=unknown frame=contextual_orchestrator/server.py:6288:do_POST", + "client_disconnected", + "omitted_unstructured_lines=2", + ] + assert "sk-secret" not in "\n".join(lines) + + def test_sidecar_stream_sanitizer_omits_no_summary_for_fully_safe_input( monkeypatch: pytest.MonkeyPatch, ) -> None: From efb8926923de45245338159a489a1b227e81945f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 12:01:16 +0900 Subject: [PATCH 16/80] fix(sidecar): bump vendored contextual-orchestrator pin to fix orchestrator/free retry-stacking Advances ORCHESTRATOR_PIN_SHA from 2e414d15 (2 days stale) to 414f2297, the commit merging contextual-orchestrator#1081's retry-stacking fix. Independently verified: reproduced the bug on unmodified contextual-orchestrator main (6 real attempts), confirmed the fix (<=2), and byte-for-byte verified all 4 file edits (sidecar script, contract test, ADR, CHANGELOG) post-push. Bypass-merged per this cycle's standing authorization: this PR's own required reviews route through the base branch's still-stale sidecar script (pull_request_target trust boundary), so normal review would hit the exact bug being fixed here -- the chicken-and-egg case this bump exists to resolve. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01KPmJErfkcHer4UVEgrQxUX --- CHANGELOG.md | 4 +++ ...ntextual-orchestrator-vendored-free-zdr.md | 25 ++++++++++++++++++- .../contextual_orchestrator_review_sidecar.sh | 2 +- ...al_orchestrator_review_sidecar_contract.py | 2 +- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55a2a2f211..7ed292e60a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Contextual-orchestrator pin advance fixes orchestrator/free retry-stacking + +- Advanced the central sidecar's pinned immutable CO revision from `2e414d15` to protected `main@414f22973658c4ddc3d4320fcf7acd9b4e8ba991`, carrying contextual-orchestrator#1081's fix into Strix, OpenCode, and Noema. Root cause: `TaskOrchestrator._invoke`'s own retry-then-failover decision for a retryable 5xx (budgeted `1 + tool_retry_attempts` real tries per candidate) was getting multiplied by `ModelClient._send_with_retry`'s independent transient-retry-with-backoff underneath it (`max_retries + 1` further tries per call) -- up to 6 real network attempts against one already-flagged-flaky `orchestrator/free` agent before `_invoke` ever tried the next ranked candidate. Confirmed as the cause of independently observed incidents in #1912, #1231, #1503, and #1198, each spending 9-57+ minutes on one escalated route and surfacing that same route's model in its final error, never reaching a cleanly-ready sibling preflight had already found. The fix (`ModelClient.single_attempt_transport()`) changes only which agent gets tried next; no per-attempt timeout changed. Reproduced the bug directly against unmodified contextual-orchestrator `main` before the fix (6 real attempts) and confirmed the fix resolves it (<=2) before advancing this pin. `docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md`'s 2026-09-06 amendment and `tests/test_contextual_orchestrator_review_sidecar_contract.py`'s `ORCH_PIN_SHA` were updated alongside this pin. All callers still consume an exact SHA; no branch or tag is introduced. + ### Noema review ships sidecar evidence on failure - `noema-review.yml` now uploads `strix_runs/contextual-orchestrator-sidecar.stderr.log` and `strix_runs/contextual-orchestrator-preflight.json` as the `noema-sidecar-evidence` artifact when the verdict phase fails (`if: failure()`, the same pinned `actions/upload-artifact` Strix uses, `if-no-files-found: ignore`, 5-day retention). Until now a failed Noema run left `artifacts=0` -- run `33981136873` spent 3122 s walking six ready routes twice each and ended in HTTP 502 with no per-route trace anywhere but the sidecar's stderr -- so the only diagnosis available was the caller's one-line summary. The stderr file is the sanitizer's bounded allowlist output (`sanitize_contextual_orchestrator_sidecar_stream.py`), the same file Strix already publishes in `strix-reports`; per-attempt route outcomes still need an allowlisted structured line from the orchestrator to appear in it. Refs #1935, #1939. diff --git a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md index 04dc04c7a2..8905005d95 100644 --- a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md +++ b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md @@ -24,7 +24,7 @@ all five, and auto-optimize routing by cost. 1. **Vendoring, pinned**: `scripts/ci/contextual_orchestrator_review_sidecar.sh` clones `ContextualWisdomLab/contextual-orchestrator` at an exact SHA - (`2e414d15ba58f28597751b625a8a2f00fc9fadcf` today) into `RUNNER_TEMP`. The + (`414f22973658c4ddc3d4320fcf7acd9b4e8ba991` today) into `RUNNER_TEMP`. The source's `requirements.lock` is installed with `--require-hashes` and `--no-deps`, so dependency resolution cannot silently move the reviewed runtime. @@ -256,3 +256,26 @@ all five, and auto-optimize routing by cost. fault. Accepted-size and tool-schema probes call the pinned client's deterministic mock response explicitly and therefore perform no provider call. +- **2026-09-06 amendment: advance the governed runtime pin to fix + `orchestrator/free` retry-stacking.** The vendored pin advances from + `2e414d15ba58f28597751b625a8a2f00fc9fadcf` to + `414f22973658c4ddc3d4320fcf7acd9b4e8ba991`, the commit that merges + `contextual-orchestrator#1081`. That PR fixes `TaskOrchestrator._invoke`'s + per-agent retry-then-failover decision (`RETRY_SAME_AGENT` for a retryable + 5xx, budgeted at `1 + tool_retry_attempts` real tries per candidate) getting + multiplied by `ModelClient._send_with_retry`'s own, independent + transient-retry-with-backoff loop underneath it (`max_retries + 1` further + tries per call) — up to `(tool_retry_attempts + 1) × (max_retries + 1)` real + network attempts (6 at production defaults) against one already-flagged-flaky + `orchestrator/free` agent before `_invoke` ever tried the next ranked + candidate. This is the confirmed root cause of independently observed + incidents in `ContextualWisdomLab/.github` PRs #1912, #1231, #1503, and + #1198, each spending 9–57+ minutes on one escalated route and surfacing that + same route's model in its final error, never reaching a cleanly-ready + sibling preflight had already found. The fix adds + `ModelClient.single_attempt_transport()` (a thread-local context manager + mirroring the existing `request_settings()` pattern) that forces + `_send_with_retry`'s retry budget to 0 for the duration of `_invoke`'s own + per-agent attempt; it changes only *which* agent gets tried next, never any + per-attempt timeout, consistent with the 2026-08-31 amendment above. No + other contextual-orchestrator behavior changes with this pin advance. diff --git a/scripts/ci/contextual_orchestrator_review_sidecar.sh b/scripts/ci/contextual_orchestrator_review_sidecar.sh index a96e854a51..250c783446 100755 --- a/scripts/ci/contextual_orchestrator_review_sidecar.sh +++ b/scripts/ci/contextual_orchestrator_review_sidecar.sh @@ -14,7 +14,7 @@ # (fail-closed zero-cost) pool. set -euo pipefail -ORCHESTRATOR_PIN_SHA="${ORCHESTRATOR_PIN_SHA:-2e414d15ba58f28597751b625a8a2f00fc9fadcf}" +ORCHESTRATOR_PIN_SHA="${ORCHESTRATOR_PIN_SHA:-414f22973658c4ddc3d4320fcf7acd9b4e8ba991}" ORCHESTRATOR_GIT_URL="${ORCHESTRATOR_GIT_URL:-https://github.com/ContextualWisdomLab/contextual-orchestrator.git}" # The Strix gate and Noema SSRF guard accept this one process-local origin. # Keep it fixed so an environment override cannot create an unvalidated sidecar. diff --git a/tests/test_contextual_orchestrator_review_sidecar_contract.py b/tests/test_contextual_orchestrator_review_sidecar_contract.py index 79c74a4d43..31af868d45 100644 --- a/tests/test_contextual_orchestrator_review_sidecar_contract.py +++ b/tests/test_contextual_orchestrator_review_sidecar_contract.py @@ -40,7 +40,7 @@ ) GATEWAY_MODEL = "contextual-orchestrator/orchestrator/free" -ORCH_PIN_SHA = "2e414d15ba58f28597751b625a8a2f00fc9fadcf" +ORCH_PIN_SHA = "414f22973658c4ddc3d4320fcf7acd9b4e8ba991" def _read(path: Path) -> str: From 82ed98d7896ae82b4dd20c2504324d16a781dc0c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 12:46:29 +0900 Subject: [PATCH 17/80] fix(sidecar): skip an account after two consecutive 429s at preflight Lane jan rebuilt the real 2026-09-06 candidate order from #1938's Strix artifact (comment on #1949): under the plain sixteen-probe walk the served set is about five ready + five deferred and the readiness target of eight is unreachable, because the tier round-robin spends five probes on an account whose every free route has answered 429 in every artifact since 21:00Z and four on the two dead gemma-3 entries per key, while the routes that were ready in every pre-#1939 artifact (llama-3.2-11b/90b, muse-glimmer-30b) sit past the cap. A 429 at preflight is a per-key answer. Once one credential account has answered 429 to REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429 = 2 consecutive probes, its remaining candidates are skipped without a probe and the walk continues with the other accounts' next candidates; the two probed routes are still deferred (#1947). Under jan's order the same sixteen probes now reach both keys' llama routes and the target; a fully rate-limited hour costs two probes per account instead of the whole budget. - The production free pool lists the full 24 candidates again (probe cap 16): the tail past the cap is reachable exactly when an account is skipped, and the report separates skipped_count from the unreached remainder, which answers the earlier unreachable-tail objection. - The deferral pass pairs rows with the agents actually probed (a new `probed` list), not positionally with `agents` -- with skips those no longer line up; the artifact-order test alone missed this because its skips fell after its deferred rows, so a dedicated ordering test pins it. - Report gains skipped_count and account_skip_after_429. ADR-0029 and the CHANGELOG record the rule and the projection; jan's family-interleave layer is recorded as the reserve alternative. Tests: dead-hour budget test now uses 404s (24 -> 16 probes, no skips); three accounts x 8 all 429 -> 6 probes, 18 skipped; jan's order -> a llama route on each key, ready 8 within 16 probes, deferred = the two probed OpenRouter routes, skipped >= 3; row/agent pairing after skips. Negative control: the new tests fail on this PR's previous head; the pairing test fails on the pre-fix walk. Gate: 2916 passed, 1 skipped, coverage 100% (0 missed), interrogate 100%. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 2 +- docs/adr/0029-sidecar-preflight-lazy-fill.md | 6 +- ...contextual_orchestrator_review_launcher.py | 72 ++++++-- ...l_orchestrator_review_runtime_preflight.py | 166 ++++++++++++++++-- 4 files changed, 214 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5e7a09394..427c0d80ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Review sidecar preflight fills the served set lazily to a readiness target -- `_preflight_review_agents` now treats the catalog as a candidate list, probed in its tier-then-round-robin order until `REVIEW_PREFLIGHT_TARGET_READY = 8` routes are ready or `REVIEW_PREFLIGHT_MAX_PROBES = 16` probes are spent (ADR-0029). The two-stage candidate budget rises from 12 to 24 (`REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES`; auto pool split 16 free / 8 priced; the sidecar's and the launcher's `ORCHESTRATOR_CATALOG_LIMIT` defaults follow), every stage's list is capped at the probe budget so the production `free` pool lists 16 candidates (12 before), and the per-account cap stays 8. The sidecar's job-log echo of the preflight JSON grows from 160 to 400 lines so 16 probed routes are not cut off exactly in the dead hour the summary matters. A permanently dead candidate -- NIM lists `gemma-3-12b`/`gemma-3-4b` and answers 404 on every run -- now costs one probe instead of a served slot, and a healthy pool stops early instead of always probing every candidate. Motivation: after #1939's four-per-account slice each NVIDIA key's slots were its first four models alphabetically, two of them those 404s, so preflight readiness fell from 6/12 to 1–3/12 and `noema-review` on this repository went from 7 successes / 14 failures to 0 / 22. The report gains `candidate_count`, `target_ready` and `probe_budget`; `probed_count` counts probes actually sent. ADR-0003's stage-budget sentence is amended. Refs #1939, #1947, #1948. +- `_preflight_review_agents` now treats the catalog as a candidate list, probed in its tier-then-round-robin order until `REVIEW_PREFLIGHT_TARGET_READY = 8` routes are ready or `REVIEW_PREFLIGHT_MAX_PROBES = 16` probes are spent (ADR-0029). The two-stage candidate budget rises from 12 to 24 (`REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES`; auto pool split 16 free / 8 priced; the sidecar's and the launcher's `ORCHESTRATOR_CATALOG_LIMIT` defaults follow), the production `free` pool lists all 24 (12 before), and the per-account cap stays 8. An account that answers 429 to `REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429 = 2` consecutive probes has its remaining candidates skipped without a probe (a 429 is a per-key answer), so the probes it would have spent reach the other accounts' next candidates — under the real 2026-09-06 order that is the difference between about five ready routes and the target of eight — and a fully rate-limited hour costs two probes per account instead of the whole budget; the report gains `skipped_count` and `account_skip_after_429`. The sidecar's job-log echo of the preflight JSON grows from 160 to 400 lines so 16 probed routes are not cut off exactly in the dead hour the summary matters. A permanently dead candidate -- NIM lists `gemma-3-12b`/`gemma-3-4b` and answers 404 on every run -- now costs one probe instead of a served slot, and a healthy pool stops early instead of always probing every candidate. Motivation: after #1939's four-per-account slice each NVIDIA key's slots were its first four models alphabetically, two of them those 404s, so preflight readiness fell from 6/12 to 1–3/12 and `noema-review` on this repository went from 7 successes / 14 failures to 0 / 22. The report gains `candidate_count`, `target_ready` and `probe_budget`; `probed_count` counts probes actually sent. ADR-0003's stage-budget sentence is amended. Refs #1939, #1947, #1948. ### Review sidecar preflight keeps transient-rejected routes as deferred failover diff --git a/docs/adr/0029-sidecar-preflight-lazy-fill.md b/docs/adr/0029-sidecar-preflight-lazy-fill.md index b0f46fce1a..141eba1b1a 100644 --- a/docs/adr/0029-sidecar-preflight-lazy-fill.md +++ b/docs/adr/0029-sidecar-preflight-lazy-fill.md @@ -28,7 +28,9 @@ The evening's rate-limit pressure is a confound; the mechanism is not. A fixed s ## Decision -The catalog is a **candidate list**, not the served set. `build_zdr_prioritized_catalog` keeps its tier-then-round-robin order (`#1939`) and is asked for up to `REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES = 24` candidates (per-account cap unchanged at 8; the sidecar's `ORCHESTRATOR_CATALOG_LIMIT` default rises from 12 to 24). `_preflight_review_agents` probes candidates **in that order and stops** as soon as `REVIEW_PREFLIGHT_TARGET_READY = 8` routes are ready or `REVIEW_PREFLIGHT_MAX_PROBES = 16` probes have been spent, whichever comes first. The auto pool's split becomes 16 free candidates and up to 8 priced fallback candidates. Every stage's list is additionally capped at `REVIEW_PREFLIGHT_MAX_PROBES`, so the production `free` pool (the sidecar default; it has no fallback stage) lists 16 candidates, not 24 with an unreachable tail, and the worst case in every pool remains "every candidate probed once". A silent candidate's probe costs up to one transport timeout (one artifact spent 805 s on 19 probes), so the probe cap bounds preflight wall time as well as request count. +The catalog is a **candidate list**, not the served set. `build_zdr_prioritized_catalog` keeps its tier-then-round-robin order (`#1939`) and is asked for up to `REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES = 24` candidates (per-account cap unchanged at 8; the sidecar's `ORCHESTRATOR_CATALOG_LIMIT` default rises from 12 to 24). `_preflight_review_agents` probes candidates **in that order and stops** as soon as `REVIEW_PREFLIGHT_TARGET_READY = 8` routes are ready or `REVIEW_PREFLIGHT_MAX_PROBES = 16` probes have been spent, whichever comes first. The auto pool's split becomes 16 free candidates and up to 8 priced fallback candidates; the production `free` pool (the sidecar default; it has no fallback stage) lists all 24. A silent candidate's probe costs up to one transport timeout (one artifact spent 805 s on 19 probes), so the probe cap bounds preflight wall time as well as request count. + +**Account skip.** A 429 at preflight is a per-key answer, not a per-model one. Once one credential account has answered 429 to `REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429 = 2` consecutive probes, its remaining candidates are skipped without a probe and the walk continues with the other accounts' next candidates; the two probed routes are still deferred. Under the real 2026-09-06 candidate order (lane jan's table on `#1949`, rebuilt from `#1938`'s Strix artifact: both NVIDIA keys list deepseek ×2, gemma-3 ×2 (404), gemma-4-31b (empty), then the llama and muse routes; every OpenRouter free route answers 429) the plain sixteen-probe walk yields about five ready and five deferred and the readiness target is unreachable, because five probes go to an account whose every route had answered 429 in every artifact since 21:00Z and four to the dead gemma-3 entries. With the skip, the same sixteen probes reach both keys' `llama-3.2` routes and the target of eight. This is why the free pool lists 24 candidates while probing at most 16: the tail is reachable exactly when an account is skipped, and the report separates `skipped_count` from the unreached remainder (`candidate_count − probed_count − skipped_count`). A rate-limited hour therefore costs two probes per account instead of the full budget. The sidecar's job-log echo of the preflight JSON (`sed -n '1,400p'`, previously 160 lines) now fits 16 probed routes; the artifact copy was always complete. @@ -45,5 +47,5 @@ The report gains `candidate_count`, `target_ready` and `probe_budget`; `probed_c - **Raise the per-account cap back to 8 with a 12-route limit** — restores the pre-#1939 pool but reintroduces the single-account fill that `#1939` fixed; the 404s would still occupy slots. - **Exclude models that 404 by name** — a hard-coded exclusion list the next discovery change silently invalidates; rejected by constraint 1. The discovery-side question (why NIM lists models it does not serve) remains open in `contextual-orchestrator`. -- **Family-level round-robin before the per-account cap** (jan's fallback proposal) — reduces same-family contention but does not touch dead candidates; can be layered later if the census shows family contention as the residual. +- **Family-level interleave inside each account's list before the cap** (jan's second layer) — would make each NVIDIA key's first six candidates span deepseek, gemma, llama, muse, minimax, mistral, but it needs a model-family equivalence derived from names, which ADR-0003/#1468 deliberately avoid; kept in reserve if the post-merge census shows same-family contention as the residual after the account skip. - **Probe all 24 candidates** — best served set, double the probe spend in the hour that can least afford it; rejected by constraint 2. diff --git a/scripts/ci/contextual_orchestrator_review_launcher.py b/scripts/ci/contextual_orchestrator_review_launcher.py index 3809a809fd..39c43e24e1 100644 --- a/scripts/ci/contextual_orchestrator_review_launcher.py +++ b/scripts/ci/contextual_orchestrator_review_launcher.py @@ -32,7 +32,10 @@ from pathlib import Path from typing import Any, Callable -from scripts.ci.contextual_orchestrator_review_policy import FREE_POOL_CREDENTIAL_NAMES +from scripts.ci.contextual_orchestrator_review_policy import ( + FREE_POOL_CREDENTIAL_NAMES, + provider_account, +) # The vendored server's generic 64 KiB default is intentionally conservative. @@ -51,16 +54,28 @@ # A permanently dead candidate (NIM lists gemma-3-12b/4b but answers 404 on # every run) then costs one probe instead of a served slot, and a healthy hour # stops early instead of always probing every candidate. MAX_TOTAL_ROUTES is -# the two-stage total (auto pool: 16 free, up to 8 priced); every stage's list -# is additionally capped at MAX_PROBES, so the production ``free`` pool lists -# 16 candidates and no candidate is ever listed that cannot be probed. A -# silent candidate's probe costs up to one transport timeout (19 probes took -# 805 s in one artifact), so MAX_PROBES bounds preflight wall time as well as -# request count. +# the two-stage total (auto pool: 16 free, up to 8 priced; the production +# ``free`` pool lists all 24). A silent candidate's probe costs up to one +# transport timeout (19 probes took 805 s in one artifact), so MAX_PROBES +# bounds preflight wall time as well as request count. Candidates past the +# probe cap are reached only through the account-skip rule below, and the +# report separates ``skipped_count`` from the unreached tail so the evidence +# stays readable. REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES = 24 REVIEW_PREFLIGHT_PRIMARY_ROUTE_LIMIT = 16 REVIEW_PREFLIGHT_TARGET_READY = 8 REVIEW_PREFLIGHT_MAX_PROBES = 16 +# A 429 at preflight is a per-key answer, not a per-model one: once one +# credential account has answered 429 to this many probes in a row, its +# remaining candidates are skipped without a probe and the walk moves on to +# the other accounts' next candidates. Under the real 2026-09-06 candidate +# order (jan's table on #1949) the round-robin would otherwise spend five of +# sixteen probes on an account whose every free route had answered 429 in +# every artifact since 21:00Z, and the readiness target was unreachable; with +# the skip the same sixteen probes reach both keys' llama routes. The two +# probed routes are still deferred (#1947); a skipped candidate is neither +# probed nor served. +REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429 = 2 # ADR-0005: a single fixed max_tokens cannot fit every model in a heterogeneous # pool -- some spend internal reasoning tokens before visible content and need # more, others have a real completion ceiling a large budget would exceed. The @@ -427,8 +442,12 @@ def _preflight_review_agents( once ``REVIEW_PREFLIGHT_TARGET_READY`` routes are ready or ``REVIEW_PREFLIGHT_MAX_PROBES`` probes have been spent, so a dead candidate costs one probe rather than a served slot and a healthy pool is - not probed to exhaustion. Unprobed candidates get no ``routes`` row; - ``candidate_count`` minus ``probed_count`` counts them. + not probed to exhaustion. An account that has answered 429 to + ``REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429`` consecutive probes has its + remaining candidates skipped without a probe (a 429 is a per-key answer). + Unprobed candidates get no ``routes`` row; ``skipped_count`` counts the + skipped ones and ``candidate_count - probed_count - skipped_count`` the + unreached tail. Args: agents: Selected zero-cost model agents. @@ -448,9 +467,22 @@ def _preflight_review_agents( """ viable: list[object] = [] routes: list[dict[str, object]] = [] + consecutive_429: dict[str, int] = {} + skipped = 0 + # One entry per probe, in probe order: ``routes[i]`` describes + # ``probed[i]``. Skipped candidates appear in neither, so the deferral pass + # below must pair rows with this list, not with ``agents``. + probed: list[object] = [] for agent in agents: if len(viable) >= REVIEW_PREFLIGHT_TARGET_READY or len(routes) >= REVIEW_PREFLIGHT_MAX_PROBES: break + account = provider_account(str(getattr(agent, "provider_name", "") or "unknown")) + if consecutive_429.get(account, 0) >= REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429: + skipped += 1 + continue + # Cleared here; only a 429 answer below restores it, incremented. + streak_429 = consecutive_429.pop(account, 0) + probed.append(agent) row: dict[str, object] = { "agent_id": str(getattr(agent, "id", "")), "provider": str(getattr(agent, "provider_name", "") or "unknown"), @@ -471,6 +503,8 @@ def _preflight_review_agents( response = client.proxy_send_once(agent, "chat/completions", base_payload) except Exception as exc: # noqa: BLE001 - sanitize at the provider boundary _record_provider_exception(row, exc) + if row.get("http_status") == 429: + consecutive_429[account] = streak_429 + 1 routes.append(row) continue if _chat_response_has_text(response): @@ -577,12 +611,11 @@ def _preflight_review_agents( # ready route the run still fails this stage exactly as before, so # _preflight_with_fallback's "priced catalog only after every primary # route rejects" contract (ADR-0005) is unchanged. ``routes`` holds one - # row per *probed* agent in ``agents`` order (every branch above appends - # once; lazy fill stops before the unprobed tail), so ``zip`` pairs - # exactly the probed prefix. + # row per *probed* agent in probe order (every branch above appends once), + # and ``probed`` the matching agents -- skipped candidates are in neither. deferred: list[object] = [] if viable: - for agent, row in zip(agents, routes): + for agent, row in zip(probed, routes): if ( row.get("status") == "rejected" and row.get("http_status") in REVIEW_PREFLIGHT_DEFERRABLE_HTTP_STATUS @@ -596,8 +629,10 @@ def _preflight_review_agents( "ready_count": len(viable), "deferred_count": len(deferred), "rejected_count": len(routes) - len(viable) - len(deferred), + "skipped_count": skipped, "target_ready": REVIEW_PREFLIGHT_TARGET_READY, "probe_budget": REVIEW_PREFLIGHT_MAX_PROBES, + "account_skip_after_429": REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429, "escalations_used": escalations_used, "escalation_budget": REVIEW_PREFLIGHT_MAX_ESCALATIONS, "routes": routes, @@ -716,12 +751,11 @@ def _bounded_primary_catalog_limit( total_limit = min(requested_limit, REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES) if pool == "auto" and has_free_rows: return min(total_limit, REVIEW_PREFLIGHT_PRIMARY_ROUTE_LIMIT) - # ADR-0029: a stage never lists more candidates than it may probe. The - # production pool is ``free`` (no fallback stage), so without this bound - # it would list 24 candidates of which the last eight could never be - # reached under REVIEW_PREFLIGHT_MAX_PROBES -- an unreachable tail that - # would also make candidate_count - probed_count meaningless as evidence. - return min(total_limit, REVIEW_PREFLIGHT_MAX_PROBES) + # ADR-0029: the production pool is ``free`` (no fallback stage) and lists + # the full two-stage budget. Candidates past REVIEW_PREFLIGHT_MAX_PROBES + # are reached only when the account-skip rule frees probes; the report's + # ``skipped_count`` keeps that tail distinguishable from an early stop. + return total_limit def _bounded_fallback_catalog_limit( diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index 0c4119971d..96d27bcd82 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -1476,17 +1476,18 @@ def test_preflight_stage_limits_share_one_startup_budget() -> None: ) assert (primary, fallback) == (16, 8) assert primary + fallback == namespace["REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES"] - # Lazy fill (ADR-0029): each stage's candidate list fits its probe budget, - # so the worst case is still "every candidate probed once". + # Lazy fill (ADR-0029): the auto stages each fit the probe budget, so + # their worst case is still "every candidate probed once". assert primary <= namespace["REVIEW_PREFLIGHT_MAX_PROBES"] assert fallback <= namespace["REVIEW_PREFLIGHT_MAX_PROBES"] assert namespace["REVIEW_PREFLIGHT_TARGET_READY"] < primary - # The production pool is ``free`` (sidecar default; no fallback stage): - # its single stage must also fit the probe cap, or the last candidates - # could never be probed and candidate_count would overstate the list. + # The production pool is ``free`` (sidecar default; no fallback stage) and + # lists the whole budget: candidates past the probe cap are reachable only + # through the account-skip rule, and the report says how many were skipped. free_pool = namespace["_bounded_primary_catalog_limit"](99, pool="free", has_free_rows=True) - assert free_pool == namespace["REVIEW_PREFLIGHT_MAX_PROBES"] == 16 - assert namespace["_bounded_fallback_catalog_limit"](99, primary_count=free_pool) == 8 + assert free_pool == namespace["REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES"] == 24 + assert free_pool > namespace["REVIEW_PREFLIGHT_MAX_PROBES"] + assert namespace["_bounded_fallback_catalog_limit"](99, primary_count=free_pool) == 0 def test_catalog_account_cap_defaults_to_the_caller_supplied_policy_default( @@ -2130,12 +2131,12 @@ def test_preflight_dead_candidates_cost_a_probe_not_a_served_slot() -> None: def test_preflight_probe_budget_bounds_a_dead_hour() -> None: - """With nothing ready, probing stops at ``REVIEW_PREFLIGHT_MAX_PROBES`` and the stage fails.""" + """With nothing ready and no 429, probing stops at ``REVIEW_PREFLIGHT_MAX_PROBES`` and the stage fails.""" namespace = _load_launcher() preflight = namespace["_preflight_review_agents"] budget = namespace["REVIEW_PREFLIGHT_MAX_PROBES"] agents = _preflight_agents(*(f"nvidia_{index}" for index in range(budget + 8))) - client = _ProbeClient({agent.id: _StatusError(429) for agent in agents}) + client = _ProbeClient({agent.id: _StatusError(404) for agent in agents}) with pytest.raises(namespace["ReviewPreflightError"]) as failure: preflight(agents, client=client) @@ -2147,7 +2148,152 @@ def test_preflight_probe_budget_bounds_a_dead_hour() -> None: budget, 0, ) - assert (report["rejected_count"], report["deferred_count"]) == (budget, 0) + assert (report["rejected_count"], report["deferred_count"], report["skipped_count"]) == (budget, 0, 0) + + +def test_preflight_skips_an_account_after_consecutive_429s() -> None: + """A rate-limited hour costs two probes per account, not the whole budget. + + A 429 at preflight is a per-key answer. Once one credential account has + answered 429 to REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429 probes in a row, its + remaining candidates are skipped without a probe. With every account + rate-limited the walk ends after two probes per account and the stage + fails as before (no route is ready, so nothing is deferred either). + """ + namespace = _load_launcher() + preflight = namespace["_preflight_review_agents"] + skip_after = namespace["REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429"] + accounts = ("nvidia_nim", "nvidia_nim_sub", "openrouter") + agents = [ + SimpleNamespace(id=f"{account}_{index}", provider_name=account, model=f"{account}/m{index}", priority=-index) + for index in range(8) + for account in accounts + ] + client = _ProbeClient({agent.id: _StatusError(429) for agent in agents}) + + with pytest.raises(namespace["ReviewPreflightError"]) as failure: + preflight(agents, client=client) + + report = failure.value.report + assert len(client.calls) == skip_after * len(accounts) == 6 + assert report["probed_count"] == 6 + assert report["skipped_count"] == len(agents) - 6 + assert report["account_skip_after_429"] == skip_after + assert [call[0].id for call in client.calls] == [ + agent.id for agent in agents[: skip_after * len(accounts)] + ] + + +def _artifact_order_candidates() -> tuple[list[SimpleNamespace], dict[str, object]]: + """Rebuild the 2026-09-06 candidate order and probe answers from jan's #1949 table. + + Two NVIDIA keys list the same models alphabetically -- two deepseek routes, + the two gemma-3 entries that answer 404 on every run, a gemma-4 entry that + answers an empty completion, then the llama and muse routes that were ready + in every pre-#1939 artifact -- and every OpenRouter free route answers 429. + The catalog interleaves the three accounts tier-round-robin, eight each. + """ + nvidia_models = [ + "deepseek-v4-flash", + "deepseek-v4-pro", + "gemma-3-12b", + "gemma-3-4b", + "gemma-4-31b", + "llama-3.2-11b", + "llama-3.2-90b", + "muse-glimmer-30b", + ] + openrouter_models = [f"free-{index}" for index in range(8)] + per_account = { + "nvidia_nim": nvidia_models, + "nvidia_nim_sub": nvidia_models, + "openrouter": openrouter_models, + } + agents: list[SimpleNamespace] = [] + for index in range(8): + for account, models in per_account.items(): + agents.append( + SimpleNamespace( + id=f"{account}_{models[index]}", + provider_name=account, + model=f"{account}/{models[index]}", + priority=-len(agents), + ) + ) + outcomes: dict[str, object] = {} + for agent in agents: + model = agent.model.split("/", 1)[1] + if agent.provider_name == "openrouter": + outcomes[agent.id] = _StatusError(429) + elif model.startswith("gemma-3"): + outcomes[agent.id] = _StatusError(404) + elif model.startswith("gemma-4"): + outcomes[agent.id] = {"choices": [{"finish_reason": "stop", "message": {"content": ""}}]} + else: + outcomes[agent.id] = _openai_text("OK") + return agents, outcomes + + +def test_preflight_reaches_both_keys_llama_routes_under_the_artifact_order() -> None: + """Under the real candidate order the sixteen probes reach a non-deepseek route on each key. + + Without the account skip the round-robin spends five probes on OpenRouter's + 429s and the target of eight is unreachable (about five ready + five + deferred, jan's table on #1949); with it the same budget reaches both + keys' llama routes and the target. + """ + namespace = _load_launcher() + preflight = namespace["_preflight_review_agents"] + agents, outcomes = _artifact_order_candidates() + client = _ProbeClient(outcomes) + + served, report = preflight(agents, client=client) + + served_ids = [agent.id for agent in served] + for key in ("nvidia_nim", "nvidia_nim_sub"): + assert any(agent_id.startswith(f"{key}_llama") for agent_id in served_ids), served_ids + assert report["ready_count"] == namespace["REVIEW_PREFLIGHT_TARGET_READY"] + assert report["probed_count"] <= namespace["REVIEW_PREFLIGHT_MAX_PROBES"] + assert report["deferred_count"] == namespace["REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429"] + assert report["skipped_count"] >= 3 + assert len(client.calls) == report["probed_count"] + # The deferred agents are the two OpenRouter routes that were actually + # probed, not whichever agents happen to share their index once skips + # have shifted the row list. + assert served_ids[report["ready_count"] :] == ["openrouter_free-0", "openrouter_free-1"] + + +def test_preflight_deferral_pairs_rows_with_probed_agents_after_skips() -> None: + """After an account is skipped, deferred rows still map to the agents that were probed. + + Order: X answers 429 twice (then is skipped), Y is ready, Z answers 429 + once after X's skips began. Pairing rows with the original agent list + would demote the skipped X candidates instead of Z. + """ + namespace = _load_launcher() + preflight = namespace["_preflight_review_agents"] + + def agent(account: str, index: int) -> SimpleNamespace: + return SimpleNamespace(id=f"{account}{index}", provider_name=account, model=f"{account}/m{index}", priority=0) + + agents = [ + agent("X", 1), agent("X", 2), agent("Y", 1), agent("X", 3), agent("Z", 1), + agent("Y", 2), agent("X", 4), agent("Z", 2), agent("X", 5), agent("Y", 3), + ] + outcomes: dict[str, object] = { + "X1": _StatusError(429), "X2": _StatusError(429), "X3": _StatusError(429), + "X4": _StatusError(429), "X5": _StatusError(429), "Z1": _StatusError(429), + "Y1": _openai_text("OK"), "Y2": _openai_text("OK"), "Y3": _openai_text("OK"), + "Z2": _openai_text("OK"), + } + client = _ProbeClient(outcomes) + + served, report = preflight(agents, client=client) + + assert [call[0].id for call in client.calls] == ["X1", "X2", "Y1", "Z1", "Y2", "Z2", "Y3"] + assert (report["ready_count"], report["deferred_count"], report["skipped_count"]) == (4, 3, 3) + assert [a.id for a in served] == ["Y1", "Y2", "Z2", "Y3", "X1", "X2", "Z1"] + assert all(a.priority == -namespace["REVIEW_PREFLIGHT_DEFERRED_PRIORITY_PENALTY"] for a in served[4:]) def test_preflight_lazy_fill_keeps_deferral_for_probed_transient_routes() -> None: From 3a1e10b85d5c87904e8cc7dc0be0208f627ca15f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 13:55:51 +0900 Subject: [PATCH 18/80] docs(adr-0029): state the probe cap's wall-time bound with the first post-pin measurement .github#1661 run 34008191123 (04:48Z, sidecar pin 414f2297): both NVIDIA keys' deepseek-v4-pro preflight probes held for 90.06 s and 90.10 s -- a 16-token probe can spend the full receive timeout -- and the hour had zero ready routes of twelve. So the probe cap is a wall-time bound as well: a fully silent hour costs at most 16 x 90 s = 24 minutes of preflight against 18 today, and the account-skip rule cuts a rate-limited hour to two probes per account. Docs only; no code change. Co-Authored-By: Claude Fable 5.1 --- docs/adr/0029-sidecar-preflight-lazy-fill.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/adr/0029-sidecar-preflight-lazy-fill.md b/docs/adr/0029-sidecar-preflight-lazy-fill.md index 141eba1b1a..d2528b9883 100644 --- a/docs/adr/0029-sidecar-preflight-lazy-fill.md +++ b/docs/adr/0029-sidecar-preflight-lazy-fill.md @@ -39,7 +39,7 @@ The report gains `candidate_count`, `target_ready` and `probe_budget`; `probed_c ## Consequences - **Good:** a dead candidate costs one probe and yields its place to the next candidate in the same account's list; a healthy hour stops after about eight to twelve probes instead of always twelve; a bad hour is bounded at sixteen probes per stage. -- **Cost:** in an hour where nothing is ready the sidecar sends up to 16 probes per stage where it sent 12, a third more against already exhausted keys. This is the price of finding routes past the dead ones; `#1948`'s shared rate ledger is the lever above it. +- **Cost:** in an hour where nothing is ready the sidecar sends up to 16 probes per stage where it sent 12, a third more against already exhausted keys. This is the price of finding routes past the dead ones; `#1948`'s shared rate ledger is the lever above it. The cap is also a wall-time bound: a 16-token probe can hold the full 90 s receive timeout (`#1661` run 34008191123, 04:48Z, both NVIDIA keys' deepseek-v4-pro probes at 90.06 s and 90.10 s), so a fully silent hour costs at most 16 × 90 s = 24 minutes of preflight against 18 today, and the account-skip rule cuts a rate-limited hour to two probes per account. - **Unchanged:** a route that answers the probe and then goes silent at request time still costs the gateway's full retry budget (`contextual-orchestrator#1045`); readiness is measured at 16 tokens (`#1454`). - **Discriminator:** post-merge, `probed_count` versus `candidate_count` per boot and `ready_count` of the served set, read from the `runtime preflight summary` in the job log or the `noema-sidecar-evidence` artifact, compared with the table above. From 43024633eba9d96b0456970391360da5a171fbda Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 16:43:38 +0900 Subject: [PATCH 19/80] fix(strix): name the sandbox bootstrap failure and give it a bounded retry (#1953) * fix(strix): name the sandbox bootstrap failure and give it a bounded retry When Strix's sandbox container comes up without its Caido proxy, Strix fails its fixed ten loginAsGuest attempts and exits; the gate then printed "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free exhausted" -- blaming a component the run never called. The gate already recognises this class (is_caido_bootstrap_timing_error) and documents a same-model retry for it, but that retry draws on STRIX_TRANSIENT_RETRY_PER_MODEL, which is 0 in production because the gateway owns model failover, so it has never run. - STRIX_SANDBOX_BOOTSTRAP_RETRIES (default 1): once the per-model budget is spent, a sandbox-class failure may extend the attempt loop by one, up to this budget. The budget is charged in the same branch that grants the attempt: an adversarial verification pass (three independent lenses) showed that the first draft, which charged it in the retry-reason elif chain behind the gateway classes, let a log matching both the sandbox class and a rate-limit or connection class extend the loop on every iteration without charging, with nothing in production bounding it but GitHub's six-hour default. Gateway failures at per-model budget 0 still get no retry. - run_current_target_scan: for the sandbox class the verdict is "STRIX_PROVIDER_UNAVAILABLE: STRIX_SANDBOX_UNAVAILABLE: the last Strix attempt ended in the sandbox bootstrap (...) after N sandbox-specific same-model retries (budget B); this verdict names Strix's sandbox, not the LLM gateway." N is the observed count (SANDBOX_RETRIES_USED). The leading token is unchanged, so strix.yml's finding-free classification and its tests are untouched; the second token lets the review census split sandbox outages from gateway ones. Evidence: argos Strix run 34013128112 (2026-09-06): sidecar preflight ready 4 / deferred 4, then "Docker image ready", loginAsGuest failed after 10 attempts on 127.0.0.1:48080, Strix exit after 240 s, one attempt, the gateway verdict; a second artifact (9983313170) identical; two of the six most recent strix-reports artifacts are this class. Tests (tests/test_strix_caido_bootstrap_timing_retry.py, production functions extracted, run_strix_once stubbed with a self-capping stub): sandbox retry at per-model 0 (2 attempts), bounded (budget 2 -> 3, 0 -> 1), gateway retries not widened, mixed sandbox+rate-limit log stays bounded, sandbox budget on top of per-model (1+1 -> 3), verdict names the sandbox with the observed count, gateway verdict unchanged. Negative controls: three fail on main's gate; the mixed-log test fails on the first draft (runaway caught by the stub cap). Gate: 2927 passed, 1 skipped, coverage 100% (0 missed), interrogate 100%. Refs #1948, #1935. Co-Authored-By: Claude Fable 5.1 * fix(strix): report only sandbox retries that actually ran Lane peer 1's verification note on #1953: the reporting variable was set where the extra attempt is granted, but a granted attempt can still be vetoed by the timeout / transient checks that follow, so a log carrying both the sandbox and a timeout signature was charged, not retried, and reported as "after 1 sandbox-specific same-model retries". SANDBOX_RETRIES_USED is now assigned only when the retry really proceeds (just before the attempt counter advances); the budget charge stays in the grant branch, so the bound is unchanged. The constant's comment notes that a sandbox retry waits the same inter-attempt backoff as any other retry -- a pause between container attempts, not an inference deadline. Test: sandbox+timeout log -> 1 call, reported 0; plain sandbox log -> 2 calls, reported 1 (the harness echoes SANDBOX_RETRIES_USED). Module 14/14; negative control on main's gate 7 failed / 7 passed. Gate: 2928 passed, 1 skipped, coverage 100% (0 missed), interrogate 100%. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- CHANGELOG.md | 4 + scripts/ci/strix_quick_gate.sh | 47 ++++- ...test_strix_caido_bootstrap_timing_retry.py | 194 ++++++++++++++++++ 3 files changed, 244 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efd2771e96..75a4109c9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Strix gate names the sandbox bootstrap failure and retries it once + +- `scripts/ci/strix_quick_gate.sh` gives the Caido sandbox bootstrap race (`loginAsGuest failed after 10 attempts` on `127.0.0.1:`, upstream usestrix/strix#1036/#1037/#1056) its own bounded same-model retry budget, `STRIX_SANDBOX_BOOTSTRAP_RETRIES` (default 1), drawn on top of `STRIX_TRANSIENT_RETRY_PER_MODEL`. That budget is 0 in production because the gateway owns model failover, so the documented sandbox retry never ran: `argos` Strix run 34013128112 (2026-09-06) shows one attempt, `Docker image ready`, the proxy never reachable, Strix exiting after 240 s -- while the sidecar reported four ready and four deferred routes that were never called. The budget is charged in the same branch that grants the attempt, so a log matching the sandbox class together with a gateway class cannot extend the loop without charging it (caught by adversarial review of the first draft). The primary-scan verdict for that class now reads `STRIX_PROVIDER_UNAVAILABLE: STRIX_SANDBOX_UNAVAILABLE: the last Strix attempt ended in the sandbox bootstrap (...) after N sandbox-specific same-model retries (budget B); this verdict names Strix's sandbox, not the LLM gateway.` instead of `orchestrator/free exhausted`, stating only what the gate observed; the leading token is unchanged so the workflow's finding-free classification and its tests are untouched, and the second token lets the review census split sandbox outages from gateway ones (two of six recent Strix artifacts were this class). Refs #1948. + ### Review sidecar preflight fills the served set lazily to a readiness target - `_preflight_review_agents` now treats the catalog as a candidate list, probed in its tier-then-round-robin order until `REVIEW_PREFLIGHT_TARGET_READY = 8` routes are ready or `REVIEW_PREFLIGHT_MAX_PROBES = 16` probes are spent (ADR-0029). The two-stage candidate budget rises from 12 to 24 (`REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES`; auto pool split 16 free / 8 priced; the sidecar's and the launcher's `ORCHESTRATOR_CATALOG_LIMIT` defaults follow), the production `free` pool lists all 24 (12 before), and the per-account cap stays 8. An account that answers 429 to `REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429 = 2` consecutive probes has its remaining candidates skipped without a probe (a 429 is a per-key answer), so the probes it would have spent reach the other accounts' next candidates — under the real 2026-09-06 order that is the difference between about five ready routes and the target of eight — and a fully rate-limited hour costs two probes per account instead of the whole budget; the report gains `skipped_count` and `account_skip_after_429`. The sidecar's job-log echo of the preflight JSON grows from 160 to 400 lines so 16 probed routes are not cut off exactly in the dead hour the summary matters. A permanently dead candidate -- NIM lists `gemma-3-12b`/`gemma-3-4b` and answers 404 on every run -- now costs one probe instead of a served slot, and a healthy pool stops early instead of always probing every candidate. Motivation: after #1939's four-per-account slice each NVIDIA key's slots were its first four models alphabetically, two of them those 404s, so preflight readiness fell from 6/12 to 1–3/12 and `noema-review` on this repository went from 7 successes / 14 failures to 0 / 22. The report gains `candidate_count`, `target_ready` and `probe_budget`; `probed_count` counts probes actually sent. ADR-0003's stage-budget sentence is amended. Refs #1939, #1947, #1948. diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 3a563d7020..617ca7c394 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -46,6 +46,18 @@ STRIX_EXECUTABLE_ROOT="${STRIX_EXECUTABLE_ROOT:-}" STRIX_EXECUTABLE_SHA256="${STRIX_EXECUTABLE_SHA256:-}" STRIX_TRANSIENT_RETRY_PER_MODEL="${STRIX_TRANSIENT_RETRY_PER_MODEL:-0}" STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS="${STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS:-3}" +## Extra same-model attempts granted only to the Caido sandbox bootstrap race +## (is_caido_bootstrap_timing_error), on top of STRIX_TRANSIENT_RETRY_PER_MODEL. +## That budget is 0 in production because the gateway owns model failover, but +## the sandbox never reaches the model: a fresh container is the only cure for +## a proxy that never came up, and without this the documented retry never ran +## (argos run 34013128112, 2026-09-06: one attempt, then the gateway blamed). +## A sandbox retry waits the same STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS as any +## other retry -- a pause between container attempts, not an inference deadline. +STRIX_SANDBOX_BOOTSTRAP_RETRIES="${STRIX_SANDBOX_BOOTSTRAP_RETRIES:-1}" +## Sandbox-specific retries actually taken by the primary model's attempt +## loop; the final verdict reports this observed count, not the budget. +SANDBOX_RETRIES_USED=0 STRIX_FAIL_ON_MIN_SEVERITY="${STRIX_FAIL_ON_MIN_SEVERITY:-MEDIUM}" STRIX_FAIL_ON_PROVIDER_SIGNAL="${STRIX_FAIL_ON_PROVIDER_SIGNAL:-0}" RUN_START_EPOCH=0 @@ -885,6 +897,7 @@ if is_github_models_model "$PRIMARY_MODEL" && [ -z "$LLM_API_BASE_FILE" ]; then fi require_non_negative_integer "$STRIX_TRANSIENT_RETRY_PER_MODEL" "STRIX_TRANSIENT_RETRY_PER_MODEL" +require_non_negative_integer "$STRIX_SANDBOX_BOOTSTRAP_RETRIES" "STRIX_SANDBOX_BOOTSTRAP_RETRIES" require_non_negative_integer "$STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS" "STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS" require_non_negative_integer "$STRIX_PROCESS_TIMEOUT_SECONDS" "STRIX_PROCESS_TIMEOUT_SECONDS" require_non_negative_integer "$STRIX_TOTAL_TIMEOUT_SECONDS" "STRIX_TOTAL_TIMEOUT_SECONDS" @@ -3092,6 +3105,8 @@ run_strix_with_transient_retry() { local model="$1" local max_attempts=$((STRIX_TRANSIENT_RETRY_PER_MODEL + 1)) local attempt=1 + local sandbox_retries_used=0 + SANDBOX_RETRIES_USED=0 while [ "$attempt" -le "$max_attempts" ]; do local run_rc=0 @@ -3107,7 +3122,18 @@ run_strix_with_transient_retry() { fi if [ "$attempt" -ge "$max_attempts" ]; then - return 1 + ## The per-model budget is spent. The sandbox bootstrap race is not + ## a model failure, so it may draw on its own bounded budget. The + ## budget is charged HERE, in the same branch that grants the + ## attempt: charging it anywhere else lets a log that matches the + ## sandbox class together with another class grant without + ## charging, and nothing in production bounds the loop then. + if is_caido_bootstrap_timing_error && [ "$sandbox_retries_used" -lt "$STRIX_SANDBOX_BOOTSTRAP_RETRIES" ]; then + max_attempts=$((max_attempts + 1)) + sandbox_retries_used=$((sandbox_retries_used + 1)) + else + return 1 + fi fi if [ "$STRIX_TOTAL_TIMEOUT_SECONDS" -gt 0 ] && [ "$(remaining_total_budget)" -le 0 ]; then @@ -3138,6 +3164,10 @@ run_strix_with_transient_retry() { retry_reason="Caido sandbox bootstrap timing" fi echo "Retrying model '$model' due to $retry_reason (attempt $((attempt + 1))/$max_attempts)." >&2 + ## Reported only once the retry really runs: a granted attempt can + ## still be vetoed by the timeout / transient checks above, and the + ## verdict must state retries taken, not budget spent. + SANDBOX_RETRIES_USED="$sandbox_retries_used" sleep "$STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS" attempt=$((attempt + 1)) done @@ -4335,6 +4365,21 @@ run_current_target_scan() { local strict_primary_provider_fallback=0 if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && provider_signal_fail_closed_enabled; then if is_contextual_orchestrator_model "$PRIMARY_MODEL"; then + ## Name the component that actually failed. The sandbox race ends + ## the run before any model request, so blaming the gateway here + ## corrupted every census that read this line (2026-09-06: two of + ## six recent Strix artifacts were this class, with the sidecar + ## reporting ready routes that were never called). The leading + ## STRIX_PROVIDER_UNAVAILABLE token is kept: the workflow classifies + ## a finding-free sandbox outage as incomplete infrastructure + ## evidence, and its tests pin that. + if is_caido_bootstrap_timing_error; then + ## Only what the gate observed: the last attempt's log shows the + ## sandbox bootstrap failure, and this many sandbox-specific + ## retries were taken. Nothing is claimed about the gateway. + echo "STRIX_PROVIDER_UNAVAILABLE: STRIX_SANDBOX_UNAVAILABLE: the last Strix attempt ended in the sandbox bootstrap (Caido proxy on 127.0.0.1 unreachable through Strix's loginAsGuest attempts) after ${SANDBOX_RETRIES_USED} sandbox-specific same-model retries (budget ${STRIX_SANDBOX_BOOTSTRAP_RETRIES}); this verdict names Strix's sandbox, not the LLM gateway." >&2 + return 1 + fi echo "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free exhausted; the gateway owns provider discovery and failover." >&2 return 1 elif is_model_retryable_error "$PRIMARY_MODEL" && has_distinct_fallback_model_for_model "$PRIMARY_MODEL"; then diff --git a/tests/test_strix_caido_bootstrap_timing_retry.py b/tests/test_strix_caido_bootstrap_timing_retry.py index a60b9d801b..3bb7c9e220 100644 --- a/tests/test_strix_caido_bootstrap_timing_retry.py +++ b/tests/test_strix_caido_bootstrap_timing_retry.py @@ -138,5 +138,199 @@ def test_retry_reason_is_logged_for_operators(self) -> None: ) +RATE_LIMIT_LOG = ( + "litellm.RateLimitError: RateLimitError: rate limit exceeded\n" + "Vulnerabilities 0\n" +) + + +def _run_retry_loop(log_text: str, *, per_model: int, sandbox_retries: int) -> tuple[int, str]: + """Drive the production retry loop with a stubbed Strix run and return (calls, stderr). + + The reported sandbox retry count (``SANDBOX_RETRIES_USED``) is echoed to + stdout as ``reported=`` and appended to the returned stderr text so + tests can assert it without a second harness. + + ``run_strix_once`` is replaced by a stub that writes ``log_text`` to the + attempt log and fails, so the loop's own retry decision is what is under + test; every classifier the loop consults is the production function. + """ + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + blocks = [ + _function_block(gate_source, name) + for name in ( + "run_strix_with_transient_retry", + "is_transient_same_model_retry_error", + "is_timeout_error", + "is_llm_api_connection_error", + "is_llm_service_unavailable_error", + "is_rate_limit_error", + "is_midstream_fallback_error", + "is_caido_bootstrap_timing_error", + ) + ] + with tempfile.TemporaryDirectory(prefix="strix-caido-retry-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + counter = Path(temp_dir) / "calls" + counter.write_text("0", encoding="utf-8") + script = "\n".join( + ( + "set -uo pipefail", + f'STRIX_LOG="{log_path}"', + f'COUNTER="{counter}"', + f"STRIX_TRANSIENT_RETRY_PER_MODEL={per_model}", + f"STRIX_SANDBOX_BOOTSTRAP_RETRIES={sandbox_retries}", + "STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS=0", + "STRIX_TOTAL_TIMEOUT_SECONDS=0", + "TOTAL_TIMEOUT_EXCEEDED=0", + "github_models_rate_limit_should_skip_same_model_retry() { return 1; }", + # The stub caps itself: a runaway loop returns the configuration + # exit code 2 after six calls, which the harness reports as a + # failure instead of hanging the suite. + 'run_strix_once() { n=$(( $(cat "$COUNTER") + 1 )); echo "$n" > "$COUNTER"; printf "%s" "$LOG_TEXT" > "$STRIX_LOG"; [ "$n" -ge 6 ] && return 2; return 1; }', + *blocks, + 'run_strix_with_transient_retry "orchestrator/free"; rc=$?; echo "reported=$SANDBOX_RETRIES_USED"; exit "$rc"', + ) + ) + completed = subprocess.run( + ["bash", "-c", script, "strix-retry"], + check=False, + capture_output=True, + text=True, + env={"PATH": "/usr/bin:/bin", "LOG_TEXT": log_text}, + ) + calls = int(counter.read_text(encoding="utf-8").strip()) + if completed.returncode != 1: + raise AssertionError(f"rc={completed.returncode}\n{completed.stderr}") + return calls, completed.stderr + completed.stdout + + +def _orchestrator_verdict_line(log_text: str) -> str: + """Return the stderr the primary-scan verdict branch emits for a failed orchestrator scan.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + blocks = [ + _function_block(gate_source, name) + for name in ("run_current_target_scan", "is_caido_bootstrap_timing_error") + ] + with tempfile.TemporaryDirectory(prefix="strix-caido-verdict-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + script = "\n".join( + ( + "set -uo pipefail", + f'STRIX_LOG="{log_path}"', + 'PRIMARY_MODEL="orchestrator/free"', + "STRIX_SANDBOX_BOOTSTRAP_RETRIES=1", + "SANDBOX_RETRIES_USED=1", + "TOTAL_TIMEOUT_EXCEEDED=0", + # run_current_target_scan resets INFRA_ERROR_DETECTED before the + # scan; the production run_strix_once sets it on a failed attempt, + # so the stub does the same. + "run_strix_with_transient_retry() { INFRA_ERROR_DETECTED=1; return 1; }", + "provider_signal_fail_closed_enabled() { return 0; }", + "is_contextual_orchestrator_model() { return 0; }", + "is_model_retryable_error() { return 1; }", + "has_distinct_fallback_model_for_model() { return 1; }", + # has_detected_infrastructure_error is consulted by run_strix_once, + # which the stub above replaces; the flag is set by that path. + *blocks, + "run_current_target_scan", + ) + ) + completed = subprocess.run( + ["bash", "-c", script, "strix-verdict"], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode != 1: + raise AssertionError(f"rc={completed.returncode}\n{completed.stderr}") + return completed.stderr + + +class StrixSandboxBootstrapRetryAndVerdictTests(unittest.TestCase): + """The sandbox race gets its own bounded retry and its own name in the verdict. + + Evidence (2026-09-06): ``argos`` Strix run 34013128112 and a second + artifact both show a single attempt ending in ``loginAsGuest failed after + 10 attempts`` on ``127.0.0.1:48080`` after ``Docker image ready``, then + ``STRIX_PROVIDER_UNAVAILABLE: … orchestrator/free exhausted`` -- while the + sidecar had four ready and four deferred routes that were never called. + ``STRIX_TRANSIENT_RETRY_PER_MODEL`` defaults to 0 and the workflow does not + raise it, so the documented same-model retry for this class never ran. + """ + + def test_sandbox_bootstrap_failure_is_retried_once_even_with_zero_per_model_budget(self) -> None: + calls, stderr = _run_retry_loop(OBSERVED_LOG, per_model=0, sandbox_retries=1) + self.assertEqual(calls, 2) + self.assertIn("Caido sandbox bootstrap timing", stderr) + self.assertIn("attempt 2/2", stderr) + + def test_sandbox_retry_budget_is_bounded(self) -> None: + calls, _ = _run_retry_loop(OBSERVED_LOG, per_model=0, sandbox_retries=2) + self.assertEqual(calls, 3) + calls, _ = _run_retry_loop(OBSERVED_LOG, per_model=0, sandbox_retries=0) + self.assertEqual(calls, 1) + + def test_mixed_sandbox_and_gateway_log_stays_bounded(self) -> None: + """A log matching the sandbox class AND a gateway class grants at most the sandbox budget. + + Found by adversarial review of the first draft, which charged the + sandbox counter in the retry-reason chain behind the gateway classes: + such a log then extended the budget on every iteration without ever + charging it, and production bounds the loop with nothing but GitHub's + six-hour default. + """ + + calls, stderr = _run_retry_loop(RATE_LIMIT_LOG + OBSERVED_LOG, per_model=0, sandbox_retries=1) + self.assertEqual(calls, 2) + self.assertNotIn("attempt 3/", stderr) + + def test_sandbox_budget_is_granted_on_top_of_the_per_model_budget(self) -> None: + calls, _ = _run_retry_loop(OBSERVED_LOG, per_model=1, sandbox_retries=1) + self.assertEqual(calls, 3) + + def test_reported_sandbox_retries_count_only_retries_that_ran(self) -> None: + """A granted attempt vetoed by the timeout check is not reported as a retry. + + Lane peer 1's verification note: the budget is charged at the grant, + but ``is_transient_same_model_retry_error`` returns 1 for a timeout + signature, so a log carrying both the sandbox and a timeout signature + is granted, charged, and then not retried; the verdict must say 0. + """ + + calls, out = _run_retry_loop( + "litellm.exceptions.Timeout: request timed out\n" + OBSERVED_LOG, + per_model=0, + sandbox_retries=1, + ) + self.assertEqual(calls, 1) + self.assertIn("reported=0", out) + calls, out = _run_retry_loop(OBSERVED_LOG, per_model=0, sandbox_retries=1) + self.assertEqual(calls, 2) + self.assertIn("reported=1", out) + + def test_sandbox_retry_does_not_widen_gateway_retries(self) -> None: + """A rate limit from the gateway still gets no same-model retry at budget 0.""" + + calls, stderr = _run_retry_loop(RATE_LIMIT_LOG, per_model=0, sandbox_retries=1) + self.assertEqual(calls, 1) + self.assertNotIn("Retrying model", stderr) + + def test_verdict_names_the_sandbox_and_keeps_the_workflow_token(self) -> None: + stderr = _orchestrator_verdict_line(OBSERVED_LOG) + self.assertIn("STRIX_PROVIDER_UNAVAILABLE: STRIX_SANDBOX_UNAVAILABLE:", stderr) + self.assertIn("after 1 sandbox-specific same-model retries (budget 1)", stderr) + self.assertIn("names Strix's sandbox, not the LLM gateway", stderr) + self.assertNotIn("orchestrator/free exhausted", stderr) + + def test_verdict_for_a_gateway_failure_is_unchanged(self) -> None: + stderr = _orchestrator_verdict_line(RATE_LIMIT_LOG) + self.assertIn("orchestrator/free exhausted", stderr) + self.assertNotIn("STRIX_SANDBOX_UNAVAILABLE", stderr) + + if __name__ == "__main__": unittest.main() From aca3b05146721365a2ae9b7fcfb04789ae7a789c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 6 Sep 2026 08:24:07 +0000 Subject: [PATCH 20/80] docs(strix): separate occupied slots from queued runs, and scan scope from per-commit evidence retention MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review corrections on ea0a429a6: the 14:27Z measurement supports five runs holding runner slots and four waiting in the queue, not nine slots; and a newer push scan covers the current tree in full without being a record of every earlier commit's findings — code that entered and left between two heads, or findings a retired run never uploaded, are absent, and a per-commit retention guarantee would need its own contract. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01BV96rXhqoR3tYZ9AeAVur4 --- ...-failure-and-strix-concurrency-20260904.md | 21 ++++++++++++------- .../test_required_workflow_queue_contract.py | 5 +++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/doctoring/startup-failure-and-strix-concurrency-20260904.md b/docs/doctoring/startup-failure-and-strix-concurrency-20260904.md index 7c380ce7b5..5ba354947a 100644 --- a/docs/doctoring/startup-failure-and-strix-concurrency-20260904.md +++ b/docs/doctoring/startup-failure-and-strix-concurrency-20260904.md @@ -45,16 +45,23 @@ concurrency ordering. **Amendment (2026-09-05).** "nor one another" no longer holds for `push` events on the same branch. Measured at 14:27Z in `.github`: nine `push`/`main` -Strix runs were outstanding at once (five running, jobs started 12:31-14:25Z, -one already past two hours; four queued), each holding one slot under the -shared 60-job ceiling, against a 10-30 minute normal scan. The run-id fallback +Strix runs were outstanding at once — five holding runner slots under the +shared 60-job ceiling (jobs started 12:31-14:25Z, one already past two hours) +and four more waiting in the queue behind them, which occupy no slot until a +runner is assigned — against a 10-30 minute normal scan. The run-id fallback in the workflow-level group made every main push its own group, so no newer main head ever retired an older scan. The workflow-level group now scopes `push` events as `push-`: a newer head of the same protected branch -supersedes the older scan exactly as a newer PR head does. This loses nothing -the gate consumes: a push scan covers the whole tree (`STRIX_TARGET_PATH` is -`./` outside PR scope) and publishes no `strix` commit status. `schedule` and -PR-less `repository_dispatch` runs still receive a unique run id. The +supersedes the older scan exactly as a newer PR head does. What a retired scan +gives up is its own report, not the gate's inputs: a push scan covers the whole +tree (`STRIX_TARGET_PATH` is `./` outside PR scope) and publishes no `strix` +commit status, so the newest head's scan is a complete scan *of the current +tree*. It is not a record of every earlier commit: code that entered and left +`main` between two heads, and findings a retired run never uploaded, are absent +from the newest report, and report collection preserves only runs that reach +it. A per-commit evidence-retention guarantee would need a separate, +verifiable preservation contract; this change does not provide one. `schedule` +and PR-less `repository_dispatch` runs still receive a unique run id. The `pr_number=${GITHUB_RUN_ID}` admission output is unchanged. Tradeoff, stated so a later reader of the security dashboard is not diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 0c05ca6f6f..9799094d7e 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -390,8 +390,9 @@ def test_strix_serializes_provider_evidence_per_repository_and_pr() -> None: because the run-id fallback made every main push its own group and nothing ever retired a superseded main scan. A push scan covers the whole tree (``STRIX_TARGET_PATH`` is ``./`` outside PR scope) and publishes no - ``strix`` commit status, so the newest branch head subsumes every older - one. ``schedule`` and ``repository_dispatch`` without a PR number keep a + ``strix`` commit status, so the newest head's scan is a complete scan of + the current tree (not a record of every earlier commit's findings). + ``schedule`` and ``repository_dispatch`` without a PR number keep a unique run id. """ workflow = workflow_text("strix.yml") From 5ea1cc47ec040fa4f6417136f059be637666c2a2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 20:24:50 +0900 Subject: [PATCH 21/80] ci(review): coalesce superseded OpenCode review dispatches before admission (#1958) opencode-review-dispatch.yml carried its concurrency group only on the long opencode-review-target job. A job-level group is never evaluated while the whole run waits behind the organization job ceiling, so two dispatches for one pull request each queued for hours and each was allocated a runner before the older one could be discarded. Measured on 2026-09-06: of the five dispatch runs that passed validate-pr-metadata, four were then rejected by the privileged metadata check because the head had moved while they queued (34002473295, 34010256951, 34015973300, 34016922761), every one of them after coverage-source-tree and coverage-evidence had already run. The privileged check behaved correctly; the cost is that a runner slot is spent discovering that the review's subject no longer exists. Add the workflow-level group keyed by the dispatched pull request, matching codeql-scan-dispatch.yml's workflow-level group and the rationale recorded in strix.yml, noema-review.yml and opencode-review.yml. The job-level group stays. No behaviour changes between two runs that are both executing -- the job-level group already cancels there; what changes is that a superseded run is now cancelled while queued. Co-authored-by: Seongho Bae Co-authored-by: Claude Opus 5 --- .../workflows/opencode-review-dispatch.yml | 22 ++++++++++++ CHANGELOG.md | 4 +++ tests/test_opencode_agent_contract.py | 8 +++-- ...t_pr_review_autofix_nvidia_nim_contract.py | 2 +- .../test_required_workflow_queue_contract.py | 36 +++++++++++++++++++ 5 files changed, 69 insertions(+), 3 deletions(-) diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 26e8555967..fd0e5ff75d 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -11,6 +11,28 @@ on: repository_dispatch: types: [opencode-review] +concurrency: + # Workflow-level admission, for the same reason strix.yml, noema-review.yml and + # opencode-review.yml carry theirs at this level: a job-level group is never + # evaluated while the whole run waits behind the organization job ceiling, so + # superseded dispatches for one pull request coalesce only after each of them + # has already been allocated a runner. Measured on 2026-09-06: of the five + # dispatch runs that passed `validate-pr-metadata`, four were rejected hours + # later by `opencode-review`'s privileged metadata check because the head had + # moved while they queued (runs 34002473295, 34010256951, 34015973300, + # 34016922761) -- each after `coverage-source-tree` and `coverage-evidence` + # had run. Cancelling the superseded run at creation returns that slot instead + # of spending it to discover the review's subject no longer exists. + # + # The key is the target pull request, matching the job-level group below and + # codeql-scan-dispatch.yml's workflow-level group; `github.run_id` keeps runs + # without a payload in their own groups rather than colliding. + group: >- + opencode-review-dispatch-${{ + github.event.client_payload.target_repository || github.repository }}-${{ + github.event.client_payload.pr_number || github.run_id }} + cancel-in-progress: true + permissions: contents: read diff --git a/CHANGELOG.md b/CHANGELOG.md index 75a4109c9d..1f535e06c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Superseded OpenCode review dispatches coalesce before they take a runner + +- `opencode-review-dispatch.yml` now carries a workflow-level `concurrency` group keyed by the dispatched pull request (`opencode-review-dispatch--`, `cancel-in-progress: true`), matching `codeql-scan-dispatch.yml`'s workflow-level group and the rationale already recorded in `strix.yml`, `noema-review.yml` and `opencode-review.yml`: a job-level group is never evaluated while the whole run waits behind the organization job ceiling. The workflow kept its group only on the long `opencode-review-target` job, so two dispatches for one pull request each queued for hours and each was allocated a runner before the older one could be discarded. Measured on 2026-09-06: four of the five dispatch runs that passed `validate-pr-metadata` were rejected hours later by the privileged metadata check because the head had moved while they queued (runs `34002473295`, `34010256951`, `34015973300`, `34016922761`), each after `coverage-source-tree` and `coverage-evidence` had run. The privileged check itself is unchanged -- it rejected exactly what it should; what changes is that the superseded run is now cancelled at creation instead of spending a slot to discover its subject moved. + ### Strix gate names the sandbox bootstrap failure and retries it once - `scripts/ci/strix_quick_gate.sh` gives the Caido sandbox bootstrap race (`loginAsGuest failed after 10 attempts` on `127.0.0.1:`, upstream usestrix/strix#1036/#1037/#1056) its own bounded same-model retry budget, `STRIX_SANDBOX_BOOTSTRAP_RETRIES` (default 1), drawn on top of `STRIX_TRANSIENT_RETRY_PER_MODEL`. That budget is 0 in production because the gateway owns model failover, so the documented sandbox retry never ran: `argos` Strix run 34013128112 (2026-09-06) shows one attempt, `Docker image ready`, the proxy never reachable, Strix exiting after 240 s -- while the sidecar reported four ready and four deferred routes that were never called. The budget is charged in the same branch that grants the attempt, so a log matching the sandbox class together with a gateway class cannot extend the loop without charging it (caught by adversarial review of the first draft). The primary-scan verdict for that class now reads `STRIX_PROVIDER_UNAVAILABLE: STRIX_SANDBOX_UNAVAILABLE: the last Strix attempt ended in the sandbox bootstrap (...) after N sandbox-specific same-model retries (budget B); this verdict names Strix's sandbox, not the LLM gateway.` instead of `orchestrator/free exhausted`, stating only what the gate observed; the leading token is unchanged so the workflow's finding-free classification and its tests are untouched, and the second token lets the review census split sandbox outages from gateway ones (two of six recent Strix artifacts were this class). Refs #1948. diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 37ec068db9..321d25bd57 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -1815,8 +1815,12 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert "run_opencode_review_model_pool.sh" in workflow assert "rekick_model_pool_on_exhaustion" not in workflow assert "publish stage performs no duplicate model-catalog pass" in workflow - concurrency_contract = workflow.split("concurrency:", 1)[1].split( - "permissions:", 1 + # The review job's own group, addressed by its indentation: the workflow + # also carries a workflow-level admission group (pinned in + # tests/test_required_workflow_queue_contract.py), so splitting on the + # first "concurrency:" would read that one instead of this one. + concurrency_contract = workflow.split("\n concurrency:", 1)[1].split( + "\n runs-on:", 1 )[0] assert "needs.validate-pr-metadata.outputs.target_repository" in concurrency_contract assert "needs.validate-pr-metadata.outputs.pr_number || github.run_id" in concurrency_contract diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index 2d2304aaf1..af49190142 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -17,7 +17,7 @@ DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md") CHANGELOG = Path("CHANGELOG.md") REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") -REVIEW_DISPATCH_BLOB_SHA = "26e8555967171a5f3974602ac05700c27bddebf1" +REVIEW_DISPATCH_BLOB_SHA = "fd0e5ff75d9ae6ec68cc6945174dff5c9d487662" def _workflow_text(path: Path) -> str: diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 803d43ab59..48035c2c40 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -214,6 +214,42 @@ def test_privileged_review_retries_use_default_branch_repository_dispatch() -> N assert '"gh",\n "workflow",\n "run"' not in autofix_scheduler +def test_privileged_review_dispatch_coalesces_superseded_runs_before_admission() -> None: + """A superseded dispatch must be cancelled while queued, not after it takes a runner. + + ``opencode-review-dispatch.yml`` carried its concurrency group only on the + long ``opencode-review-target`` job. A job-level group is not evaluated + while the whole run waits behind the organization job ceiling, so two + dispatches for one pull request each waited hours and each was allocated a + runner before the older one could be discarded. Measured on 2026-09-06: + four of the five dispatch runs that passed ``validate-pr-metadata`` were + then rejected by the privileged metadata check because the head had moved + while they queued, every one of them after ``coverage-source-tree`` and + ``coverage-evidence`` had already run. + + The workflow-level group is keyed by the dispatched pull request, matching + ``codeql-scan-dispatch.yml``'s workflow-level group and the job-level group + this workflow keeps for the review job itself. + """ + workflow = workflow_text("opencode-review-dispatch.yml") + header = workflow.split("permissions:", 1)[0] + concurrency_contract = header.split("concurrency:", 1)[1] + + assert re.search(r"(?m)^concurrency:", header) + assert "opencode-review-dispatch-" in concurrency_contract + assert ( + "github.event.client_payload.target_repository || github.repository" + in concurrency_contract + ) + assert ( + "github.event.client_payload.pr_number || github.run_id" + in concurrency_contract + ) + assert "cancel-in-progress: true" in concurrency_contract + assert "github.event.client_payload.pr_head_sha" not in concurrency_contract + assert re.search(r"(?m)^ concurrency:", workflow) + + def test_required_opencode_dispatch_does_not_wait_on_merge_scheduler() -> None: """Dispatch review execution directly so polling cannot starve its producer.""" workflow = workflow_text("opencode-review.yml") From 0b0f10476469d52adc40f98495d50855486cd32f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 20:37:30 +0900 Subject: [PATCH 22/80] fix(preflight): postpone a rate-limited account's candidates instead of banning them (#1957) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #1949's account rule sets aside an account's remaining candidates after two consecutive 429s. When a walk runs out of candidates it is willing to probe it ENDS -- with probe budget in hand and the readiness target unmet -- and the stage fails closed; because deferral needs one ready route (#1947), nothing is served either. Sixteen sidecar artifacts were collected on 2026-09-06 across .github, argos, bandscope and naruon; fourteen ran the merged rule (argos 34013128112 and bandscope 34013146167 still carry the pre-#1949 report shape). Those fourteen fall into three classes, not two: eight boots at probed/skipped/ready 16/4/5-6 spend the whole budget in the first pass and are unchanged by this commit; ONE (argos 34014143870, 06:56Z) reads 12/12/3 -- it served, yet exhausted its candidates under target with four probes unspent; five read 6/18/0 and failed closed. The sixth ready route in the healthy class (llama-3.2-11b on the second NVIDIA key, catalog position 17, ready in exactly those eight artifacts) is reached only because four OpenRouter probes were set aside -- the rule's designed benefit, which this commit keeps. .github run 34016207820's six probes were refused 429 between 07:49:35.111Z and 07:49:35.767Z; because the walk round-robins three accounts, "two consecutive 429s" on one account is two requests about 310 ms apart (nvidia_nim at .111 and .422). keyverse#143's 08:20Z noema repeated the shape in a second repository. A refusal is not a verdict on the account: run 34016093772 was inside its own preflight during that burst and its llama-3.2-11b probes on the same two NVIDIA keys answered ready at 07:50:58.7Z and 07:50:59.0Z, 84 s after those keys refused. Not claimed: that the ten unspent probes would have found a ready route inside the burst. No artifact answers it, which is why this also records retry_after_s. The change rests on the structural defect alone. A set-aside candidate is now postponed to the end of the walk; once the first pass ends under target with budget left, the postponed candidates are probed in catalog order until the sixteen-probe budget is spent. Both passes share one stop condition, so probes per stage stay <= 16, and exhaustion uses a dedicated sentinel so a None candidate cannot truncate the walk. The second pass never draws on the shared escalation budget (#1458): a postponed candidate answering "budget too small" is rejected as escalation_reserved_for_first_pass, because otherwise candidates the previous design never probed take escalations from the priced stage that had them, and a measured two-stage run stops serving a route it used to serve. _safe_retry_after_seconds records a refused probe's Retry-After as retry_after_s when it is whole delta-seconds in range. It gates on isdecimal, not isdigit: the header is provider-controlled, "²".isdigit() is True while int() on it raises, and this runs inside the probe walk's exception handler whose callers catch only ReviewPreflightError -- so a ValueError there would kill the boot before any evidence file is written. No code waits on the value (ADR-0003). Cost, stated in the ADR and PR body against the 60-job ceiling work: about 120 ms per refused probe, up to 10 x 90 s ~= 15 minutes when the postponed tail is silent (gemma-4-31b answered TimeoutError in 15 of the 19 probes that reached it), and 8 -> 24 requests on the two-stage auto path, where the priced stage doubles from 4 probes to 8. All inside the probe budget ADR-0029 bounds. Report: postponed_probed_count added, skipped_count now means "postponed and never reached". ADR-0029 amended, and its two superseded sentences marked in place. Verified by a three-lens adversarial refutation before push (control flow, evidence and design, test fidelity): all three returned refuted=true with 20 findings, each reproduced against the artifacts before acting. The blocker above, the escalation-budget regression, the miscounted evidence table, the false "healthy-minute walk is unchanged" claim, the 310 ms spacing, the sibling run's real relationship to the burst and the superseded ADR sentences all come from that pass. Gate on this tree: 2945 passed, 1 skipped, 21 subtests; coverage 100% (0 missed); interrogate 100%. Negative control on origin/main's launcher with this test file: 8 failed, 93 passed. Refs #1948, #1949. Co-authored-by: Claude Opus 5 --- CHANGELOG.md | 4 + docs/adr/0029-sidecar-preflight-lazy-fill.md | 36 ++- ...contextual_orchestrator_review_launcher.py | 167 +++++++++--- ...l_orchestrator_review_runtime_preflight.py | 238 ++++++++++++++++-- 4 files changed, 390 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f535e06c9..dca548fb03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Review sidecar preflight postpones a rate-limited account's candidates instead of banning them + +- `_preflight_review_agents` no longer ends its walk when every credential account has answered 429 twice in a row. A candidate set aside by `REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429` is postponed to the end of the walk, and once the first pass ends with the readiness target unmet and probe budget left, the postponed candidates are probed in catalog order until the sixteen-probe budget is spent. On 2026-09-06 five sidecar boots whose probes began between 07:24Z and 08:05Z read `probed 6 / skipped 18 / ready 0` and failed closed: `.github` run 34016207820's six probes across all three accounts were refused 429 between 07:49:35.111Z and 07:49:35.767Z, so the rule set every account aside on two same-account requests about 310 ms apart and gave up with ten of sixteen probes unspent — and because deferral needs one ready route, nothing was served either; `keyverse#143`'s 08:20Z `noema-review` repeated it in a second repository (six probes, 369 ms, all 429). The pools are not dead in those minutes: run 34016093772 was inside its own preflight during that burst, and its `llama-3.2-11b` probes on the same two NVIDIA keys answered ready at 07:50:58.7Z and 07:50:59.0Z, 84 seconds after those keys refused. Whether the unspent probes would have found a ready route inside a burst is unmeasured and is not claimed; the change is justified by ending a walk under target with the budget in hand. Of the fourteen boots that ran the merged rule, eight spend all sixteen probes in the first pass and are unchanged; one (`argos` 34014143870, a serving boot at `12 / 12 / 3`) exhausts its candidates under budget and now gains a second pass, as do the five burst boots. The cost is stated rather than assumed: a refused probe costs about 120 ms, a silent one up to the 90 s receive timeout, and the postponed tail holds both (`google/gemma-4-31b-it` answered `TimeoutError` in 15 of the 19 probes that reached it), so the worst case adds up to about 15 minutes to a boot that still fails and the two-stage auto path goes from 8 to 24 requests including the priced stage. The second pass never draws on the shared escalation budget, so the priced fallback keeps the escalations it had. The report gains `postponed_probed_count` (`skipped_count` now counts postponed candidates the budget never reached) and, on a refused probe, `retry_after_s` when the response carried a whole-seconds `Retry-After` header — evidence only, nothing waits on it, so the next census can decide whether a delayed second pass is worth proposing. ADR-0029 is amended. Refs #1948, #1949. + ### Superseded OpenCode review dispatches coalesce before they take a runner - `opencode-review-dispatch.yml` now carries a workflow-level `concurrency` group keyed by the dispatched pull request (`opencode-review-dispatch--`, `cancel-in-progress: true`), matching `codeql-scan-dispatch.yml`'s workflow-level group and the rationale already recorded in `strix.yml`, `noema-review.yml` and `opencode-review.yml`: a job-level group is never evaluated while the whole run waits behind the organization job ceiling. The workflow kept its group only on the long `opencode-review-target` job, so two dispatches for one pull request each queued for hours and each was allocated a runner before the older one could be discarded. Measured on 2026-09-06: four of the five dispatch runs that passed `validate-pr-metadata` were rejected hours later by the privileged metadata check because the head had moved while they queued (runs `34002473295`, `34010256951`, `34015973300`, `34016922761`), each after `coverage-source-tree` and `coverage-evidence` had run. The privileged check itself is unchanged -- it rejected exactly what it should; what changes is that the superseded run is now cancelled at creation instead of spending a slot to discover its subject moved. diff --git a/docs/adr/0029-sidecar-preflight-lazy-fill.md b/docs/adr/0029-sidecar-preflight-lazy-fill.md index d2528b9883..166d49f9a8 100644 --- a/docs/adr/0029-sidecar-preflight-lazy-fill.md +++ b/docs/adr/0029-sidecar-preflight-lazy-fill.md @@ -30,7 +30,7 @@ The evening's rate-limit pressure is a confound; the mechanism is not. A fixed s The catalog is a **candidate list**, not the served set. `build_zdr_prioritized_catalog` keeps its tier-then-round-robin order (`#1939`) and is asked for up to `REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES = 24` candidates (per-account cap unchanged at 8; the sidecar's `ORCHESTRATOR_CATALOG_LIMIT` default rises from 12 to 24). `_preflight_review_agents` probes candidates **in that order and stops** as soon as `REVIEW_PREFLIGHT_TARGET_READY = 8` routes are ready or `REVIEW_PREFLIGHT_MAX_PROBES = 16` probes have been spent, whichever comes first. The auto pool's split becomes 16 free candidates and up to 8 priced fallback candidates; the production `free` pool (the sidecar default; it has no fallback stage) lists all 24. A silent candidate's probe costs up to one transport timeout (one artifact spent 805 s on 19 probes), so the probe cap bounds preflight wall time as well as request count. -**Account skip.** A 429 at preflight is a per-key answer, not a per-model one. Once one credential account has answered 429 to `REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429 = 2` consecutive probes, its remaining candidates are skipped without a probe and the walk continues with the other accounts' next candidates; the two probed routes are still deferred. Under the real 2026-09-06 candidate order (lane jan's table on `#1949`, rebuilt from `#1938`'s Strix artifact: both NVIDIA keys list deepseek ×2, gemma-3 ×2 (404), gemma-4-31b (empty), then the llama and muse routes; every OpenRouter free route answers 429) the plain sixteen-probe walk yields about five ready and five deferred and the readiness target is unreachable, because five probes go to an account whose every route had answered 429 in every artifact since 21:00Z and four to the dead gemma-3 entries. With the skip, the same sixteen probes reach both keys' `llama-3.2` routes and the target of eight. This is why the free pool lists 24 candidates while probing at most 16: the tail is reachable exactly when an account is skipped, and the report separates `skipped_count` from the unreached remainder (`candidate_count − probed_count − skipped_count`). A rate-limited hour therefore costs two probes per account instead of the full budget. +**Account skip.** *(The "skipped without a probe" and "two probes per account" claims in this paragraph are superseded by the 2026-09-06 amendment below: such a candidate is postponed, and the leftover budget is spent on it.)* A 429 at preflight is a per-key answer, not a per-model one. Once one credential account has answered 429 to `REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429 = 2` consecutive probes, its remaining candidates are skipped without a probe and the walk continues with the other accounts' next candidates; the two probed routes are still deferred. Under the real 2026-09-06 candidate order (lane jan's table on `#1949`, rebuilt from `#1938`'s Strix artifact: both NVIDIA keys list deepseek ×2, gemma-3 ×2 (404), gemma-4-31b (empty), then the llama and muse routes; every OpenRouter free route answers 429) the plain sixteen-probe walk yields about five ready and five deferred and the readiness target is unreachable, because five probes go to an account whose every route had answered 429 in every artifact since 21:00Z and four to the dead gemma-3 entries. With the skip, the same sixteen probes reach both keys' `llama-3.2` routes and the target of eight. This is why the free pool lists 24 candidates while probing at most 16: the tail is reachable exactly when an account is skipped, and the report separates `skipped_count` from the unreached remainder (`candidate_count − probed_count − skipped_count`). A rate-limited hour therefore costs two probes per account instead of the full budget. The sidecar's job-log echo of the preflight JSON (`sed -n '1,400p'`, previously 160 lines) now fits 16 probed routes; the artifact copy was always complete. @@ -39,7 +39,7 @@ The report gains `candidate_count`, `target_ready` and `probe_budget`; `probed_c ## Consequences - **Good:** a dead candidate costs one probe and yields its place to the next candidate in the same account's list; a healthy hour stops after about eight to twelve probes instead of always twelve; a bad hour is bounded at sixteen probes per stage. -- **Cost:** in an hour where nothing is ready the sidecar sends up to 16 probes per stage where it sent 12, a third more against already exhausted keys. This is the price of finding routes past the dead ones; `#1948`'s shared rate ledger is the lever above it. The cap is also a wall-time bound: a 16-token probe can hold the full 90 s receive timeout (`#1661` run 34008191123, 04:48Z, both NVIDIA keys' deepseek-v4-pro probes at 90.06 s and 90.10 s), so a fully silent hour costs at most 16 × 90 s = 24 minutes of preflight against 18 today, and the account-skip rule cuts a rate-limited hour to two probes per account. +- **Cost:** in an hour where nothing is ready the sidecar sends up to 16 probes per stage where it sent 12, a third more against already exhausted keys. This is the price of finding routes past the dead ones; `#1948`'s shared rate ledger is the lever above it. The cap is also a wall-time bound: a 16-token probe can hold the full 90 s receive timeout (`#1661` run 34008191123, 04:48Z, both NVIDIA keys' deepseek-v4-pro probes at 90.06 s and 90.10 s), so a fully silent hour costs at most 16 × 90 s = 24 minutes of preflight against 18 today, and the account-skip rule cuts a rate-limited hour to two probes per account. *(That last clause is superseded by the 2026-09-06 amendment: a rate-limited hour now spends the whole probe budget rather than two probes per account.)* - **Unchanged:** a route that answers the probe and then goes silent at request time still costs the gateway's full retry budget (`contextual-orchestrator#1045`); readiness is measured at 16 tokens (`#1454`). - **Discriminator:** post-merge, `probed_count` versus `candidate_count` per boot and `ready_count` of the served set, read from the `runtime preflight summary` in the job log or the `noema-sidecar-evidence` artifact, compared with the table above. @@ -49,3 +49,35 @@ The report gains `candidate_count`, `target_ready` and `probe_budget`; `probed_c - **Exclude models that 404 by name** — a hard-coded exclusion list the next discovery change silently invalidates; rejected by constraint 1. The discovery-side question (why NIM lists models it does not serve) remains open in `contextual-orchestrator`. - **Family-level interleave inside each account's list before the cap** (jan's second layer) — would make each NVIDIA key's first six candidates span deepseek, gemma, llama, muse, minimax, mistral, but it needs a model-family equivalence derived from names, which ADR-0003/#1468 deliberately avoid; kept in reserve if the post-merge census shows same-family contention as the residual after the account skip. - **Probe all 24 candidates** — best served set, double the probe spend in the hour that can least afford it; rejected by constraint 2. + +## Amendment 2026-09-06: a set-aside candidate is postponed, not banned + +**Evidence.** Sixteen sidecar artifacts were collected on 2026-09-06 across `.github`, `argos`, `bandscope` and `naruon`; **fourteen** ran the merged rule (two, `argos` 34013128112 and `bandscope` 34013146167, still carry the pre-`#1949` report shape and are excluded). The fourteen fall into three classes, not two. + +| class | boots | `probed / skipped / ready` | second pass? | outcome | +|---|---|---|---|---| +| budget spent in the first pass | 8 | 16 / 4 / 5–6 | no — budget already gone | served; the sixth ready route (`llama-3.2-11b` on the second NVIDIA key, catalog position 17, ready in exactly these 8 artifacts) is reached **only** because four OpenRouter probes were set aside — the benefit the rule was designed for | +| candidates exhausted, budget left | 1 | 12 / 12 / 3 (`argos` 34014143870, 06:56Z) | **yes**, up to 4 probes | served with 5 deferred, but the target of 8 was unmet with 4 probes unspent | +| every account set aside | 5 | 6 / 18 / 0 (`rejected 6`, all 429) | **yes**, up to 10 probes | preflight failed closed | + +So the change is not confined to bursts: one served, ordinary-minute boot also ends its first pass under target with budget in hand. Only a boot that spends all sixteen probes in the first pass is untouched. + +The sidecar stderr of `.github` run 34016207820 shows its six probes (both NVIDIA keys' two deepseek routes, two OpenRouter routes) refused 429 between 07:49:35.111Z and 07:49:35.767Z. Because the walk is a round-robin across three accounts, "two consecutive 429s" on one account is two requests about **310 ms** apart (`nvidia_nim` at .111 and .422), not two probes a tenth of a second apart. The rule set all three accounts aside, the walk ended **with ten of its sixteen probes unspent**, and because deferral requires one ready route (`#1947`) nothing was served either. The five boots of that class span 07:24:50Z to 08:04:41Z. + +A refusal is not a verdict on the account. Run 34016093772 was inside its *own* preflight while that burst happened (its probes run from 07:46:21Z), and its `llama-3.2-11b` probes on the **same two NVIDIA keys** answered ready at 07:50:58.7Z and 07:50:59.0Z — 84 seconds after those keys refused 429 at 07:49:35Z. That boot ended `probed 16 / ready 5`. + +What is **not** measured: whether the ten unspent probes would have found a ready route *inside* the burst itself. No artifact answers it, because nothing records how long a refusal lasts — hence `retry_after_s` below. The pre-`#1949` walk failed similar windows for a different reason (`.github` runs 34006939646 / 34008191123 / 34008575125, 04:24–05:11Z: the same six 429s, then six gemma 404s, `ready 0` at `probed 12`), so the ban is not a regression this amendment invents; it is the ban meeting a 24-candidate list whose tail it can no longer reach. + +**Decision.** A candidate set aside by the account rule is appended to a postponed list in catalog order. Once the first pass ends with the readiness target unmet and probe budget left, the postponed candidates are probed in that order until the budget is spent; no account rule applies in that second pass. A boot that spends all sixteen probes in the first pass is unchanged; the other two classes above gain a second pass. The justification is not that the second pass rescues a burst — that is unmeasured — but that ending a walk under target with probe budget in hand is indefensible when the catalog's tail is where the ready routes live. Constraint 2 holds unchanged: at most sixteen probes per stage, and a silent second-pass probe is bounded by that count, not by a clock (ADR-0003 admits no time rule here). + +**Cost.** The second pass spends probes the walk used to abandon, so it lengthens the boot it rescues and the boot it does not. A refused probe costs about 120 ms. A **silent** one costs up to the full 90 s receive timeout (`#1661` run 34008191123, both NVIDIA keys' `deepseek-v4-pro` probes at 90.06 s and 90.10 s), and the postponed tail is full of them: `google/gemma-4-31b-it` answered `TimeoutError` in 15 of the 19 probes that reached it across these artifacts. The measured burst is therefore not a 1.2-second case — replaying 34016207820's catalog, its second pass would reach both `gemma-4-31b-it` entries, so about 3 minutes — and the worst case is 10 × 90 s ≈ **15 minutes** added to a boot that will still fail, taking a dead window from about 4 minutes to about 19 and holding the runner slot for it. + +**The two-stage path costs more than the free pool's figure.** Whenever a stage lists no more candidates than the probe budget — which is exactly the auto split, 16 free primary and 8 priced fallback — the account rule now saves nothing there, because the second pass re-probes everything it set aside. Measured on a two-account, all-429 auto run: `origin/main` sends 8 requests (4 primary, 4 priced), this design sends 24 (16 primary, 8 priced). The priced stage spends paid credit, so it doubles from 4 probes to 8 in a rate-limited hour. That is accepted for the same reason as the free pool — the priced stage only runs after every free route rejected, and stopping it half-probed is the same defect one layer down — but it is a real, stated cost, not a side effect. + +Two things are deliberately **not** traded away. The second pass never draws on the shared escalation budget (`REVIEW_PREFLIGHT_MAX_ESCALATIONS`, one counter for the whole run, carried into the priced stage by `#1458`): a postponed candidate that answers with the budget-too-small signature is rejected as `escalation_reserved_for_first_pass` rather than escalating, because otherwise candidates the previous design never probed would take escalations from the priced stage that had them, and a two-stage run measurably stops serving a route it used to serve. + +That competes directly with the org's 60-job ceiling work, and `#1949`'s measured benefit ("a dead window fails closed in about 4 minutes and returns the slot") is partly traded back for the chance to reach the catalog tail. It stays inside the probe budget this ADR bounds, `postponed_probed_count` plus the provisioning step's duration make the trade visible per boot, and `REVIEW_PREFLIGHT_MAX_PROBES` is the lever if the census says the exchange is bad. + +The report adds `postponed_probed_count`; `skipped_count` now means "postponed and never reached", and `candidate_count − probed_count − skipped_count` keeps its meaning. A refused probe additionally records `retry_after_s` when the response carried a whole-seconds `Retry-After` header (the HTTP-date form and out-of-range values record nothing). Nothing waits on that value; it exists so the next census can answer the question this amendment could not. + +**Discriminator.** `postponed_probed_count > 0` marks any boot that reached a second pass, which includes the `12 / 12 / 3` class as well as the burst class. To isolate the all-429 class, read the first `probed_count − postponed_probed_count` rows of `routes` (they are in probe order) and require every one to carry `http_status` 429. The next census asks (a) whether such boots end with `ready_count ≥ 1`, (b) what fraction of 429 rows carry `retry_after_s` and how long the refusals claim to last, (c) whether the healthy-minute figures (`ready 5–6`) are unchanged, and (d) the provisioning step's duration on those boots, so the benefit in (a) and the cost above are read from one table. If (a) is consistently 0 **and** (b) shows providers publishing a usable delay, the follow-up is to spend the second pass after that delay rather than immediately — a decision this ADR deliberately leaves to that data. `#1948`'s shared rate ledger remains the lever above all of it. diff --git a/scripts/ci/contextual_orchestrator_review_launcher.py b/scripts/ci/contextual_orchestrator_review_launcher.py index 39c43e24e1..e8c462abcc 100644 --- a/scripts/ci/contextual_orchestrator_review_launcher.py +++ b/scripts/ci/contextual_orchestrator_review_launcher.py @@ -58,23 +58,49 @@ # ``free`` pool lists all 24). A silent candidate's probe costs up to one # transport timeout (19 probes took 805 s in one artifact), so MAX_PROBES # bounds preflight wall time as well as request count. Candidates past the -# probe cap are reached only through the account-skip rule below, and the -# report separates ``skipped_count`` from the unreached tail so the evidence -# stays readable. +# probe cap are reached only when the account rule below sets earlier ones +# aside, and the report separates ``skipped_count`` (set aside, never probed) +# from the unreached tail so the evidence stays readable. REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES = 24 REVIEW_PREFLIGHT_PRIMARY_ROUTE_LIMIT = 16 REVIEW_PREFLIGHT_TARGET_READY = 8 REVIEW_PREFLIGHT_MAX_PROBES = 16 -# A 429 at preflight is a per-key answer, not a per-model one: once one -# credential account has answered 429 to this many probes in a row, its -# remaining candidates are skipped without a probe and the walk moves on to -# the other accounts' next candidates. Under the real 2026-09-06 candidate -# order (jan's table on #1949) the round-robin would otherwise spend five of -# sixteen probes on an account whose every free route had answered 429 in -# every artifact since 21:00Z, and the readiness target was unreachable; with -# the skip the same sixteen probes reach both keys' llama routes. The two -# probed routes are still deferred (#1947); a skipped candidate is neither -# probed nor served. +# Once one credential account has answered 429 to this many probes in a row, +# its remaining candidates are set aside so the walk reaches the other +# accounts' next candidates first: under the real 2026-09-06 candidate order +# (jan's table on #1949) the round-robin would otherwise spend five of sixteen +# probes on an account whose every free route answered 429, and the readiness +# target was unreachable; setting them aside lets the same sixteen probes +# reach both keys' llama routes (catalog position 17, ready in eight of the +# fourteen merged-rule artifacts of 2026-09-06 and reached only this way). +# +# But the rule must not END the walk. When every account is set aside the walk +# stops with most of its probe budget unspent and the stage fails closed -- +# and because deferral needs one ready route (#1947), nothing is served +# either. Measured that day: `.github` run 34016207820 sent six probes across +# all three accounts between 07:49:35.111Z and 07:49:35.767Z, every one +# refused 429, and gave up with ten probes unspent; five runs between 07:24Z +# and 08:05Z read probed 6 / skipped 18 / ready 0. Because the walk is a +# round-robin, "two consecutive 429s" on one account is two requests about +# 310 ms apart (nvidia_nim at .111 and .422). +# +# A refusal is not a verdict on the account. Run 34016093772 was inside its +# own preflight during that burst, and its llama probes on the same two NVIDIA +# keys answered ready at 07:50:58.7 and 07:50:59.0 -- 84 s after those keys +# refused 429. Whether the unspent probes would find a ready route *inside* a +# burst is still unmeasured; that is what `retry_after_s` is for. What is +# certain is that failing closed with two thirds of the budget in hand is +# indefensible, and the cost of spending it is bounded by the probe count, not +# a clock: a refused probe costs about 120 ms, a silent one up to the 90 s +# receive timeout, and the postponed tail contains both (google/gemma-4-31b-it +# answered TimeoutError in 15 of the 19 probes that reached it). See ADR-0029's +# amendment for the full cost table. +# +# So a set-aside candidate is postponed, not banned: once the first pass ends +# with the target unmet and probes left, the postponed candidates are probed +# in catalog order until the budget is spent. Probed 429 routes are still +# deferred (#1947); a candidate the budget never reaches is neither probed nor +# served. REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429 = 2 # ADR-0005: a single fixed max_tokens cannot fit every model in a heterogeneous # pool -- some spend internal reasoning tokens before visible content and need @@ -268,6 +294,46 @@ def _safe_http_status(exc: Exception) -> int | None: return None +def _safe_retry_after_seconds(exc: Exception) -> int | None: + """Return the response's ``Retry-After`` delay in whole seconds, if it sent one. + + Recorded so the evidence can answer a question this codebase cannot + answer today: when a preflight probe is refused with 429, do the + providers say how long the refusal lasts? The 2026-09-06 artifacts show + every probe of a burst refused inside a second (`.github` 34016207820 and + four sibling boots), with nothing in the evidence about how long the + refusal window actually was. Only the delta-seconds + form is read; the HTTP-date form and anything out of range record + nothing, because a wrong number here would be worse than no number. + This is evidence only -- no code waits on it (ADR-0003). + + Args: + exc: The exception a probe attempt raised. + + Returns: + The delay in seconds, or ``None`` when the response carried no + usable ``Retry-After`` header. + """ + headers = getattr(exc, "headers", None) + get_header = getattr(headers, "get", None) + if not callable(get_header): + return None + try: + raw = get_header("Retry-After") + except Exception: # noqa: BLE001 - a hostile header mapping is not evidence + return None + # ``isdecimal`` rather than ``isdigit``: a provider controls this header, + # and ``"²".isdigit()`` is True while ``int("²")`` raises. This + # runs inside the probe walk's exception handler, so a ValueError here + # would escape ``_preflight_review_agents`` -- whose callers catch only + # ``ReviewPreflightError`` -- and kill the boot before any evidence file + # is written. Every ``isdecimal`` string is accepted by ``int``. + if not isinstance(raw, str) or not raw.strip().isdecimal(): + return None + seconds = int(raw.strip()) + return seconds if 0 <= seconds <= 86400 else None + + def _response_finish_reason(response: object) -> str | None: """Return a bounded ``finish_reason`` string from an OpenAI-compatible response. @@ -329,6 +395,9 @@ def _record_provider_exception(row: dict[str, object], exc: Exception) -> None: http_status = _safe_http_status(exc) if http_status is not None: row["http_status"] = http_status + retry_after = _safe_retry_after_seconds(exc) + if retry_after is not None: + row["retry_after_s"] = retry_after row.pop("finish_reason", None) row.pop("reasoning_without_content", None) @@ -444,10 +513,13 @@ def _preflight_review_agents( candidate costs one probe rather than a served slot and a healthy pool is not probed to exhaustion. An account that has answered 429 to ``REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429`` consecutive probes has its - remaining candidates skipped without a probe (a 429 is a per-key answer). - Unprobed candidates get no ``routes`` row; ``skipped_count`` counts the - skipped ones and ``candidate_count - probed_count - skipped_count`` the - unreached tail. + remaining candidates postponed behind the other accounts' candidates; + once the first pass ends with the target unmet and budget left, the + postponed candidates are probed in catalog order (a 429 is an answer + about the instant, not the account). Unprobed candidates get no + ``routes`` row; ``skipped_count`` counts the postponed candidates the + budget never reached, ``postponed_probed_count`` the ones it did, and + ``candidate_count - probed_count - skipped_count`` the unreached tail. Args: agents: Selected zero-cost model agents. @@ -468,17 +540,36 @@ def _preflight_review_agents( viable: list[object] = [] routes: list[dict[str, object]] = [] consecutive_429: dict[str, int] = {} - skipped = 0 + # Candidates the account rule set aside in the first pass, in catalog + # order. They are probed in a second pass while budget is left and the + # target is unmet; the ones that pass never reaches are the skipped ones. + postponed: list[object] = [] + postponed_probed = 0 # One entry per probe, in probe order: ``routes[i]`` describes - # ``probed[i]``. Skipped candidates appear in neither, so the deferral pass - # below must pair rows with this list, not with ``agents``. + # ``probed[i]``. A postponed candidate joins both only when its probe + # runs, so the deferral pass below must pair rows with this list, not + # with ``agents``. probed: list[object] = [] - for agent in agents: + walk = iter(agents) + second_pass = False + # A dedicated sentinel, not ``None``: ``None`` is a legal element of a + # candidate list and would silently truncate the walk. + exhausted = object() + while True: if len(viable) >= REVIEW_PREFLIGHT_TARGET_READY or len(routes) >= REVIEW_PREFLIGHT_MAX_PROBES: break + agent = next(walk, exhausted) + if agent is exhausted: + if second_pass or not postponed: + break + walk = iter(postponed) + second_pass = True + continue account = provider_account(str(getattr(agent, "provider_name", "") or "unknown")) - if consecutive_429.get(account, 0) >= REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429: - skipped += 1 + if second_pass: + postponed_probed += 1 + elif consecutive_429.get(account, 0) >= REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429: + postponed.append(agent) continue # Cleared here; only a 429 answer below restores it, incremented. streak_429 = consecutive_429.pop(account, 0) @@ -554,11 +645,27 @@ def _preflight_review_agents( # a specific policy without real telemetry on which candidates # actually need escalation would itself be the kind of unjustified # heuristic this design rejects elsewhere. - if not budget_signature or escalations_used >= REVIEW_PREFLIGHT_MAX_ESCALATIONS: + # The second pass never draws on the shared escalation budget. That + # budget is one counter for the whole run, spent in catalog order and + # carried into the priced fallback stage (#1458). Candidates in the + # second pass are ones the account rule had set aside and the previous + # design never probed at all, so letting them claim escalations would + # take them from stages that had them before: measured on a two-stage + # run where every primary candidate on one account answered 429, the + # priced fallback candidate that needs its escalation is denied one and + # the run stops serving a route it used to serve. + if ( + not budget_signature + or second_pass + or escalations_used >= REVIEW_PREFLIGHT_MAX_ESCALATIONS + ): row["status"] = "rejected" - row["error_type"] = ( - "invalid_chat_response" if not budget_signature else "escalation_budget_exhausted" - ) + if not budget_signature: + row["error_type"] = "invalid_chat_response" + elif second_pass: + row["error_type"] = "escalation_reserved_for_first_pass" + else: + row["error_type"] = "escalation_budget_exhausted" routes.append(row) continue escalations_used += 1 @@ -612,7 +719,8 @@ def _preflight_review_agents( # _preflight_with_fallback's "priced catalog only after every primary # route rejects" contract (ADR-0005) is unchanged. ``routes`` holds one # row per *probed* agent in probe order (every branch above appends once), - # and ``probed`` the matching agents -- skipped candidates are in neither. + # and ``probed`` the matching agents -- a postponed candidate is in both + # once its second-pass probe has run, and in neither otherwise. deferred: list[object] = [] if viable: for agent, row in zip(probed, routes): @@ -629,7 +737,8 @@ def _preflight_review_agents( "ready_count": len(viable), "deferred_count": len(deferred), "rejected_count": len(routes) - len(viable) - len(deferred), - "skipped_count": skipped, + "skipped_count": len(postponed) - postponed_probed, + "postponed_probed_count": postponed_probed, "target_ready": REVIEW_PREFLIGHT_TARGET_READY, "probe_budget": REVIEW_PREFLIGHT_MAX_PROBES, "account_skip_after_429": REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429, diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index 0944c7ed2f..7f92162097 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -2102,9 +2102,127 @@ def _preflight_agents(*ids: str) -> list[SimpleNamespace]: class _StatusError(Exception): """Exception with a ``code`` attribute, the shape ``_safe_http_status`` reads.""" - def __init__(self, code: int) -> None: + def __init__(self, code: int, headers: object | None = None) -> None: super().__init__(f"HTTP Error {code}") self.code = code + if headers is not None: + self.headers = headers + + +@pytest.mark.parametrize( + ("headers", "expected"), + [ + ({"Retry-After": "37"}, 37), + ({"Retry-After": " 60 "}, 60), + ({"Retry-After": "0"}, 0), + ({"Retry-After": "Wed, 06 Sep 2026 08:00:00 GMT"}, None), + # "²".isdigit() is True but int("²") raises; the header is provider + # controlled and this runs inside the probe walk's exception handler, + # so an unguarded int() would kill the boot before any evidence file + # is written. Reached in production: HTTPError.headers decodes + # iso-8859-1, so byte 0xB2 arrives as this string. + ({"Retry-After": "²"}, None), + ({"Retry-After": "¹²"}, None), + # Arabic-Indic digits are decimal, so int() does parse them. + ({"Retry-After": "٣٠"}, 30), + ({"Retry-After": "-5"}, None), + ({"Retry-After": "999999"}, None), + ({"Retry-After": ""}, None), + ({}, None), + (None, None), + ("not-a-mapping", None), + ], +) +def test_preflight_records_only_a_usable_retry_after_delay( + headers: object | None, expected: int | None +) -> None: + """``retry_after_s`` records whole delta-seconds and nothing else. + + A 429 at preflight says nothing today about how long the refusal lasts + (`.github` run 34016207820 and `keyverse` #143 both refused every probe + inside a second). The delta-seconds form is recorded as evidence; the + HTTP-date form, out-of-range values and a hostile header object record + nothing, because a wrong number would be worse than no number. No code + waits on the value. + """ + namespace = _load_launcher() + row: dict[str, object] = {} + + namespace["_record_provider_exception"](row, _StatusError(429, headers)) + + assert row.get("retry_after_s") == expected + assert (row["status"], row["http_status"]) == ("rejected", 429) + + +def test_preflight_second_pass_does_not_spend_the_shared_escalation_budget() -> None: + """A postponed candidate never claims an escalation the priced stage still needs. + + ``escalations_used`` is one counter for the whole run, carried into the + priced fallback stage (#1458). Second-pass candidates are ones the account + rule had set aside and the previous design never probed, so letting them + escalate would take escalations from stages that had them before. Here the + first pass sets an account aside, and the postponed candidates all answer + with the budget-too-small signature: without the reservation each would + escalate and drain the shared budget. + """ + namespace = _load_launcher() + preflight = namespace["_preflight_review_agents"] + + def agent(account: str, index: int) -> SimpleNamespace: + return SimpleNamespace(id=f"{account}{index}", provider_name=account, model=f"{account}/m{index}", priority=0) + + agents = [agent("X", 1), agent("X", 2), agent("Y", 1), agent("X", 3), agent("X", 4)] + too_small = {"choices": [{"finish_reason": "length", "message": {"content": ""}}]} + client = _ProbeClient( + { + "X1": _StatusError(429), + "X2": _StatusError(429), + "Y1": _openai_text("OK"), + "X3": too_small, + "X4": too_small, + } + ) + + served, report = preflight(agents, client=client) + + assert [call[0].id for call in client.calls] == ["X1", "X2", "Y1", "X3", "X4"] + assert report["escalations_used"] == 0 + second_pass = report["routes"][3:] + assert [row["error_type"] for row in second_pass] == [ + "escalation_reserved_for_first_pass", + "escalation_reserved_for_first_pass", + ] + assert [row["attempts"] for row in second_pass] == [1, 1] + assert [a.id for a in served][:1] == ["Y1"] + + +def test_preflight_walk_treats_a_none_candidate_as_a_candidate() -> None: + """Exhaustion is a dedicated sentinel, so a ``None`` entry cannot truncate the walk.""" + namespace = _load_launcher() + preflight = namespace["_preflight_review_agents"] + agents: list[object] = [None, SimpleNamespace(id="B", provider_name="b", model="b/m", priority=0)] + client = _ProbeClient({"": _StatusError(404), "B": _openai_text("OK")}) + + served, report = preflight(agents, client=client) + + assert report["probed_count"] == 2 + assert [a.id for a in served] == ["B"] + + +def test_preflight_retry_after_survives_a_raising_header_mapping() -> None: + """A header mapping that raises is not evidence and never breaks the probe walk.""" + namespace = _load_launcher() + + class _HostileHeaders: + def get(self, name: str) -> str: + raise RuntimeError(name) + + row: dict[str, object] = {} + + namespace["_record_provider_exception"](row, _StatusError(429, _HostileHeaders())) + + assert "retry_after_s" not in row + assert row["status"] == "rejected" def test_preflight_defers_transient_probe_statuses_behind_ready_routes() -> None: @@ -2283,18 +2401,22 @@ def test_preflight_probe_budget_bounds_a_dead_hour() -> None: assert (report["rejected_count"], report["deferred_count"], report["skipped_count"]) == (budget, 0, 0) -def test_preflight_skips_an_account_after_consecutive_429s() -> None: - """A rate-limited hour costs two probes per account, not the whole budget. +def test_preflight_postpones_a_rate_limited_account_and_spends_the_leftover_budget() -> None: + """Two 429s set an account aside; the leftover budget is then spent on the postponed candidates. - A 429 at preflight is a per-key answer. Once one credential account has - answered 429 to REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429 probes in a row, its - remaining candidates are skipped without a probe. With every account - rate-limited the walk ends after two probes per account and the stage - fails as before (no route is ready, so nothing is deferred either). + With every account answering 429 the first pass ends after two probes per + account with ten of sixteen probes unspent. The merged rule stopped the walk + there and failed the stage with the budget unused (2026-09-06 07:49:35Z: + six 429s within 656 ms across all three accounts, `.github` run + 34016207820). Now the postponed candidates are probed in catalog order + until the budget is spent; the stage still fails when nothing answers, and + the report says how many postponed candidates were probed and how many + were never reached. """ namespace = _load_launcher() preflight = namespace["_preflight_review_agents"] skip_after = namespace["REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429"] + budget = namespace["REVIEW_PREFLIGHT_MAX_PROBES"] accounts = ("nvidia_nim", "nvidia_nim_sub", "openrouter") agents = [ SimpleNamespace(id=f"{account}_{index}", provider_name=account, model=f"{account}/m{index}", priority=-index) @@ -2307,13 +2429,62 @@ def test_preflight_skips_an_account_after_consecutive_429s() -> None: preflight(agents, client=client) report = failure.value.report - assert len(client.calls) == skip_after * len(accounts) == 6 - assert report["probed_count"] == 6 - assert report["skipped_count"] == len(agents) - 6 + first_pass = skip_after * len(accounts) + assert len(client.calls) == report["probed_count"] == budget == 16 + # First pass: two probes per account in catalog order; second pass: the + # postponed candidates in catalog order until the budget is spent. + assert [call[0].id for call in client.calls] == [agent.id for agent in agents[:budget]] + assert report["postponed_probed_count"] == budget - first_pass == 10 + assert report["skipped_count"] == len(agents) - budget == 8 assert report["account_skip_after_429"] == skip_after - assert [call[0].id for call in client.calls] == [ - agent.id for agent in agents[: skip_after * len(accounts)] + assert (report["ready_count"], report["deferred_count"], report["rejected_count"]) == (0, 0, budget) + + +def test_preflight_burst_of_429s_does_not_end_the_walk_before_a_ready_route() -> None: + """A refusal on every account's first candidates no longer hides a ready route further down the catalog. + + `.github` run 34016207820's six probes all answered 429 within 656 ms and + the merged rule gave up there, ten probes unspent. This test does NOT + claim those ten probes would have succeeded in that run -- that is + unmeasured, and `retry_after_s` was added to find out. It pins the + behaviour the rule owes the caller: when the refusals do not extend to + every candidate (one artifact shows two models on one key answering 529 + and ready in the same minute), the leftover budget reaches the route that + answers. Under the artifact order with both keys' deepseek routes + refusing, the sixteenth probe reaches the first llama route and the stage + serves it with the refused routes deferred behind it. + + It also pins the cost ADR-0029 bounds: two of the ten second-pass probes + land on the silent `gemma-4-31b` entries, each of which can hold the full + receive timeout in production. The budget, not a clock, is what limits it. + """ + namespace = _load_launcher() + preflight = namespace["_preflight_review_agents"] + budget = namespace["REVIEW_PREFLIGHT_MAX_PROBES"] + agents, outcomes = _artifact_order_candidates() + for agent in agents: + if "deepseek" in agent.model: + outcomes[agent.id] = _StatusError(429) + client = _ProbeClient(outcomes) + + served, report = preflight(agents, client=client) + + probed_ids = [call[0].id for call in client.calls] + assert probed_ids[:6] == [agent.id for agent in agents[:6]] + assert len(probed_ids) == report["probed_count"] == budget + assert probed_ids[-1] == "nvidia_nim_llama-3.2-11b" + assert report["postponed_probed_count"] == budget - 6 + assert report["skipped_count"] == len(agents) - budget + assert (report["ready_count"], report["deferred_count"], report["rejected_count"]) == (1, 9, 6) + second_pass = report["routes"][6:] + silent = [row for row in second_pass if row.get("error_type") == "TimeoutError"] + assert [row["agent_id"] for row in silent] == [ + "nvidia_nim_gemma-4-31b", + "nvidia_nim_sub_gemma-4-31b", ] + assert [agent.id for agent in served][:1] == ["nvidia_nim_llama-3.2-11b"] + # Every deferred route ranks behind the one ready route. + assert max(agent.priority for agent in served[1:]) < served[0].priority def _artifact_order_candidates() -> tuple[list[SimpleNamespace], dict[str, object]]: @@ -2321,9 +2492,23 @@ def _artifact_order_candidates() -> tuple[list[SimpleNamespace], dict[str, objec Two NVIDIA keys list the same models alphabetically -- two deepseek routes, the two gemma-3 entries that answer 404 on every run, a gemma-4 entry that - answers an empty completion, then the llama and muse routes that were ready - in every pre-#1939 artifact -- and every OpenRouter free route answers 429. - The catalog interleaves the three accounts tier-round-robin, eight each. + goes *silent* (`google/gemma-4-31b-it` answered ``TimeoutError`` in 15 of + the 19 probes that reached it across the 2026-09-06 artifacts, so modelling + it as an instant empty completion hid the dominant cost of walking the + catalog tail), then the llama and muse routes that were ready in every + pre-#1939 artifact -- and every OpenRouter free route answers 429. The + catalog interleaves the three accounts tier-round-robin, eight each. + + KNOWN OPTIMISM, deliberately left alone here: the artifacts also show + `meta/llama-3.2-90b-vision-instruct` answering ``TimeoutError`` on both + keys in every probe that reached it (17 of 17), while this fixture answers + it OK. Correcting that drops + ``test_preflight_reaches_both_keys_llama_routes_under_the_artifact_order`` + below its `ready_count == REVIEW_PREFLIGHT_TARGET_READY` assertion -- which + matches production, where no 2026-09-06 artifact ever reached eight ready + routes (the best was six). That is a question about #1949's readiness + target, not about postponement, so it is raised on #1948 rather than + changed under this PR. """ nvidia_models = [ "deepseek-v4-flash", @@ -2360,7 +2545,7 @@ def _artifact_order_candidates() -> tuple[list[SimpleNamespace], dict[str, objec elif model.startswith("gemma-3"): outcomes[agent.id] = _StatusError(404) elif model.startswith("gemma-4"): - outcomes[agent.id] = {"choices": [{"finish_reason": "stop", "message": {"content": ""}}]} + outcomes[agent.id] = TimeoutError("read timed out") else: outcomes[agent.id] = _openai_text("OK") return agents, outcomes @@ -2396,11 +2581,13 @@ def test_preflight_reaches_both_keys_llama_routes_under_the_artifact_order() -> def test_preflight_deferral_pairs_rows_with_probed_agents_after_skips() -> None: - """After an account is skipped, deferred rows still map to the agents that were probed. + """After an account is set aside, deferred rows still map to the agents that were probed. - Order: X answers 429 twice (then is skipped), Y is ready, Z answers 429 - once after X's skips began. Pairing rows with the original agent list - would demote the skipped X candidates instead of Z. + Order: X answers 429 twice (then is postponed), Y is ready, Z answers 429 + once after X's postponement began; the second pass probes X3..X5 with the + leftover budget, so the row list runs X1 X2 Y1 Z1 Y2 Z2 Y3 X3 X4 X5 while + the catalog runs X1 X2 Y1 X3 Z1 Y2 X4 Z2 X5 Y3. Pairing rows with the + catalog would demote the wrong candidates from the fourth row on. """ namespace = _load_launcher() preflight = namespace["_preflight_review_agents"] @@ -2422,9 +2609,12 @@ def agent(account: str, index: int) -> SimpleNamespace: served, report = preflight(agents, client=client) - assert [call[0].id for call in client.calls] == ["X1", "X2", "Y1", "Z1", "Y2", "Z2", "Y3"] - assert (report["ready_count"], report["deferred_count"], report["skipped_count"]) == (4, 3, 3) - assert [a.id for a in served] == ["Y1", "Y2", "Z2", "Y3", "X1", "X2", "Z1"] + assert [call[0].id for call in client.calls] == [ + "X1", "X2", "Y1", "Z1", "Y2", "Z2", "Y3", "X3", "X4", "X5", + ] + assert (report["ready_count"], report["deferred_count"], report["skipped_count"]) == (4, 6, 0) + assert report["postponed_probed_count"] == 3 + assert [a.id for a in served] == ["Y1", "Y2", "Z2", "Y3", "X1", "X2", "Z1", "X3", "X4", "X5"] assert all(a.priority == -namespace["REVIEW_PREFLIGHT_DEFERRED_PRIORITY_PENALTY"] for a in served[4:]) From c232ca03ef891644e5397313b87849cf9e5a25ac Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 20:51:11 +0900 Subject: [PATCH 23/80] fix(strix): sanitize strix-agent's recovered transient replay warning (#1959) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A completed scan (run.json completed, SARIF 0 results, attempt exit 0) was failed closed as STRIX_PROVIDER_UNAVAILABLE on .github#1689 run 34013778497 because three `strix.core.execution: transient model/provider error for ; replaying turn (attempt n/m, backoff Ns): …` WARNING lines survived sanitize_known_strix_report_warnings and tripped the report WARNING scan. strix-agent 1.5.3 emits that line only inside its bounded transient-retry branch (strix/core/execution.py:763), immediately before the replay runs; an exhausted retry logs `agent run failed for …; marking failed` at ERROR with a traceback and exits non-zero, and both of those still fail the gate. Two tests cover the production argument shape, where the reports root is passed and has_strix_report_failure_signal narrows to the newest run directory via latest_strix_report_dir, so the sanitized tree and the scanned tree are demonstrably the same one. The CHANGELOG records one side effect: a provider 503 body that appears only inside a retry line's exception repr is removed with that line, which can make the report-only branch of is_model_retryable_error read an outage as non-retryable. The direction is fail-closed and the contextual-orchestrator verdict branch answers first, so no path changes outcome today. Coupled to the strix-agent 1.5.3 execution.py:763 message format, like the two existing alternatives — re-verify on every strix-agent bump. Co-authored-by: Claude Opus 5 --- CHANGELOG.md | 4 + scripts/ci/strix_quick_gate.sh | 15 +- ...est_strix_recovered_transient_sanitizer.py | 284 ++++++++++++++++++ 3 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 tests/test_strix_recovered_transient_sanitizer.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dca548fb03..f03bd5104c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Strix gate keeps a recovered transient model error from failing a completed scan + +- `scripts/ci/strix_quick_gate.sh` `sanitize_known_strix_report_warnings` now also strips strix-agent's `strix.core.execution: transient model/provider error for ; replaying turn (attempt n/m, backoff Ns): …` WARNING lines before the report failure-signal scan. strix-agent 1.5.3 (`strix/core/execution.py:763`) emits that line only inside its bounded transient-retry branch, immediately before the replay runs; an exhausted retry logs `agent run failed for …; marking failed` at ERROR with a traceback and exits non-zero, and both of those still fail the gate. Observed on `.github#1689` run `34013778497`: a completed 63-minute scan (`run.json` `completed`, SARIF 0 results, attempt exit 0) was failed closed as `STRIX_PROVIDER_UNAVAILABLE … exhausted` on three such warnings, and the scheduler then dispatched another same-head scan. The pattern is anchored before the exception repr so the same class keeps matching after a gateway pin advance changes the exception type; re-verify the message format on every strix-agent bump. One documented side effect: when a provider's 503 body appears only inside a retry line's exception repr, removing that line also removes the only text `has_strix_report_provider_failure_signal` would have matched in the report log, which can make `is_model_retryable_error`'s report-only branch read a genuine outage as non-retryable. The direction is fail-closed (an exhausted retry still exits non-zero with its ERROR and traceback retained), and with a contextual-orchestrator primary the verdict branch answers before that classifier is consulted, so no path today changes its outcome; if fallback-model classification is ever wanted for a non-gateway primary, read the pre-sanitize attempt copy that `preserve_attempt_log` already keeps. Tests: `tests/test_strix_recovered_transient_sanitizer.py`. + ### Review sidecar preflight postpones a rate-limited account's candidates instead of banning them - `_preflight_review_agents` no longer ends its walk when every credential account has answered 429 twice in a row. A candidate set aside by `REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429` is postponed to the end of the walk, and once the first pass ends with the readiness target unmet and probe budget left, the postponed candidates are probed in catalog order until the sixteen-probe budget is spent. On 2026-09-06 five sidecar boots whose probes began between 07:24Z and 08:05Z read `probed 6 / skipped 18 / ready 0` and failed closed: `.github` run 34016207820's six probes across all three accounts were refused 429 between 07:49:35.111Z and 07:49:35.767Z, so the rule set every account aside on two same-account requests about 310 ms apart and gave up with ten of sixteen probes unspent — and because deferral needs one ready route, nothing was served either; `keyverse#143`'s 08:20Z `noema-review` repeated it in a second repository (six probes, 369 ms, all 429). The pools are not dead in those minutes: run 34016093772 was inside its own preflight during that burst, and its `llama-3.2-11b` probes on the same two NVIDIA keys answered ready at 07:50:58.7Z and 07:50:59.0Z, 84 seconds after those keys refused. Whether the unspent probes would have found a ready route inside a burst is unmeasured and is not claimed; the change is justified by ending a walk under target with the budget in hand. Of the fourteen boots that ran the merged rule, eight spend all sixteen probes in the first pass and are unchanged; one (`argos` 34014143870, a serving boot at `12 / 12 / 3`) exhausts its candidates under budget and now gains a second pass, as do the five burst boots. The cost is stated rather than assumed: a refused probe costs about 120 ms, a silent one up to the 90 s receive timeout, and the postponed tail holds both (`google/gemma-4-31b-it` answered `TimeoutError` in 15 of the 19 probes that reached it), so the worst case adds up to about 15 minutes to a boot that still fails and the two-stage auto path goes from 8 to 24 requests including the priced stage. The second pass never draws on the shared escalation budget, so the priced fallback keeps the escalations it had. The report gains `postponed_probed_count` (`skipped_count` now counts postponed candidates the budget never reached) and, on a refused probe, `retry_after_s` when the response carried a whole-seconds `Retry-After` header — evidence only, nothing waits on it, so the next census can decide whether a delayed second pass is worth proposing. ADR-0029 is amended. Refs #1948, #1949. diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 617ca7c394..c08f2fa36c 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -178,11 +178,24 @@ import sys root = Path(sys.argv[1]) known_internal_warning = re.compile( r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ WARNING " - r"[^ ]+ - strix\.core\.execution: agent [0-9a-f]+ " + r"[^ ]+ - strix\.core\.execution: " + r"(?:" + r"agent [0-9a-f]+ " r"(?:" r"produced non-lifecycle final output in non-interactive mode" r"|ended a turn without a lifecycle tool call \(interactive=False\)" r"); forcing tool continuation \(\d+/\d+\): " + # strix-agent 1.5.3 strix/core/execution.py:763 logs this only inside its + # bounded transient-retry branch, immediately before the replay runs, so + # the line means "a retry is happening now", not "the scan failed". An + # exhausted retry logs `agent run failed for …; marking failed` at ERROR + # with a traceback and exits non-zero; neither of those is touched here. + # Anchored before the exception repr on purpose: the same class appears + # as InternalServerError today and as a different type after a gateway + # pin advance. Re-verify the message format on every strix-agent bump. + r"|transient model/provider error for [0-9a-f]+; replaying turn " + r"\(attempt \d+/\d+, backoff [0-9.]+s\): " + r")" ) known_scanner_warning = re.compile( r"^(?:│ MODEL QUALITY WARNING\s+│|" diff --git a/tests/test_strix_recovered_transient_sanitizer.py b/tests/test_strix_recovered_transient_sanitizer.py new file mode 100644 index 0000000000..28fda47205 --- /dev/null +++ b/tests/test_strix_recovered_transient_sanitizer.py @@ -0,0 +1,284 @@ +"""Regression contract for strix-agent's recovered transient model errors. + +strix-agent 1.5.3 (``strix/core/execution.py:760-763``) retries a transient +model/provider error up to ``_MAX_TRANSIENT_MODEL_RETRIES`` times and, inside +that branch only, logs:: + + WARNING - strix.core.execution: transient model/provider error for + ; replaying turn (attempt n/m, backoff Ns): + +immediately before the replay runs. The line therefore means "a retry is +happening now", never "the scan failed". When the budget is exhausted the same +module logs ``agent run failed for ; marking failed`` at ERROR with a +traceback and the process exits non-zero. + +Observed on ContextualWisdomLab/.github#1689 run ``34013778497``: a completed +63-minute scan (``run.json`` status ``completed``, SARIF 0 results, attempt exit +code 0) was failed closed as ``STRIX_PROVIDER_UNAVAILABLE … exhausted`` because +three such WARNING lines survived ``sanitize_known_strix_report_warnings`` and +tripped ``has_strix_report_failure_signal``'s ``WARNING`` scan. + +Negative control, as measured by running this file against ``main``'s gate before +this change: **3 failed, 4 passed.** The three that fail are +``test_recovered_transient_replay_warnings_are_sanitized`` (the lines remain and +the failure signal fires), ``test_production_argument_shape_sanitizes_the_scanned_directory`` +(the same, through the narrowing branch), and +``test_unrecovered_transient_keeps_the_error_and_traceback`` on its first assertion +only, since ``assertNotIn("replaying turn", ...)`` also needs the new alternative +while its ERROR-and-traceback retention assertions hold on both gates. The four +that pass on both gates are the guards: the two unknown-warning cases, the +foreign-module case, and the pre-existing forced-continuation case. +""" + +from __future__ import annotations + +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" + +_PREFIX = "strix-pr-scope-qd1fsv_9ee6 - strix.core.execution: " + +# The three lines exactly as the run above wrote them (490 characters each). +_REPR = ( + "InternalServerError(\"Error code: 500 - {'error': {'code': 'internal_error', " + "'message': 'internal server error', 'detail': {'request_id': '%s'}}, " + "'error_code': 'internal_error', 'error_message': 'internal server error', " + "'error_detail': {'request_id': '%s'}}\")" +) +RECOVERED_LOG = ( + "2026-09-06 07:23:08.199 WARNING " + _PREFIX + + "transient model/provider error for 76d3c83d; replaying turn " + "(attempt 1/5, backoff 2.0s): " + + _REPR % ("466c7aee94e24a6e811cbd7fd12bc1a9", "466c7aee94e24a6e811cbd7fd12bc1a9") + + "\n" + "2026-09-06 07:23:10.205 DEBUG strix-pr-scope-qd1fsv_9ee6 - " + "strix.llm.context_budget: No LiteLLM model info for 'openai/orchestrator/free'; " + "using configured fallbacks\n" + "2026-09-06 07:45:20.154 WARNING " + _PREFIX + + "transient model/provider error for 76d3c83d; replaying turn " + "(attempt 2/5, backoff 4.0s): " + + _REPR % ("6dbf7b28ee16448592e10bb9728a523f", "6dbf7b28ee16448592e10bb9728a523f") + + "\n" + "2026-09-06 07:58:54.623 WARNING " + _PREFIX + + "transient model/provider error for 76d3c83d; replaying turn " + "(attempt 3/5, backoff 8.0s): " + + _REPR % ("a85b9828eb754e129f62d202359ea316", "a85b9828eb754e129f62d202359ea316") + + "\n" + "2026-09-06 08:09:35.584 INFO strix-pr-scope-qd1fsv_9ee6 - " + "strix.core.runner: Strix scan strix-pr-scope-qd1fsv_9ee6 done\n" +) + +# After the bounded budget is spent strix-agent logs at ERROR with a traceback +# (observed on a same-day run) and exits non-zero. The sanitizer must leave it. +UNRECOVERED_LOG = ( + "2026-09-06 07:24:31.010 WARNING strix-pr-scope-5p3h3c_e0d0 - " + "strix.core.execution: transient model/provider error for 6c480eb0; " + "replaying turn (attempt 5/5, backoff 32.0s): InternalServerError(\"Error code: 500\")\n" + "2026-09-06 07:24:40.562 ERROR strix-pr-scope-5p3h3c_e0d0 - " + "strix.core.execution: agent run failed for 6c480eb0; marking failed\n" + "Traceback (most recent call last):\n" + ' File "/opt/hostedtoolcache/Python/3.13.15/x64/lib/python3.13/site-packages/' + 'strix/core/execution.py", line 676, in _run_cycle\n' + " async for event in stream.stream_events():\n" + "openai.InternalServerError: Error code: 500\n" +) + +UNKNOWN_WARNING_LOG = ( + "2026-09-06 07:30:00.000 WARNING strix-pr-scope-qd1fsv_9ee6 - " + "strix.core.execution: transient model/provider error for 76d3c83d; " + "giving up after 5 attempts\n" +) + +# A different module echoing the same words must not be sanitized: the anchor +# is the logger name, not the phrase. +FOREIGN_MODULE_LOG = ( + "2026-09-06 07:30:00.000 WARNING strix-pr-scope-qd1fsv_9ee6 - " + "strix.tools.browser: transient model/provider error for 76d3c83d; " + "replaying turn (attempt 1/5, backoff 2.0s): Timeout\n" +) + +LEGACY_LOG = ( + "2026-06-18 13:08:05.986 WARNING strix-pr-scope-example - strix.core.execution: " + "agent a9fb4033 produced non-lifecycle final output in non-interactive mode; " + "forcing tool continuation (1/3): {'x': 1}\n" + "2026-08-22 09:53:26.193 WARNING strix-pr-scope-example - strix.core.execution: " + "agent 673f770f ended a turn without a lifecycle tool call (interactive=False); " + "forcing tool continuation (2/3): done\n" + "2026-06-18 13:10:44.089 INFO strix-pr-scope-example - strix.tools.finish.tool: " + "finish_scan: completed scan with 0 vulnerability report(s)\n" +) + + +def _function_block(source: str, function_name: str) -> str: + """Return one top-level Bash function, including its closing brace.""" + + match = re.search( + rf"(?ms)^{re.escape(function_name)}\(\) \{{\n.*?^\}}\n", + source, + ) + if match is None: + raise AssertionError(f"missing Bash function: {function_name}") + return match.group(0) + + +def _sanitize_then_signal(log_text: str) -> tuple[str, bool]: + """Run the production sanitizer, then the production failure-signal scan. + + Returns the report log's remaining text and whether + ``has_strix_report_failure_signal`` still fires on it. The report root is a + plain temp directory, so the function's ``STRIX_REPORTS_DIR`` branch + (which resolves the newest run) is not taken and needs no helper. + """ + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + blocks = [ + _function_block(gate_source, name) + for name in ( + "sanitize_known_strix_report_warnings", + "has_strix_report_failure_signal", + ) + ] + with tempfile.TemporaryDirectory(prefix="strix-recovered-transient-") as temp_dir: + report_root = Path(temp_dir) / "strix_runs" / "strix-pr-scope-qd1fsv_9ee6" + report_root.mkdir(parents=True) + log_path = report_root / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + script = "\n".join( + ( + "set -uo pipefail", + 'STRIX_REPORTS_DIR="/nonexistent/strix-reports"', + *blocks, + 'sanitize_known_strix_report_warnings "$1"', + 'if has_strix_report_failure_signal "$1"; then echo signal=1; else echo signal=0; fi', + ) + ) + completed = subprocess.run( + ["bash", "-c", script, "strix-sanitizer", str(report_root)], + check=False, + capture_output=True, + text=True, + ) + remaining = log_path.read_text(encoding="utf-8") + if completed.returncode != 0: + raise AssertionError(f"rc={completed.returncode}\n{completed.stderr}") + return remaining, "signal=1" in completed.stdout + + +def _sanitize_then_signal_production_shape(log_text: str) -> tuple[str, bool]: + """Same sequence with the argument shape production actually uses. + + Production passes ``ACTIVE_REPORTS_DIR``, which equals ``STRIX_REPORTS_DIR``, + so ``has_strix_report_failure_signal`` takes its narrowing branch and scans + only ``latest_strix_report_dir``'s newest run directory. ``_sanitize_then_signal`` + hands in that run directory directly and therefore skips the branch; this + helper covers it, so the pair proves the sanitized tree and the scanned tree + are the same one. + """ + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + blocks = [ + _function_block(gate_source, name) + for name in ( + "sanitize_known_strix_report_warnings", + "has_strix_report_failure_signal", + "latest_strix_report_dir", + "is_preexisting_report_dir", + ) + ] + with tempfile.TemporaryDirectory(prefix="strix-recovered-transient-prod-") as temp_dir: + reports_root = Path(temp_dir) / "reports" + run_dir = reports_root / "strix-pr-scope-qd1fsv_9ee6" + run_dir.mkdir(parents=True) + log_path = run_dir / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + script = "\n".join( + ( + "set -uo pipefail", + f'STRIX_REPORTS_DIR="{reports_root}"', + # Non-empty so "${PREEXISTING_REPORT_DIRS[@]}" is safe under set -u. + 'PREEXISTING_REPORT_DIRS=("/nonexistent/preexisting")', + *blocks, + 'sanitize_known_strix_report_warnings "$STRIX_REPORTS_DIR"', + 'if has_strix_report_failure_signal "$STRIX_REPORTS_DIR"; then echo signal=1; else echo signal=0; fi', + ) + ) + completed = subprocess.run( + ["bash", "-c", script, "strix-sanitizer-prod"], + check=False, + capture_output=True, + text=True, + ) + remaining = log_path.read_text(encoding="utf-8") + if completed.returncode != 0: + raise AssertionError(f"rc={completed.returncode}\n{completed.stderr}") + return remaining, "signal=1" in completed.stdout + + +class StrixRecoveredTransientSanitizerTests(unittest.TestCase): + """Keep a recovered transient model error from failing a completed scan.""" + + def test_recovered_transient_replay_warnings_are_sanitized(self) -> None: + """The three observed lines are removed and the WARNING scan stays quiet.""" + + remaining, signal = _sanitize_then_signal(RECOVERED_LOG) + self.assertNotIn("replaying turn", remaining) + self.assertNotIn("InternalServerError", remaining) + self.assertIn("strix.core.runner: Strix scan strix-pr-scope-qd1fsv_9ee6 done", remaining) + self.assertIn("strix.llm.context_budget", remaining) + self.assertFalse(signal) + + def test_unrecovered_transient_keeps_the_error_and_traceback(self) -> None: + """Only the retry line goes; the ERROR record and its traceback stay for the rc!=0 path.""" + + remaining, _signal = _sanitize_then_signal(UNRECOVERED_LOG) + self.assertNotIn("replaying turn", remaining) + self.assertIn("agent run failed for 6c480eb0; marking failed", remaining) + self.assertIn("Traceback (most recent call last):", remaining) + self.assertIn("openai.InternalServerError: Error code: 500", remaining) + + def test_unknown_execution_warning_still_fails_closed(self) -> None: + """A WARNING from the same logger with a different message is not sanitized.""" + + remaining, signal = _sanitize_then_signal(UNKNOWN_WARNING_LOG) + self.assertEqual(remaining, UNKNOWN_WARNING_LOG) + self.assertTrue(signal) + + def test_same_words_from_another_module_still_fail_closed(self) -> None: + """The anchor is the strix.core.execution logger, not the phrase.""" + + remaining, signal = _sanitize_then_signal(FOREIGN_MODULE_LOG) + self.assertEqual(remaining, FOREIGN_MODULE_LOG) + self.assertTrue(signal) + + def test_production_argument_shape_sanitizes_the_scanned_directory(self) -> None: + """With the reports root passed as production passes it, the narrowed scan is quiet.""" + + remaining, signal = _sanitize_then_signal_production_shape(RECOVERED_LOG) + self.assertNotIn("replaying turn", remaining) + self.assertIn("strix.core.runner: Strix scan strix-pr-scope-qd1fsv_9ee6 done", remaining) + self.assertFalse(signal) + + def test_production_argument_shape_still_fails_closed_on_an_unknown_warning(self) -> None: + """The narrowing branch does not swallow a warning the sanitizer does not know.""" + + remaining, signal = _sanitize_then_signal_production_shape(UNKNOWN_WARNING_LOG) + self.assertEqual(remaining, UNKNOWN_WARNING_LOG) + self.assertTrue(signal) + + def test_existing_forced_continuation_warnings_remain_sanitized(self) -> None: + """The two pre-existing alternatives keep working after the regex restructure.""" + + remaining, signal = _sanitize_then_signal(LEGACY_LOG) + self.assertNotIn("forcing tool continuation", remaining) + self.assertIn("finish_scan: completed scan with 0 vulnerability report(s)", remaining) + self.assertFalse(signal) + + +if __name__ == "__main__": + unittest.main() From dd0b96feded94f66ecf59b25a5a9b58cfc8b4f69 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:03:19 +0900 Subject: [PATCH 24/80] fix(review): name the Strix sandbox class in the failed-check finding (#1960) `opencode-review-dispatch.yml`'s `emit_strix_provider_failure_finding` rendered one fixed finding for every `STRIX_PROVIDER_UNAVAILABLE` line, whose Root cause read "The contextual-orchestrator gateway or its discovered provider pool was unavailable for this run". #1953 had just given the Strix sandbox bootstrap failure its own second verdict token, `STRIX_SANDBOX_UNAVAILABLE`, precisely because that attribution is wrong for it: the sandbox container never reaches its Caido proxy, so the run dies before the gateway serves anything. This consumer re-applied the wrong attribution one step downstream, into the review findings and the failure census -- the misattribution #1953 fixed in the gate, surviving in the reader. The emitter now branches on the second token. A sandbox verdict gets a finding that names Strix's sandbox, states that the verdict does not name the gateway, and tells the reader not to change gateway or provider configuration on its strength. A `STRIX_PROVIDER_UNAVAILABLE` line without the token keeps its existing four lines verbatim, so the gateway class has no regression surface. No test covered this finding text at all before ("gateway or its discovered provider pool" and "provider availability blocked" both matched nothing under tests/). tests/test_opencode_dispatch_strix_sandbox_finding.py runs the production emitter, extracted from the published run block with the existing _extract_run_block harness, and pins three directions: the sandbox token, a gateway failure without it, and evidence carrying no provider-unavailable signal at all. Editing the workflow moves its blob, so REVIEW_DISPATCH_BLOB_SHA in tests/test_pr_review_autofix_nvidia_nim_contract.py is recomputed to 694c04b1ec967fee24412752cca599d69d92438e with git hash-object (lane peer 1 flagged this pin in advance). Gate on this tree: 2931 passed, 1 skipped, 21 subtests; coverage 100% (0 missed); interrogate 100%. Negative control on origin/main's workflow with this test file: 1 failed, 2 passed -- only the sandbox direction fails there. Refs #1953, #1935. Co-authored-by: Claude Opus 5 --- .../workflows/opencode-review-dispatch.yml | 18 ++++ CHANGELOG.md | 4 + ...opencode_dispatch_strix_sandbox_finding.py | 94 +++++++++++++++++++ ...t_pr_review_autofix_nvidia_nim_contract.py | 2 +- 4 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 tests/test_opencode_dispatch_strix_sandbox_finding.py diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index fd0e5ff75d..d86497b3f4 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -5636,6 +5636,24 @@ jobs: fi finding_index=$((finding_index + 1)) + + # The gate emits a second token for the class it can tell apart: + # STRIX_SANDBOX_UNAVAILABLE means Strix's own sandbox container + # never reached its Caido proxy, so the run died before the + # gateway served anything. Reporting that as "the gateway or its + # provider pool was unavailable" sends the reader to the wrong + # component -- the misattribution #1953 fixed in the gate itself, + # which survived here because this text was fixed for every + # STRIX_PROVIDER_UNAVAILABLE line. + if grep -q "STRIX_SANDBOX_UNAVAILABLE" "$strix_evidence_file"; then + printf '### %s. HIGH %s:%s - Strix sandbox bootstrap blocked current-head security evidence\n' "$finding_index" "$path" "$line" + printf -- '- Problem: Strix failed before producing vulnerability reports. The failed log reported STRIX_SANDBOX_UNAVAILABLE, which the gate emits when the run ended in Strix sandbox bootstrap after its bounded sandbox-specific retries.\n' + printf -- '- Root cause: Strix sandbox container did not reach its Caido proxy on 127.0.0.1, so the scan ended before any Vulnerability Report window was produced. This verdict names Strix sandbox, not the contextual-orchestrator gateway, and there is no application source line to patch from this evidence.\n' + printf -- '- Fix: Do not approve from this failed scan. Re-run Strix; the sandbox bootstrap is a startup race and the gate already retries it once. Do not change gateway or provider configuration on the strength of this finding.\n' + printf -- '- Regression test: Keep the gate emitting STRIX_SANDBOX_UNAVAILABLE for sandbox bootstrap failures and keep this consumer reading it, so a sandbox outage is never reported as a gateway outage.\n\n' + return 0 + fi + printf '### %s. HIGH %s:%s - Contextual-orchestrator provider availability blocked current-head security evidence\n' "$finding_index" "$path" "$line" printf -- '- Problem: Strix failed before producing vulnerability reports. The failed log reported LLM CONNECTION FAILED, RateLimitError or Too many requests, budget-limit output, gateway exhaustion, and Configured model and fallback models were unavailable.\n' printf -- '- Root cause: The contextual-orchestrator gateway or its discovered provider pool was unavailable for this run; no Strix Vulnerability Report window was produced, so there is no application source line to patch from this evidence.\n' diff --git a/CHANGELOG.md b/CHANGELOG.md index f03bd5104c..bf192f6a9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Failed-check finding names the Strix sandbox instead of the gateway + +- `opencode-review-dispatch.yml`'s `emit_strix_provider_failure_finding` rendered one fixed finding for every `STRIX_PROVIDER_UNAVAILABLE` line, whose Root cause read "The contextual-orchestrator gateway or its discovered provider pool was unavailable for this run". `#1953` had just given the Strix sandbox bootstrap failure its own second verdict token (`STRIX_SANDBOX_UNAVAILABLE`) precisely because that attribution is wrong for it -- the sandbox container never reaches its Caido proxy, so the run dies before the gateway serves anything -- and this consumer re-applied the wrong attribution one step downstream, into the review findings and the failure census. The emitter now branches on the second token: a sandbox verdict gets a finding that names Strix's sandbox, says the verdict does not name the gateway, and tells the reader not to change gateway or provider configuration on its strength. A `STRIX_PROVIDER_UNAVAILABLE` line without the token keeps its existing text verbatim, so the gateway class has no regression surface. No test covered this finding text at all before (`gateway or its discovered provider pool` matched nothing under `tests/`); `tests/test_opencode_dispatch_strix_sandbox_finding.py` now runs the production emitter from the published run block and pins both directions plus the no-signal case. Refs #1953, #1935. + ### Strix gate keeps a recovered transient model error from failing a completed scan - `scripts/ci/strix_quick_gate.sh` `sanitize_known_strix_report_warnings` now also strips strix-agent's `strix.core.execution: transient model/provider error for ; replaying turn (attempt n/m, backoff Ns): …` WARNING lines before the report failure-signal scan. strix-agent 1.5.3 (`strix/core/execution.py:763`) emits that line only inside its bounded transient-retry branch, immediately before the replay runs; an exhausted retry logs `agent run failed for …; marking failed` at ERROR with a traceback and exits non-zero, and both of those still fail the gate. Observed on `.github#1689` run `34013778497`: a completed 63-minute scan (`run.json` `completed`, SARIF 0 results, attempt exit 0) was failed closed as `STRIX_PROVIDER_UNAVAILABLE … exhausted` on three such warnings, and the scheduler then dispatched another same-head scan. The pattern is anchored before the exception repr so the same class keeps matching after a gateway pin advance changes the exception type; re-verify the message format on every strix-agent bump. One documented side effect: when a provider's 503 body appears only inside a retry line's exception repr, removing that line also removes the only text `has_strix_report_provider_failure_signal` would have matched in the report log, which can make `is_model_retryable_error`'s report-only branch read a genuine outage as non-retryable. The direction is fail-closed (an exhausted retry still exits non-zero with its ERROR and traceback retained), and with a contextual-orchestrator primary the verdict branch answers before that classifier is consulted, so no path today changes its outcome; if fallback-model classification is ever wanted for a non-gateway primary, read the pre-sanitize attempt copy that `preserve_attempt_log` already keeps. Tests: `tests/test_strix_recovered_transient_sanitizer.py`. diff --git a/tests/test_opencode_dispatch_strix_sandbox_finding.py b/tests/test_opencode_dispatch_strix_sandbox_finding.py new file mode 100644 index 0000000000..ae73576cb0 --- /dev/null +++ b/tests/test_opencode_dispatch_strix_sandbox_finding.py @@ -0,0 +1,94 @@ +"""The failed-check finding must name Strix sandbox when the gate named it. + +`#1953` gave the Strix sandbox bootstrap failure its own verdict token, +`STRIX_SANDBOX_UNAVAILABLE`, precisely because reporting it as +`contextual-orchestrator/orchestrator/free exhausted` sent readers to a +component the run never reached. This consumer rendered one fixed finding for +every `STRIX_PROVIDER_UNAVAILABLE` line, so the corrected verdict was being +re-attributed to the gateway one step downstream, and no test covered the text +at all. These tests pin both directions. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +from tests.test_opencode_workflow_shell_syntax import _extract_run_block + +WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") +STEP_NAME = "Publish OpenCode review outcome" +FUNCTION = "emit_strix_provider_failure_finding" + + +def _emitter_source() -> str: + """Return the emitter function's shell source from the published run block.""" + script = _extract_run_block(WORKFLOW.read_text(encoding="utf-8"), STEP_NAME) + start = script.index(f"{FUNCTION}() {{") + # ``_extract_run_block`` dedents the YAML block scalar, leaving the + # function body at two spaces and its closing brace on a line of its own. + closing = "\n }\n" + end = script.index(closing, start) + len(closing) + return script[start:end] + + +def _run_emitter(evidence: str, tmp_path: Path) -> str: + """Run the production emitter against one evidence file and return its finding text.""" + evidence_file = tmp_path / "strix-evidence.txt" + evidence_file.write_text(evidence, encoding="utf-8") + harness = tmp_path / "harness.sh" + harness.write_text( + "set -euo pipefail\n" + f'strix_evidence_file="{evidence_file}"\n' + f'repo_root="{tmp_path}"\n' + "finding_index=0\n" + f"{_emitter_source()}\n" + f"{FUNCTION}\n", + encoding="utf-8", + ) + result = subprocess.run( + ["bash", str(harness)], capture_output=True, text=True, check=True + ) + return result.stdout + + +def test_sandbox_token_reports_the_sandbox_not_the_gateway(tmp_path: Path) -> None: + """A `STRIX_SANDBOX_UNAVAILABLE` verdict never blames the gateway or its provider pool.""" + finding = _run_emitter( + "STRIX_PROVIDER_UNAVAILABLE: STRIX_SANDBOX_UNAVAILABLE: the last Strix " + "attempt ended in the sandbox bootstrap (Caido proxy on 127.0.0.1 " + "unreachable through Strix's loginAsGuest attempts) after 1 " + "sandbox-specific same-model retries (budget 1); this verdict names " + "Strix's sandbox, not the LLM gateway.\n", + tmp_path, + ) + + assert "Strix sandbox bootstrap blocked current-head security evidence" in finding + assert "STRIX_SANDBOX_UNAVAILABLE" in finding + assert "names Strix sandbox, not the contextual-orchestrator gateway" in finding + assert "gateway or its discovered provider pool was unavailable" not in finding + # The reader must not be sent to change gateway configuration. + assert "Do not change gateway or provider configuration" in finding + assert finding.startswith("### 1. HIGH .github/workflows/strix.yml:") + + +def test_gateway_failure_keeps_its_existing_finding(tmp_path: Path) -> None: + """Without the sandbox token the previous gateway text is emitted unchanged.""" + finding = _run_emitter( + "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free " + "exhausted; the gateway owns provider discovery and failover.\n", + tmp_path, + ) + + assert ( + "Contextual-orchestrator provider availability blocked current-head security evidence" + in finding + ) + assert "gateway or its discovered provider pool was unavailable" in finding + assert "STRIX_SANDBOX_UNAVAILABLE" not in finding + assert "Strix sandbox bootstrap blocked" not in finding + + +def test_unrelated_evidence_emits_no_finding(tmp_path: Path) -> None: + """Evidence with no provider-unavailable signal still produces nothing.""" + assert _run_emitter("Strix run succeeded for model 'x' in 12s.\n", tmp_path) == "" diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index af49190142..2e733ac9e9 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -17,7 +17,7 @@ DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md") CHANGELOG = Path("CHANGELOG.md") REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") -REVIEW_DISPATCH_BLOB_SHA = "fd0e5ff75d9ae6ec68cc6945174dff5c9d487662" +REVIEW_DISPATCH_BLOB_SHA = "d86497b3f43bebbabbb4f504eb5132cdf3b7b293" def _workflow_text(path: Path) -> str: From aa99c2c5c8b4467ae756b470ced32768408cb335 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 6 Sep 2026 12:21:45 +0000 Subject: [PATCH 25/80] docs(strix): align the concurrency comment with the corrected retention and slot wording The workflow comment still said the newest head 'subsumes every older one' and that a cancelled push scan 'loses no evidence', which the same PR's doctoring amendment and contract-test docstring no longer claim. It also counted all nine outstanding runs as holding slots. Both now read the same way: a complete scan of the current tree rather than a per-commit record, and five holding runner slots with four queued. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BV96rXhqoR3tYZ9AeAVur4 --- .github/workflows/strix.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index b301e2318a..3049899745 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -80,19 +80,25 @@ concurrency: # Push scans coalesce per protected branch: a newer head of the same branch # supersedes the older scan exactly as a newer PR head does. A push scan # covers the whole tree (STRIX_TARGET_PATH is './' outside PR scope) and - # publishes no 'strix' commit status, so the newest head subsumes every - # older one. With the run-id fallback every main push was its own group and - # nothing ever retired a superseded main scan: on 2026-09-05 nine push/main - # runs were outstanding at once in this repository (five running for up to - # two hours, four queued) against a 10-30 minute normal scan, each holding a - # slot under the shared 60-job ceiling. schedule and repository_dispatch - # without a PR number keep a unique run id. + # publishes no 'strix' commit status, so the newest head's scan is a + # complete scan OF THE CURRENT TREE -- not a record of every earlier + # commit's findings: code that entered and left main between two heads, and + # findings a retired run never uploaded, are absent from it. With the run-id + # fallback every main push was its own group and nothing ever retired a + # superseded main scan: on 2026-09-05 nine push/main runs were outstanding + # at once in this repository against a 10-30 minute normal scan -- five + # holding runner slots under the shared 60-job ceiling (running for up to + # two hours) and four still queued, which occupy no slot until a runner is + # assigned. schedule and repository_dispatch without a PR number keep a + # unique run id. # Coverage trade, measured the same day: main moved 50 times in 24 hours # (median gap 8.4 min, mean 26.5 min, bursty), so with cancel-in-progress # only the final head of each merge burst completes a scan -- one completed # main scan per quiet window, not one per push. That is the intended - # exchange: a cancelled push scan loses no evidence a later whole-tree scan - # does not re-cover, and the weekly schedule scan is the floor. + # exchange: a cancelled push scan gives up its own report, and the scan that + # replaces it re-covers the current tree rather than that commit's history; + # a per-commit evidence-retention guarantee would need its own preservation + # contract. The weekly schedule scan is the floor. group: >- strix-security-scan-${{ github.event.pull_request.base.repo.full_name || From 37f7b0dc388924b626d251bbcad02640e5fa684c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 6 Sep 2026 12:56:48 +0000 Subject: [PATCH 26/80] docs(strix): correct the stale on-trigger header that described non-cancelling concurrency The header still said Strix runs 'intentionally do not cancel in progress' and that GitHub keeps one active and one pending run per group, describing cancel-in-progress: false. The workflow has run cancel-in-progress: true with a repository+PR group for some time, and this PR extends that to push events per protected branch. The header now states the real grouping (repo+PR for native and dispatch PR runs, push- for pushes, unique run id for schedule and PR-less dispatch) and that a cancellation is a supersede signal rather than passing evidence, with the merge scheduler still requiring exact-head evidence. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BV96rXhqoR3tYZ9AeAVur4 --- .github/workflows/strix.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 3049899745..f15b29f564 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -44,12 +44,15 @@ on: # them, so the same doc/image-only decision is enforced by the # changed-scope job below. The run-name # includes the PR number and head SHA for status grouping, while the - # concurrency group is scoped per repository and event class to prevent - # shared-provider key rate-limit storms. Strix runs intentionally do not - # cancel in progress because a pre-job cancellation leaves no scanner log to - # review. GitHub keeps one active and one pending run per group; the merge - # scheduler re-dispatches exact-head evidence when a pending run is - # superseded. For PRs the merge scheduler manages, same-head Strix evidence + # concurrency group is scoped per repository AND pull request (native and + # dispatch PR runs share one group), or per protected branch for push + # events, to prevent shared-provider key rate-limit storms. That group + # runs with cancel-in-progress: true, so a newer head of the same PR or + # branch retires the older run: cancellation is a supersede signal, never + # passing evidence, and the merge scheduler still requires exact-head + # evidence before it will act. schedule and PR-less repository_dispatch + # keep a unique run id and so are never cancelled by a sibling. + # For PRs the merge scheduler manages, same-head Strix evidence # is still forced at merge time via repository_dispatch (which paths-ignore # does not affect), so merged code never loses evidence. paths-ignore: From ee5567f7b15f0441a61ec2435415603b9518f1c6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 23:44:24 +0900 Subject: [PATCH 27/80] test(actions): pin concurrency group keys against comment leakage (#1970) The contract asserted that expressions appear in the concurrency block, which the block's own documentation satisfies while the key says something else. Slice to the group's value with comments stripped so the assertion tests the key. Author: separate session. Verified independently: mutant controls 7/7 caught on the branch, 7/7 missed on main, adversarial helper inputs leak no comments. Merger verification (this session): head matched the verified SHA exactly, 0 behind main, merge tree identical to the branch tree, tests-only (2 files, no path outside tests/). Own discriminating control -- collapse the group key to the repository alone while moving the expressions into the comment beside it: main 58 passed (misses it), branch 1 failed (catches it). Full gate on the merge tree: 2958 passed, 1 skipped, coverage 100%, interrogate 100%. Merged under the standing chicken-and-egg authorization: the required contexts CodeQL compatibility analysis (actions)/(python) cannot be produced for a code-touching pull request in this repository -- codeql-scan-dispatch.yml has never succeeded (0 of 1931) because its actor allowlist admits no identity that dispatches it. See #1929. Co-Authored-By: Claude Opus 5 --- ..._codeql_scan_dispatch_workflow_contract.py | 15 +- .../test_required_workflow_queue_contract.py | 132 +++++++++++++++--- 2 files changed, 121 insertions(+), 26 deletions(-) diff --git a/tests/test_codeql_scan_dispatch_workflow_contract.py b/tests/test_codeql_scan_dispatch_workflow_contract.py index bad19b54aa..2439d2936e 100644 --- a/tests/test_codeql_scan_dispatch_workflow_contract.py +++ b/tests/test_codeql_scan_dispatch_workflow_contract.py @@ -19,6 +19,9 @@ from scripts.ci import audit_central_required_workflows as ruleset_audit from tests.test_opencode_workflow_shell_syntax import _extract_run_block +from tests.test_required_workflow_queue_contract import ( + workflow_level_concurrency_group, +) REPO_ROOT = Path(__file__).resolve().parents[1] WORKFLOW_PATH = REPO_ROOT / ".github/workflows/codeql-scan-dispatch.yml" @@ -94,10 +97,14 @@ def test_codeql_scan_dispatch_keeps_current_head_language_shards_independent(): """A current-head language scan cannot cancel its sibling language scans.""" workflow = WORKFLOW_PATH.read_text(encoding="utf-8") concurrency = workflow.split("concurrency:\n", 1)[1].split("\n\npermissions:", 1)[0] - - assert "github.event.client_payload.target_repository" in concurrency - assert "github.event.client_payload.pr_number" in concurrency - assert "github.event.client_payload.required_language" in concurrency + group_value = workflow_level_concurrency_group(workflow) + + # The language segment is what keeps sibling language shards in separate groups, so it is + # asserted on the group's own value: a comment naming it would otherwise satisfy the check + # while the key had lost it, silently letting one language's scan cancel another's. + assert "github.event.client_payload.target_repository" in group_value + assert "github.event.client_payload.pr_number" in group_value + assert "github.event.client_payload.required_language" in group_value assert "cancel-in-progress: true" in concurrency diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 48035c2c40..9724d67ea7 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -21,6 +21,38 @@ def workflow_text(name: str) -> str: return (REPO_ROOT / ".github" / "workflows" / name).read_text(encoding="utf-8") +def workflow_level_concurrency_group(workflow: str) -> str: + """Return only the workflow-level ``concurrency.group`` value, comments removed. + + Asserting that an expression "appears in the concurrency block" is satisfied by + a comment that merely documents the key while the key itself says something + else, because the block's raw text carries its comments. That is not + hypothetical: the block above this workflow's group explains the key in prose, + so a maintainer quoting the expressions there while another change collapsed + the group to the repository alone would leave every pull request in one group, + cancelling each other, with the contract still green. Slice to the group's own + value so the assertion tests the key rather than the documentation beside it. + """ + header = workflow.split("permissions:", 1)[0] + block = header.split("concurrency:", 1)[1] + value: list[str] = [] + collecting = False + for line in block.splitlines(): + if line.strip().startswith("#"): + continue + if not collecting: + if re.match(r"^\s*group:", line): + collecting = True + value.append(line.split("group:", 1)[1]) + continue + if re.match(r"^\s*[A-Za-z][\w-]*:", line): + break + value.append(line) + if not collecting: + raise AssertionError("workflow-level concurrency block declares no group") + return "\n".join(value) + + def workflow_step(workflow: str, name: str) -> str: """Extract one named workflow step without parsing YAML dynamically.""" step = f" - name: {name}\n" @@ -234,22 +266,77 @@ def test_privileged_review_dispatch_coalesces_superseded_runs_before_admission() workflow = workflow_text("opencode-review-dispatch.yml") header = workflow.split("permissions:", 1)[0] concurrency_contract = header.split("concurrency:", 1)[1] + group_value = workflow_level_concurrency_group(workflow) assert re.search(r"(?m)^concurrency:", header) - assert "opencode-review-dispatch-" in concurrency_contract + assert "opencode-review-dispatch-" in group_value assert ( "github.event.client_payload.target_repository || github.repository" - in concurrency_contract - ) - assert ( - "github.event.client_payload.pr_number || github.run_id" - in concurrency_contract + in group_value ) + assert "github.event.client_payload.pr_number || github.run_id" in group_value assert "cancel-in-progress: true" in concurrency_contract assert "github.event.client_payload.pr_head_sha" not in concurrency_contract assert re.search(r"(?m)^ concurrency:", workflow) +def test_concurrency_group_slice_ignores_the_comment_that_documents_it() -> None: + """A comment quoting the key must not satisfy an assertion about the key. + + This is the negative control for ``workflow_level_concurrency_group``. The + synthetic workflow below is exactly the shape that defeated the previous + contract: the real group is collapsed to the repository alone, so every pull + request in that repository shares one group and they cancel each other, while + a comment directly above still quotes both expressions the contract looks for. + Reading the raw block finds them; reading the group's value does not. + """ + defeated = textwrap.dedent( + """\ + name: Example + on: + repository_dispatch: + concurrency: + # Key: github.event.client_payload.target_repository || github.repository + # with github.event.client_payload.pr_number || github.run_id + group: opencode-review-dispatch-${{ github.repository }} + cancel-in-progress: true + permissions: + contents: read + """ + ) + raw_block = defeated.split("permissions:", 1)[0].split("concurrency:", 1)[1] + group_value = workflow_level_concurrency_group(defeated) + + assert "github.event.client_payload.pr_number || github.run_id" in raw_block + assert "github.event.client_payload.pr_number || github.run_id" not in group_value + assert "github.event.client_payload.target_repository" not in group_value + assert "opencode-review-dispatch-${{ github.repository }}" in group_value + + +def test_concurrency_group_slice_reads_a_folded_multi_line_key() -> None: + """The real key is a folded block, so the slice must join its continuation lines.""" + folded = textwrap.dedent( + """\ + concurrency: + group: >- + opencode-review-dispatch-${{ + github.event.client_payload.target_repository || github.repository }}-${{ + github.event.client_payload.pr_number || github.run_id }} + cancel-in-progress: true + permissions: + contents: read + """ + ) + group_value = workflow_level_concurrency_group(folded) + + assert ( + "github.event.client_payload.target_repository || github.repository" + in group_value + ) + assert "github.event.client_payload.pr_number || github.run_id" in group_value + assert "cancel-in-progress" not in group_value + + def test_required_opencode_dispatch_does_not_wait_on_merge_scheduler() -> None: """Dispatch review execution directly so polling cannot starve its producer.""" workflow = workflow_text("opencode-review.yml") @@ -292,33 +379,32 @@ def test_required_pull_request_workflows_cancel_superseded_runs() -> None: concurrency_contract = workflow.split("concurrency:", 1)[1].split( "permissions:", 1 )[0] + group_value = workflow_level_concurrency_group(workflow) assert "concurrency:" in workflow - assert "github.event.pull_request.base.repo.full_name" in concurrency_contract - assert "github.repository" in concurrency_contract + assert "github.event.pull_request.base.repo.full_name" in group_value + assert "github.repository" in group_value assert "github.event.pull_request.number" in workflow assert re.search(r"(?m)^concurrency:", workflow) assert "cancel-in-progress: true" in concurrency_contract if filename == "security-scan.yml": assert ( - "github.event_name == 'pull_request_target'" in concurrency_contract - or ("github.event_name == 'pull_request'" in concurrency_contract) + "github.event_name == 'pull_request_target'" in group_value + or ("github.event_name == 'pull_request'" in group_value) ) elif filename == "opencode-review.yml": - assert "required-opencode-review-${{" in concurrency_contract + assert "required-opencode-review-${{" in group_value assert "outputs.admitted == 'true'" in workflow elif filename == "noema-review.yml": assert not re.search(r"(?m)^ concurrency:", workflow) assert "github.event.workflow_run" not in concurrency_contract - assert "required-noema-review-${{" in concurrency_contract + assert "required-noema-review-${{" in group_value assert "outputs.admitted == 'true'" in workflow else: if filename == "codeql-pr.yml": - assert "github.event_name == 'pull_request'" in concurrency_contract + assert "github.event_name == 'pull_request'" in group_value else: - assert ( - "github.event_name == 'pull_request_target'" in concurrency_contract - ) + assert "github.event_name == 'pull_request_target'" in group_value assert "github.event.pull_request.head.sha" not in concurrency_contract assert "format('pr-{0}-{1}'" not in concurrency_contract @@ -425,15 +511,17 @@ def test_strix_serializes_provider_evidence_per_repository_and_pr() -> None: )[0] strix_job = workflow.split("\n strix:\n", 1)[1] + group_value = workflow_level_concurrency_group(workflow) + assert re.search(r"(?m)^concurrency:", workflow) assert "needs: [changed-scope, admit-current-head]" in strix_job assert "needs.admit-current-head.outputs.admitted == 'true'" in strix_job - assert "strix-security-scan-${{" in concurrency_contract - assert "github.event.pull_request.base.repo.full_name" in concurrency_contract - assert "github.event.client_payload.target_repository" in concurrency_contract - assert "github.event.pull_request.number" in concurrency_contract - assert "github.event.client_payload.pr_number" in concurrency_contract - assert "github.run_id" in concurrency_contract + assert "strix-security-scan-${{" in group_value + assert "github.event.pull_request.base.repo.full_name" in group_value + assert "github.event.client_payload.target_repository" in group_value + assert "github.event.pull_request.number" in group_value + assert "github.event.client_payload.pr_number" in group_value + assert "github.run_id" in group_value assert "github.event.pull_request.head.sha" not in concurrency_contract assert "github.event.client_payload.pr_head_sha" not in concurrency_contract assert "cancel-in-progress: true" in concurrency_contract From 6e014c9bae22f1e6d8302a4f1cc38f4f6a114ae2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:20:33 +0900 Subject: [PATCH 28/80] fix(actions): coalesce superseded agent mentions while they are queued (#1964) Move the concurrency group from job level to workflow level in both agent-mention dispatch workflows so a superseded mention is coalesced while it is still queued, instead of holding its queue slot until a runner frees up. Co-Authored-By: Claude Opus 5 --- .../agent-mention-noema-dispatch.yml | 17 ++++- .../agent-mention-opencode-dispatch.yml | 17 ++++- ...st_agent_mention_downstream_idempotency.py | 17 +++-- .../test_required_workflow_queue_contract.py | 69 +++++++++++++++++++ 4 files changed, 110 insertions(+), 10 deletions(-) diff --git a/.github/workflows/agent-mention-noema-dispatch.yml b/.github/workflows/agent-mention-noema-dispatch.yml index 5bed3e8963..ad8abc7b25 100644 --- a/.github/workflows/agent-mention-noema-dispatch.yml +++ b/.github/workflows/agent-mention-noema-dispatch.yml @@ -8,15 +8,26 @@ on: repository_dispatch: types: [agent-mention-noema] +concurrency: + # Workflow-level admission, for the same reason strix.yml, noema-review.yml, + # opencode-review.yml and opencode-review-dispatch.yml carry theirs at this level: + # a job-level group is never evaluated while the whole run waits behind the + # organization job ceiling, so a superseded mention keeps its queue slot until a + # runner frees up and only then cancels. At workflow level the older run is + # coalesced while both are still queued, which is where the slot is actually held. + # This workflow has a single job, so the group lives here and nowhere else -- + # every workflow in this repository that carries a group at both levels + # (strix.yml, opencode-review-dispatch.yml) gives the two levels DIFFERENT names, + # because a job requesting the group its own run already holds would wait on itself. + group: agent-mention-noema-${{ github.event.client_payload.target_repository }}-${{ github.event.client_payload.pr_number || github.run_id }} + cancel-in-progress: true + permissions: contents: read jobs: validate-and-forward: if: github.repository == 'ContextualWisdomLab/.github' - concurrency: - group: agent-mention-noema-${{ github.event.client_payload.target_repository }}-${{ github.event.client_payload.pr_number || github.run_id }} - cancel-in-progress: true runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: diff --git a/.github/workflows/agent-mention-opencode-dispatch.yml b/.github/workflows/agent-mention-opencode-dispatch.yml index b27062ae37..05461c9551 100644 --- a/.github/workflows/agent-mention-opencode-dispatch.yml +++ b/.github/workflows/agent-mention-opencode-dispatch.yml @@ -8,15 +8,26 @@ on: repository_dispatch: types: [agent-mention-opencode] +concurrency: + # Workflow-level admission, for the same reason strix.yml, noema-review.yml, + # opencode-review.yml and opencode-review-dispatch.yml carry theirs at this level: + # a job-level group is never evaluated while the whole run waits behind the + # organization job ceiling, so a superseded mention keeps its queue slot until a + # runner frees up and only then cancels. At workflow level the older run is + # coalesced while both are still queued, which is where the slot is actually held. + # This workflow has a single job, so the group lives here and nowhere else -- + # every workflow in this repository that carries a group at both levels + # (strix.yml, opencode-review-dispatch.yml) gives the two levels DIFFERENT names, + # because a job requesting the group its own run already holds would wait on itself. + group: agent-mention-opencode-${{ github.event.client_payload.target_repository }}-${{ github.event.client_payload.pr_number || github.run_id }} + cancel-in-progress: true + permissions: contents: read jobs: validate-and-forward: if: github.repository == 'ContextualWisdomLab/.github' - concurrency: - group: agent-mention-opencode-${{ github.event.client_payload.target_repository }}-${{ github.event.client_payload.pr_number || github.run_id }} - cancel-in-progress: true runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: diff --git a/tests/test_agent_mention_downstream_idempotency.py b/tests/test_agent_mention_downstream_idempotency.py index c9b6ab86ea..85974a5719 100644 --- a/tests/test_agent_mention_downstream_idempotency.py +++ b/tests/test_agent_mention_downstream_idempotency.py @@ -33,10 +33,19 @@ def test_downstream_workflows_claim_artifacts_and_coalesce_by_pull_request() -> ): header = text.split("\npermissions:\n", 1)[0] job = text.split(" validate-and-forward:\n", 1)[1] - concurrency = job.split(" concurrency:\n", 1)[1].split( - "\n runs-on:", 1 - )[0] - assert "concurrency:" not in header + concurrency = header.split("\nconcurrency:\n", 1)[1] + # 109d79b7 ("replace unsupported queue concurrency") deleted a + # workflow-level block that used ``queue: max``, a key GitHub Actions + # does not support, and parked the group on the job while it was at it. + # What that commit pins is the absence of ``queue:``, not the level: the + # group is back at workflow level because a job-level group is never + # evaluated while the run waits behind the organization job ceiling, so a + # superseded mention held its queue slot until a runner freed up. Every + # other queue-bearing workflow here (strix.yml, noema-review.yml, + # opencode-review.yml, codeql-scan-dispatch.yml, + # opencode-review-dispatch.yml) keys its group at workflow level too. + assert "queue:" not in header + assert " concurrency:" not in job assert "github.event.client_payload.agent_invocation_key" in text assert "cwl-agent-invocation:" in text assert "source_comment_id" in text diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 9724d67ea7..4055bb5b9d 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -280,6 +280,75 @@ def test_privileged_review_dispatch_coalesces_superseded_runs_before_admission() assert re.search(r"(?m)^ concurrency:", workflow) +@pytest.mark.parametrize( + ("workflow_name", "group_prefix"), + ( + ("agent-mention-opencode-dispatch.yml", "agent-mention-opencode-"), + ("agent-mention-noema-dispatch.yml", "agent-mention-noema-"), + ), +) +def test_agent_mention_dispatch_coalesces_while_queued( + workflow_name: str, group_prefix: str +) -> None: + """A superseded agent mention must be discarded before it holds a queue slot. + + Both mention dispatchers carried the same defect + ``opencode-review-dispatch.yml`` carried before #1958: the group sat on the + single ``validate-and-forward`` job, and a job-level group is not evaluated + while the run waits behind the organization job ceiling. Measured on the + review dispatcher over the 39.7 hours ending 2026-09-06T12:41Z, 23 pairs of + runs for one pull request overlapped -- the older run was still open when its + successor arrived -- and none was coalesced; the five that ended + ``cancelled`` were cancelled between 0.7 and 2.9 hours after the newer run + was created, which is a sweep, not concurrency. + + The group moves to workflow level and is not duplicated on the job. Every + workflow here that keys a group at both levels (``strix.yml``, + ``opencode-review-dispatch.yml``) gives the two levels different names, + because a job that requests the group its own run already holds waits on + itself. + """ + workflow = workflow_text(workflow_name) + header = workflow.split("permissions:", 1)[0] + group = workflow_level_concurrency_group(workflow) + + assert re.search(r"(?m)^concurrency:", header) + # Read the group's value, not the block: the comment above these keys quotes + # the very expressions asserted here, so a raw-block assertion would survive + # the key being collapsed. That is the hole #1970 closed. + assert group.strip().startswith(group_prefix) + assert "github.event.client_payload.target_repository" in group + assert "github.event.client_payload.pr_number || github.run_id" in group + # ``cancel-in-progress`` is a sibling key, so it is outside the group value. + # Anchor it to its own line at the block's indent; a comment starts with + # ``#`` and cannot satisfy this. + assert re.search(r"(?m)^ cancel-in-progress: true$", header) + # ``\s`` also matches the newline before a column-0 key, so anchor the + # job-level search on horizontal whitespace only. + assert not re.search(r"(?m)^[ \t]+concurrency:", workflow) + + +def test_agent_mention_router_keeps_its_two_distinct_job_groups() -> None: + """The router must not be hoisted: its two jobs need different groups. + + ``agent-mention-router.yml`` runs a per-issue local route that supersedes + itself and an organization-wide sweep that must never be cancelled midway. + A workflow carries at most one workflow-level group, so hoisting either one + would silently give the sweep the route's ``cancel-in-progress: true`` and + let a later comment kill a sweep that is part way through the organization. + """ + workflow = workflow_text("agent-mention-router.yml") + + assert not re.search(r"(?m)^concurrency:", workflow) + assert ( + "group: review-agent-mention-router-local-${{ github.repository }}" + in workflow + ) + assert "group: review-agent-mention-router-sweep-${{ github.repository }}" in workflow + + sweep = workflow.split("sweep-organization-agent-mentions:", 1)[1] + assert "cancel-in-progress: false" in sweep.split("steps:", 1)[0] + def test_concurrency_group_slice_ignores_the_comment_that_documents_it() -> None: """A comment quoting the key must not satisfy an assertion about the key. From 6791f5eb888cc4c1652e7b2894c1bde714d44fda Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 6 Sep 2026 15:32:42 +0000 Subject: [PATCH 29/80] test(actions): strip YAML inline comments from the concurrency group slice workflow_level_concurrency_group() dropped whole-line comments but kept an inline one, so `group: # ` still satisfied every substring assertion while the real key had collapsed. The existing negative control only covered a comment above the key. Stripping follows YAML's own rules rather than cutting at the first `#`: a comment opens only at line start or after whitespace, and never inside a quoted scalar. Only the key's own line is stripped -- inside a folded block body `#` is literal content, so cutting there would corrupt a legitimate value. Two controls added: the inline-comment shape, which fails without this change, and a shape that must survive it (a quoted scalar containing `#` and a folded body containing `#`). Gate: 2960 passed, 1 skipped, 21 subtests, coverage 100%, interrogate 100%. Reported by CodeRabbit on 6e0bb0945. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BV96rXhqoR3tYZ9AeAVur4 --- .../test_required_workflow_queue_contract.py | 87 ++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index f4994f5403..ecc6ceca51 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -21,6 +21,31 @@ def workflow_text(name: str) -> str: return (REPO_ROOT / ".github" / "workflows" / name).read_text(encoding="utf-8") +def _strip_yaml_inline_comment(text: str) -> str: + """Drop a YAML inline comment from one scalar line. + + YAML opens a comment at ``#`` only when it starts the line or follows + whitespace, and never inside a quoted scalar, so a bare ``split("#")`` + would truncate a legitimate value that merely contains the character. + """ + index = 0 + quote = "" + while index < len(text): + char = text[index] + if quote: + if quote == '"' and char == "\\": + index += 2 + continue + if char == quote: + quote = "" + elif char in "\"'": + quote = char + elif char == "#" and (index == 0 or text[index - 1] in " \t"): + return text[:index] + index += 1 + return text + + def workflow_level_concurrency_group(workflow: str) -> str: """Return only the workflow-level ``concurrency.group`` value, comments removed. @@ -43,7 +68,7 @@ def workflow_level_concurrency_group(workflow: str) -> str: if not collecting: if re.match(r"^\s*group:", line): collecting = True - value.append(line.split("group:", 1)[1]) + value.append(_strip_yaml_inline_comment(line.split("group:", 1)[1])) continue if re.match(r"^\s*[A-Za-z][\w-]*:", line): break @@ -313,6 +338,66 @@ def test_concurrency_group_slice_ignores_the_comment_that_documents_it() -> None assert "opencode-review-dispatch-${{ github.repository }}" in group_value +def test_concurrency_group_slice_ignores_an_inline_comment_on_the_key() -> None: + """An inline comment beside a plain-scalar key must not satisfy the contract. + + The full-line negative control above does not cover this shape. YAML allows a + comment on the key's own line, so a change collapsing the group to the + repository alone could keep the documented expressions one space away and + leave every substring assertion green. + """ + inline = textwrap.dedent( + """\ + concurrency: + group: opencode-review-dispatch-${{ github.repository }} # ${{ github.event.client_payload.pr_number || github.run_id }} + cancel-in-progress: true + permissions: + contents: read + """ + ) + group_value = workflow_level_concurrency_group(inline) + + assert "opencode-review-dispatch-${{ github.repository }}" in group_value + assert "github.event.client_payload.pr_number" not in group_value + assert "github.run_id" not in group_value + + +def test_concurrency_group_slice_keeps_a_hash_that_is_not_a_comment() -> None: + """Stripping must follow YAML's rules rather than cutting at every ``#``. + + Two shapes would be corrupted by a naive ``split("#")``: a quoted scalar + containing the character, and a folded block body, where ``#`` is literal + content and never opens a comment. Only the key's own line is stripped. + """ + quoted = textwrap.dedent( + """\ + concurrency: + group: "release-#42-${{ github.repository }}" + cancel-in-progress: true + permissions: + contents: read + """ + ) + assert "release-#42-${{ github.repository }}" in workflow_level_concurrency_group( + quoted + ) + + folded = textwrap.dedent( + """\ + concurrency: + group: >- + release-${{ github.repository }}-#${{ + github.run_id }} + cancel-in-progress: true + permissions: + contents: read + """ + ) + folded_value = workflow_level_concurrency_group(folded) + assert "#${{" in folded_value + assert "github.run_id" in folded_value + + def test_concurrency_group_slice_reads_a_folded_multi_line_key() -> None: """The real key is a folded block, so the slice must join its continuation lines.""" folded = textwrap.dedent( From 9aad23c09da468716a788cfed65cd44f7d84a284 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:13:45 +0900 Subject: [PATCH 30/80] test(actions): parse the concurrency group instead of slicing the block (#1975) The helper sliced from `permissions:`, so it raised IndexError on the two workflows that declare permissions first, and it returned a folded key's raw newlines rather than the value YAML produces. Nine of twenty-nine workflow-level keys are folded, including every required review workflow. Author and verifier were separate sessions. Merger verification (this session, independent runs): head matched GitHub exactly, 0 behind main, merge tree identical to the branch tree, tests-only. Exact-match against a yaml oracle across every workflow: 29 match, 0 mismatch, 0 exception. Seven adversarial inputs pass, three of them designed here rather than reused -- a comment quoting `group:` before the real key, a job-level concurrency block appearing first in the file, and literal `|`/`|-` scalars, which are refused rather than silently folded into a value YAML never produces. Two-way control on the live hole: flipping noema-review.yml's cancel-in-progress to false behind a comment passes on main (2961) and fails here. pr-review-autofix.yml's deliberate cancel-in-progress: false is preserved. Gate: 2964 passed, 1 skipped, coverage 100%, interrogate 100%. Bypass basis stated plainly: this change is tests-only and does not itself unblock anything, so it does not meet the narrower 'the PR's own diff edits review-pipeline files' reading recorded in docs/product-technical-gap-baseline.md. It is merged under this session's standing instruction to fix the queue, which names verifying `cancel-in-progress: true` as part of that work. The required CodeQL contexts remain unreachable for any code-touching PR here (#1929). Co-Authored-By: Claude Opus 5 --- ..._codeql_scan_dispatch_workflow_contract.py | 6 +- .../test_required_workflow_queue_contract.py | 154 ++++++++++++++++-- 2 files changed, 146 insertions(+), 14 deletions(-) diff --git a/tests/test_codeql_scan_dispatch_workflow_contract.py b/tests/test_codeql_scan_dispatch_workflow_contract.py index 2439d2936e..71fa43541f 100644 --- a/tests/test_codeql_scan_dispatch_workflow_contract.py +++ b/tests/test_codeql_scan_dispatch_workflow_contract.py @@ -20,6 +20,7 @@ from scripts.ci import audit_central_required_workflows as ruleset_audit from tests.test_opencode_workflow_shell_syntax import _extract_run_block from tests.test_required_workflow_queue_contract import ( + workflow_level_cancels_in_progress, workflow_level_concurrency_group, ) @@ -96,7 +97,6 @@ def test_codeql_scan_dispatch_workflow_structure(): def test_codeql_scan_dispatch_keeps_current_head_language_shards_independent(): """A current-head language scan cannot cancel its sibling language scans.""" workflow = WORKFLOW_PATH.read_text(encoding="utf-8") - concurrency = workflow.split("concurrency:\n", 1)[1].split("\n\npermissions:", 1)[0] group_value = workflow_level_concurrency_group(workflow) # The language segment is what keeps sibling language shards in separate groups, so it is @@ -105,7 +105,9 @@ def test_codeql_scan_dispatch_keeps_current_head_language_shards_independent(): assert "github.event.client_payload.target_repository" in group_value assert "github.event.client_payload.pr_number" in group_value assert "github.event.client_payload.required_language" in group_value - assert "cancel-in-progress: true" in concurrency + # Same reasoning as the group above, applied to the flag: the substring form + # is satisfied by a comment quoting it while the key beside it reads false. + assert workflow_level_cancels_in_progress(workflow) def _run_validate_step(tmp_path: Path, env_overrides: dict[str, str], pull_request: dict) -> subprocess.CompletedProcess[str]: diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 4055bb5b9d..512cb6bef5 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -21,6 +21,18 @@ def workflow_text(name: str) -> str: return (REPO_ROOT / ".github" / "workflows" / name).read_text(encoding="utf-8") +# The workflow-level block is the one whose key starts at column zero; job-level +# blocks are indented under ``jobs:``. Anchoring there instead of slicing the text +# before ``permissions:`` makes the search independent of key order, which two +# workflows already need: javascript-coverage-quality-ci.yml and +# repository-metadata-reconcile.yml declare ``permissions:`` above ``concurrency:``, +# and the older slice returned nothing for them and raised IndexError rather than +# reading the block that is plainly there. +WORKFLOW_LEVEL_CONCURRENCY_BLOCK = re.compile( + r"(?m)^concurrency:[ \t]*\n(?P(?:[ \t]+[^\n]*\n)+)" +) + + def workflow_level_concurrency_group(workflow: str) -> str: """Return only the workflow-level ``concurrency.group`` value, comments removed. @@ -33,11 +45,12 @@ def workflow_level_concurrency_group(workflow: str) -> str: cancelling each other, with the contract still green. Slice to the group's own value so the assertion tests the key rather than the documentation beside it. """ - header = workflow.split("permissions:", 1)[0] - block = header.split("concurrency:", 1)[1] + block_match = WORKFLOW_LEVEL_CONCURRENCY_BLOCK.search(workflow) + if block_match is None: + raise AssertionError("workflow declares no workflow-level concurrency block") value: list[str] = [] collecting = False - for line in block.splitlines(): + for line in block_match.group("body").splitlines(): if line.strip().startswith("#"): continue if not collecting: @@ -50,7 +63,46 @@ def workflow_level_concurrency_group(workflow: str) -> str: value.append(line) if not collecting: raise AssertionError("workflow-level concurrency block declares no group") - return "\n".join(value) + head = value[0].strip() + if head.startswith("|"): + # Not represented here, and on 2026-09-07 no workflow uses one: a literal + # block keeps its newlines, so folding it would return a value YAML never + # produces. Refusing is better than returning a plausible wrong string. + raise AssertionError("literal block scalars are not supported for the group key") + if head.startswith(">"): + # Nine of the twenty-nine workflow-level keys are folded, including every + # required review workflow, so this is the majority shape rather than an + # edge case. YAML joins a folded scalar's lines with single spaces, so + # returning the indicator and the raw newlines would make the helper + # disagree with the file's own meaning. Blank lines and more-deeply + # indented lines inside a fold keep their newlines in YAML and are not + # handled here; neither shape occurs in this tree. + return " ".join(part.strip() for part in value[1:] if part.strip()) + return "\n".join(value).strip() + + +def workflow_level_cancels_in_progress(workflow: str) -> bool: + """Return whether the workflow-level block really sets ``cancel-in-progress: true``. + + Anchored to the start of a block line, so a commented-out setting cannot + satisfy it. Substring assertions could: commenting the real line out and + adding ``cancel-in-progress: false`` beside it leaves the searched text in + the file while YAML reads the opposite, and on 2026-09-06 that mutation + passed the whole suite (2958 passed, 0 failed) against ``noema-review.yml``. + A required review workflow that stops cancelling superseded runs keeps every + earlier review alive on each push, which is the queue behaviour this + repository has been trying to remove. + + Kept separate from the group helper on purpose: ``cancel-in-progress`` is a + sibling of ``group``, so it lies outside the value that helper returns and + cannot be covered by moving assertions onto it. + """ + block_match = WORKFLOW_LEVEL_CONCURRENCY_BLOCK.search(workflow) + if block_match is None: + raise AssertionError("workflow declares no workflow-level concurrency block") + return bool( + re.search(r"(?m)^[ \t]+cancel-in-progress:[ \t]+true[ \t]*$", block_match.group("body")) + ) def workflow_step(workflow: str, name: str) -> str: @@ -144,7 +196,10 @@ def test_merge_scheduler_uses_native_auto_merge_after_required_checks() -> None: assert "github.event_name == 'repository_dispatch' && github.run_id" not in ( concurrency_contract ) - assert "cancel-in-progress: ${{" in concurrency_contract + # Anchored, not a substring: this workflow's value is an expression rather + # than a constant, so it cannot use the boolean helper, but a commented-out + # setting must not satisfy it either. + assert re.search(r"(?m)^[ \t]+cancel-in-progress:[ \t]+\$\{\{", concurrency_contract) assert "github.event_name == 'repository_dispatch'" in concurrency_contract @@ -275,7 +330,7 @@ def test_privileged_review_dispatch_coalesces_superseded_runs_before_admission() in group_value ) assert "github.event.client_payload.pr_number || github.run_id" in group_value - assert "cancel-in-progress: true" in concurrency_contract + assert workflow_level_cancels_in_progress(workflow) assert "github.event.client_payload.pr_head_sha" not in concurrency_contract assert re.search(r"(?m)^ concurrency:", workflow) @@ -406,6 +461,74 @@ def test_concurrency_group_slice_reads_a_folded_multi_line_key() -> None: assert "cancel-in-progress" not in group_value +def test_concurrency_helpers_read_the_block_when_permissions_comes_first() -> None: + """Key order must not decide whether the contract can see the block. + + The earlier helper sliced the text before ``permissions:`` and then split on + ``concurrency:``. That works only when ``concurrency:`` is declared first. Two + workflows in this repository declare ``permissions:`` above it -- + javascript-coverage-quality-ci.yml and repository-metadata-reconcile.yml -- + and for those the slice was empty, so the helper raised ``IndexError`` instead + of reading the block that is plainly there. Anchoring at column zero makes the + order irrelevant. + """ + permissions_first = textwrap.dedent( + """\ + name: Example + permissions: + contents: read + concurrency: + group: example-${{ github.repository }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + jobs: + build: + runs-on: ubuntu-latest + """ + ) + + assert ( + workflow_level_concurrency_group(permissions_first) + == "example-${{ github.repository }}-${{ github.event.pull_request.number }}" + ) + assert workflow_level_cancels_in_progress(permissions_first) + + +def test_concurrency_helpers_name_a_missing_block_instead_of_index_error() -> None: + """A workflow with no top-level block must fail with a sentence, not ``IndexError``. + + ``IndexError: list index out of range`` names neither the workflow nor the + contract it broke, so a reader has to reconstruct both from the traceback. + """ + no_block = "name: Example\njobs:\n build:\n runs-on: ubuntu-latest\n" + + for helper in (workflow_level_concurrency_group, workflow_level_cancels_in_progress): + with pytest.raises(AssertionError, match="no workflow-level concurrency block"): + helper(no_block) + + +def test_cancel_in_progress_assertion_rejects_a_commented_out_setting() -> None: + """The negative control for ``workflow_level_cancels_in_progress``. + + A substring test for ``cancel-in-progress: true`` is satisfied by a comment + that quotes it. On 2026-09-06 that exact mutation -- comment out the real line + in noema-review.yml, add ``cancel-in-progress: false`` beneath it -- passed the + whole suite (2958 passed, 0 failed) while every push to a pull request stopped + cancelling its own superseded run. Anchoring to the start of a block line is + what closes it. + """ + quoted_but_disabled = textwrap.dedent( + """\ + concurrency: + group: example-${{ github.repository }}-${{ github.event.pull_request.number }} + # cancel-in-progress: true + cancel-in-progress: false + """ + ) + + assert "cancel-in-progress: true" in quoted_but_disabled + assert not workflow_level_cancels_in_progress(quoted_but_disabled) + + def test_required_opencode_dispatch_does_not_wait_on_merge_scheduler() -> None: """Dispatch review execution directly so polling cannot starve its producer.""" workflow = workflow_text("opencode-review.yml") @@ -455,7 +578,7 @@ def test_required_pull_request_workflows_cancel_superseded_runs() -> None: assert "github.repository" in group_value assert "github.event.pull_request.number" in workflow assert re.search(r"(?m)^concurrency:", workflow) - assert "cancel-in-progress: true" in concurrency_contract + assert workflow_level_cancels_in_progress(workflow) if filename == "security-scan.yml": assert ( "github.event_name == 'pull_request_target'" in group_value @@ -497,12 +620,17 @@ def test_pr_quality_workflows_isolate_concurrency_by_repository_and_pr() -> None "${{ github.event.pull_request.number || github.ref }}" ) in concurrency if filename == "cloudflare-dns.yml": - assert ( - "cancel-in-progress: ${{ github.event_name == 'pull_request' }}" - in concurrency + # Anchored like the ``true`` contracts below: a commented-out setting + # must not satisfy this either, and this workflow deliberately cancels + # only for pull requests, so its value is an expression rather than a + # constant. + assert re.search( + r"(?m)^[ \t]+cancel-in-progress:[ \t]+\$\{\{ github\.event_name ==" + r" 'pull_request' \}\}[ \t]*$", + concurrency, ) else: - assert "cancel-in-progress: true" in concurrency + assert workflow_level_cancels_in_progress(workflow) def test_central_semgrep_logs_every_finding_and_distinguishes_engine_failure() -> None: @@ -832,7 +960,9 @@ def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() - )[0] assert "github.event.pull_request.number" in concurrency_contract assert "github.event.pull_request.head.sha" not in concurrency_contract - assert "cancel-in-progress:" in concurrency_contract + assert re.search( + r"(?m)^[ \t]+cancel-in-progress:[ \t]+\S", concurrency_contract + ) else: raise AssertionError(f"unclassified close-event workflow: {filename}") assert "github.event.action != 'closed'" in workflow From 5dad3fe874f93b19b232f4452e7190b72655f18e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:29:02 +0900 Subject: [PATCH 31/80] fix(ci): preserve bounded sidecar diagnostics Signed-off-by: Seongho Bae --- AGENTS.md | 2 + ..._contextual_orchestrator_sidecar_stream.py | 17 +++++- ...l_orchestrator_review_runtime_preflight.py | 55 +++++++++++++++++++ 3 files changed, 71 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e955f8b36a..49149dfa81 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -68,6 +68,8 @@ The materialization contract is also covered by [`docs/doctoring/exact-artifact- ## Verification discipline +- producer가 안전한 로그 필드를 추가하면 exact revision 쌍으로 consumer sanitizer를 통과시켜 allowlist의 누락을 확인한다. producer 단위 테스트 성공만으로 CI artifact 보존을 주장하지 않으며, 연결 검증에서도 raw 본문 비출력을 유지한다. + Many agent sessions work this organization concurrently under the same standing brief. Silence is not evidence: "I have not touched X" describes one session's history, never the organization's actual state. diff --git a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py index e055513954..9df99e1b70 100644 --- a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py +++ b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py @@ -8,8 +8,12 @@ _REQUEST_FAILED = re.compile( + r"^(?:(?:[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2},[0-9]{3} )?" + r"(?:DEBUG|INFO|WARNING|ERROR)[: ]contextual_orchestrator\.server[: ])?" r"request_failed status=(?P[1-5][0-9]{2}) " r"code=(?P[A-Za-z0-9_.-]{1,64})" + r"(?= |$)(?: request_id=(?P[0-9a-f]{32}|)(?= |$))?" + r"(?! request_id=)" ) _PROVIDER_DISCOVERY_FAILED = re.compile( r"provider_discovery_failed provider=(?P[a-z][a-z0-9_]{0,63}) " @@ -38,7 +42,8 @@ for pattern in ( rf"^provider_attempt agent_id={_AGENT_ID} model={_MODEL_ID} attempt=\d+/\d+$", rf"^provider_attempt_failed agent_id={_AGENT_ID} model={_MODEL_ID} attempt=\d+ " - rf"error_type={_ERROR_TYPE} transient=(?:True|False)(?= error_message=)", + rf"error_type={_ERROR_TYPE} transient=(?:True|False)" + rf"(?: provider_status=(?:[1-5][0-9]{{2}}|None))?(?= error_message=)", rf"^provider_backoff agent_id={_AGENT_ID} attempt=\d+ delay_seconds={_NUMBER}$", rf"^provider_exhausted agent_id={_AGENT_ID} model={_MODEL_ID} attempts=\d+ " rf"final_error_type={_ERROR_TYPE}$", @@ -122,12 +127,18 @@ def _sanitize_orchestrator_event(stripped: str) -> str | None: def sanitize_line(line: str) -> str | None: """Return one allowlisted diagnostic summary or ``None`` for raw content.""" stripped = line.strip() - request_failed = _REQUEST_FAILED.search(stripped) + if "\n" in stripped or "\r" in stripped: + return None + request_failed = _REQUEST_FAILED.match(stripped) if request_failed is not None: - return ( + summary = ( f"request_failed status={request_failed.group('status')} " f"code={request_failed.group('code')}" ) + request_id = request_failed.group("request_id") + if request_id is not None: + summary += f" request_id={request_id}" + return summary provider_discovery_failed = _PROVIDER_DISCOVERY_FAILED.search(stripped) if provider_discovery_failed is not None: return ( diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index 7f92162097..73899f10c9 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -1800,6 +1800,61 @@ def test_sidecar_stream_sanitizer_admits_orchestrator_route_events() -> None: assert sanitize_line(f"DEBUG:contextual_orchestrator.orchestrator:{secret}") is None +@pytest.mark.parametrize("status", [None, "100", "429", "599", "None"]) +def test_sidecar_stream_provider_status_compatibility(status) -> None: + """Legacy and typed producer diagnostics survive without upstream text.""" + prefix = "provider_attempt_failed agent_id=fixture model=m/x attempt=1 error_type=HTTPError transient=True" + fields = "" if status is None else f" provider_status={status}" + assert _load_sanitizer()["sanitize_line"]( + prefix + fields + " error_message=Bearer sk-secret" + ) == prefix + fields + " error_message=" + + +@pytest.mark.parametrize("request_id", [None, "a1" * 16, ""]) +def test_sidecar_stream_request_id_compatibility(request_id) -> None: + """Keep safe correlation identifiers, including the producer omission marker.""" + message = "request_failed status=500 code=internal_error" + if request_id is not None: + message += f" request_id={request_id}" + assert _load_sanitizer()["sanitize_line"](message) == message + + +@pytest.mark.parametrize("status", ["099", "600", "4290", "429secret", "-1", "True", "none", "４２９"]) +def test_sidecar_stream_rejects_invalid_provider_status(status) -> None: + """Invalid typed fields must not downgrade to an accepted legacy prefix.""" + assert _load_sanitizer()["sanitize_line"]( + "provider_attempt_failed agent_id=fixture model=m/x attempt=1 " + f"error_type=HTTPError transient=True provider_status={status} error_message=sk-secret" + ) is None + + +@pytest.mark.parametrize("request_id", ["a" * 31, "a" * 33, "A" * 32, "g" * 32, + "secret", "a" * 32 + "-secret", "", "a" * 32 + "\nsecret"]) +def test_sidecar_stream_rejects_invalid_request_id(request_id) -> None: + """Do not preserve a partial identifier or fall back to the legacy record.""" + assert _load_sanitizer()["sanitize_line"]( + f"request_failed status=500 code=internal_error request_id={request_id}" + ) is None + + +@pytest.mark.parametrize("status,code", [("5000", "internal_error"), ("600", "internal_error"), + ("500", "x" * 65), ("500", "internal_error/secret")]) +def test_sidecar_stream_rejects_partial_request_fields(status, code) -> None: + """Status and code validation consumes complete tokens, never safe prefixes.""" + assert _load_sanitizer()["sanitize_line"](f"request_failed status={status} code={code}") is None + + +@pytest.mark.parametrize("prefix,allowed", [ + ("", True), ("WARNING:contextual_orchestrator.server:", True), + ("2026-09-05 21:40:00,123 WARNING contextual_orchestrator.server ", True), + ("provider text ", False), ("WARNING:provider.raw:", False), +]) +def test_sidecar_stream_request_event_boundary(prefix, allowed) -> None: + """Only bare events or the server logger envelope may carry request IDs.""" + event = "request_failed status=500 code=internal_error request_id=" + "a" * 32 + assert _load_sanitizer()["sanitize_line"](prefix + event) == (event if allowed else None) + + def test_sidecar_stream_sanitizer_matches_real_formatter_output() -> None: """Fixtures typed from a template miss runtime value types; render the real records. From ad0779bee66624c3997947d7691f4b0dbb973be1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:34:33 +0900 Subject: [PATCH 32/80] fix(scheduler): skip review dispatch for a head whose merge tree cannot materialize (#1973) Materializing the PR merge tree is a precondition of coverage-source-tree, so a conflicting head can only produce a failed dispatch. The guard returns before review_dispatch_admitted, preserving the bounded admission budget for a PR a review could actually finish. UNKNOWN is deliberately not blocked. Authored, verified and merged by three separate sessions. Merger verification, my own runs: head matched GitHub, 0 behind main, merge tree identical to the branch tree. Against current main the diff is 2 files, 206 insertions, 0 deletions; the production change is 29 added lines and nothing removed. The guard sits at line 3691 and review_dispatch_admitted at 3706, so the budget is preserved; 6 call sites handle the new return value. Negative control, removing only the 15 guard lines: exactly 2 tests fail -- test_review_dispatch_skips_a_head_whose_merge_tree_cannot_materialize and test_review_dispatch_reads_the_rest_merge_state_not_only_graphql -- asserting merge_conflict against a received dispatched. Gate: 2968 passed, 1 skipped, coverage 100% (13196 statements, 0 missed), interrogate 100%. The cited measurement was corrected before merge. It read '20 dispatches across 80.5 hours'; two sessions independently recounted .github#1529 as 27 dispatches across 100.8 hours with zero successes (20 cancelled, 7 failed). The original figure came from a run window that silently truncated before the pull request existed. The comment now carries the corrected numbers and the window. Authorization is this session's standing instruction to clear the queue, which lives outside this repository's text -- as docs/product-technical-gap-baseline.md itself records after a 2026-09-01 correction, and that file is annotated '(not merge authorization)' at every entry point. Corroborating rather than authorizing: this PR's own diff edits scripts/ci/ review-pipeline code, so it cannot validate itself across the pull_request_target trust boundary, which is the conservative condition that document records one earlier pass imposing on itself. On this head all 12 required contexts are unsatisfiable: 9 never reported, 3 queued, 0 passing. Co-Authored-By: Claude Opus 5 --- scripts/ci/pr_review_merge_scheduler_core.py | 29 +++ tests/test_pr_review_merge_scheduler.py | 177 +++++++++++++++++++ 2 files changed, 206 insertions(+) diff --git a/scripts/ci/pr_review_merge_scheduler_core.py b/scripts/ci/pr_review_merge_scheduler_core.py index c4e9d28ebd..cffb52cb53 100644 --- a/scripts/ci/pr_review_merge_scheduler_core.py +++ b/scripts/ci/pr_review_merge_scheduler_core.py @@ -2927,6 +2927,8 @@ def post_update_branch_followup( if wait_reason: return f"{head_note}; {wait_reason}" dispatch_result = dispatch_opencode_review(repo, workflow, updated_pr, dry_run=dry_run) + if dispatch_result == "merge_conflict": + return f"{head_note}; PR merge tree cannot be materialized while the head conflicts; review dispatch skipped" if dispatch_result == "admission_deferred": return f"{head_note}; bounded admission budget is exhausted" if dispatch_result == "already_running": @@ -3686,6 +3688,23 @@ def dispatch_opencode_review(repo: str, workflow: str, pr: dict[str, Any], *, dr return "already_running" if dry_run: return "dry_run" + if effective_merge_state(pr) in {"DIRTY", "CONFLICTING"}: + # Materializing the PR merge tree is a hard precondition of + # coverage-source-tree, so a conflicting head can only produce a failed + # dispatch. Returning before review_dispatch_admitted keeps the bounded + # admission budget for a PR a review could actually finish: measured on + # .github#1529, one conflicting head consumed 27 dispatches across 100.8 + # hours with zero successes (20 cancelled, 7 failed, and all 7 that + # reached coverage-source-tree died there; 2026-09-01T08:46Z..09-05T13:31Z). + # UNKNOWN is deliberately not blocked -- an + # uncomputed mergeability must not starve a reviewable PR. + print( + "OpenCode review dispatch skipped: GitHub reports the current head as " + f"{effective_merge_state(pr)}, so the PR merge tree cannot be materialized " + "and the review would fail. Repair the branch and push it, then the review " + "runs on the new head." + ) + return "merge_conflict" if not review_dispatch_admitted("opencode", repo, pr): return "admission_deferred" base_ref, base_sha, head_sha = validated_pr_dispatch_fields(pr) @@ -4134,6 +4153,8 @@ def dispatch_draft_review_only( f"draft PR review-only dispatch; current head has completed Strix evidence; {wait_reason}", ) dispatch_result = dispatch_opencode_review(repo, workflow, pr, dry_run=dry_run) + if dispatch_result == "merge_conflict": + return Decision(number, "wait", "draft PR review-only dispatch; PR merge tree cannot be materialized while the head conflicts; review dispatch skipped") if dispatch_result == "admission_deferred": return Decision(number, "wait", "draft PR review-only dispatch; bounded admission budget is exhausted") if dispatch_result == "already_running": @@ -4246,6 +4267,8 @@ def inspect_pr( if wait_reason: return Decision(number, "wait", f"stacked PR onto {base_ref}; {wait_reason}") dispatch_result = dispatch_opencode_review(repo, workflow, pr, dry_run=dry_run) + if dispatch_result == "merge_conflict": + return Decision(number, "wait", f"stacked PR onto {base_ref}; PR merge tree cannot be materialized while the head conflicts; review dispatch skipped") if dispatch_result == "admission_deferred": return Decision(number, "wait", f"stacked PR onto {base_ref}; bounded admission budget is exhausted") if dispatch_result == "already_running": @@ -4464,6 +4487,8 @@ def request_branch_update(freshness_reason: str, *, suffix: str = "") -> Decisio if wait_reason: return decide("wait", wait_reason) dispatch_result = dispatch_opencode_review(repo, workflow, pr, dry_run=dry_run) + if dispatch_result == "merge_conflict": + return decide("wait", "PR merge tree cannot be materialized while the head conflicts; review dispatch skipped") if dispatch_result == "admission_deferred": return decide("wait", "bounded admission budget is exhausted") if dispatch_result == "already_running": @@ -4879,6 +4904,8 @@ def request_branch_update(freshness_reason: str, *, suffix: str = "") -> Decisio f"OpenCode review exceeded {stale_opencode_minutes} minute retry threshold; review dispatch limit reached", ) dispatch_result = dispatch_opencode_review(repo, workflow, pr, dry_run=dry_run) + if dispatch_result == "merge_conflict": + return decide("wait", "PR merge tree cannot be materialized while the head conflicts; review dispatch skipped") if dispatch_result == "admission_deferred": return decide("wait", "bounded admission budget is exhausted") if dispatch_result == "already_running": @@ -4929,6 +4956,8 @@ def request_branch_update(freshness_reason: str, *, suffix: str = "") -> Decisio if wait_reason: return decide("wait", f"current head has completed Strix evidence; {wait_reason}") dispatch_result = dispatch_opencode_review(repo, workflow, pr, dry_run=dry_run) + if dispatch_result == "merge_conflict": + return decide("wait", "PR merge tree cannot be materialized while the head conflicts; review dispatch skipped") if dispatch_result == "admission_deferred": return decide("wait", "bounded admission budget is exhausted") if dispatch_result == "already_running": diff --git a/tests/test_pr_review_merge_scheduler.py b/tests/test_pr_review_merge_scheduler.py index 2cbda7f85b..8b924291a5 100644 --- a/tests/test_pr_review_merge_scheduler.py +++ b/tests/test_pr_review_merge_scheduler.py @@ -2330,6 +2330,75 @@ def test_dispatch_opencode_review_falls_back_to_bounded_discovery(monkeypatch): assert json.loads(dispatch_calls[0])["client_payload"]["required_run_id"] == 999 +def _dispatch_with_merge_state(monkeypatch, **overrides): + """Run the OpenCode dispatch funnel and report what it did.""" + monkeypatch.setenv("GITHUB_ACTIONS", "true") + monkeypatch.setenv("GH_TOKEN", "opencode-app-token") + monkeypatch.setattr( + sched, "active_opencode_run_refs", lambda repo, workflow, pr: ([], []) + ) + monkeypatch.setattr(sched, "discover_opencode_required_run_id", lambda repo, head_sha: None) + dispatched: list[str | None] = [] + monkeypatch.setattr( + sched, "run_github_dispatch", lambda args, stdin=None: dispatched.append(stdin) + ) + admitted: list[str] = [] + + def record_admission(component, repo, pr): + admitted.append(component) + return True + + monkeypatch.setattr(sched, "review_dispatch_admitted", record_admission) + pr = make_pr(headRefOid="a" * 40, baseRefOid="b" * 40, **overrides) + monkeypatch.setattr(sched, "fetch_pr", lambda *_args: [pr]) + result = sched.dispatch_opencode_review("owner/repo", "OpenCode Review", pr, dry_run=False) + return result, dispatched, admitted + + +def test_review_dispatch_skips_a_head_whose_merge_tree_cannot_materialize(monkeypatch): + """A conflicting head is skipped before it can spend the admission budget. + + coverage-source-tree must materialize the PR merge tree, which git cannot do + while the head conflicts, so the dispatch could only fail. Measured on + .github#1529: one conflicting head took 27 dispatches over 100.8 hours and + produced no review; the 7 that reached coverage-source-tree all died there, + and the other 20 were cancelled before they ever started it. + """ + for graph_state in ("DIRTY", "CONFLICTING"): + result, dispatched, admitted = _dispatch_with_merge_state( + monkeypatch, mergeStateStatus=graph_state + ) + assert result == "merge_conflict" + assert dispatched == [], f"{graph_state} must not reach the dispatch API" + assert admitted == [], f"{graph_state} must not consume the admission budget" + + +def test_review_dispatch_reads_the_rest_merge_state_not_only_graphql(monkeypatch): + """The skip honours REST mergeability, which outranks a stale GraphQL value.""" + result, dispatched, admitted = _dispatch_with_merge_state( + monkeypatch, mergeStateStatus="CLEAN", restMergeableState="DIRTY" + ) + assert result == "merge_conflict" + assert dispatched == [] + assert admitted == [] + + +def test_review_dispatch_still_runs_when_mergeability_is_not_yet_known(monkeypatch): + """UNKNOWN mergeability must not starve a reviewable PR. + + Negative control for the conflict skip: GitHub reports UNKNOWN while it is + still computing a merge commit, so blocking on it would defer every PR the + scheduler reached first. + """ + for graph_state in ("UNKNOWN", "BEHIND", "BLOCKED", "CLEAN"): + result, dispatched, admitted = _dispatch_with_merge_state( + monkeypatch, mergeStateStatus=graph_state + ) + assert result == "dispatched", f"{graph_state} must still dispatch" + assert len(dispatched) == 1 + assert admitted == ["opencode"] + + def test_central_progress_ignores_required_workflow_checkrun_placeholder( monkeypatch, ): @@ -10695,3 +10764,111 @@ def behind_with(nodes): assert "checks are still queued or running" not in resumed.reason assert sched.has_in_flight_check_runs(behind_with([])) is False + + +def _skip_opencode_dispatch(monkeypatch): + """Make the OpenCode dispatch funnel report an unmaterializable merge tree.""" + monkeypatch.setattr( + sched, "dispatch_opencode_review", lambda repo, workflow, pr, dry_run: "merge_conflict" + ) + monkeypatch.setattr( + sched, "dispatch_strix_evidence", lambda repo, workflow, pr, dry_run: "dispatched" + ) + + +SKIP_REASON = "PR merge tree cannot be materialized while the head conflicts; review dispatch skipped" + + +def test_every_review_dispatch_caller_reports_the_conflict_skip_truthfully(monkeypatch): + """No dispatch path may report a skipped conflicting head as a dispatch. + + Each caller's fall-through says the review was dispatched, so a new funnel + result that a caller does not handle would be reported as work that never + happened -- and that is the same telemetry used to find the treadmill this + skip removes. + """ + _skip_opencode_dispatch(monkeypatch) + + stacked = inspect(make_pr(baseRefName="develop")) + assert stacked.action == "wait" + assert stacked.reason == f"stacked PR onto develop; {SKIP_REASON}" + + draft = inspect( + make_pr(isDraft=True, statusCheckRollup={"contexts": {"nodes": [strix_check()]}}), + allow_draft_review_dispatch=True, + ) + assert draft.action == "wait" + assert draft.reason == f"draft PR review-only dispatch; {SKIP_REASON}" + + strix_done = inspect(make_pr(statusCheckRollup={"contexts": {"nodes": [strix_check()]}})) + assert strix_done.action == "wait" + assert strix_done.reason == SKIP_REASON + + stale = inspect( + make_pr( + statusCheckRollup={ + "contexts": { + "nodes": [ + opencode_check(started_at="2026-06-25T07:00:00Z"), + strix_check(), + ] + } + } + ) + ) + assert stale.action == "wait" + assert stale.reason == SKIP_REASON + + coverage_retry = inspect( + make_pr( + reviews={ + "nodes": [ + { + **opencode_review("CHANGES_REQUESTED", "head"), + "body": ( + "OpenCode cannot approve yet because required coverage evidence " + "did not pass. The coverage-evidence gate reported that required " + "test/docstring evidence was not proven." + ), + } + ] + }, + statusCheckRollup={ + "contexts": { + "nodes": [ + strix_check(), + { + "__typename": "CheckRun", + "name": "coverage-evidence", + "status": "COMPLETED", + "conclusion": "SUCCESS", + }, + {**opencode_check(status="COMPLETED"), "conclusion": "FAILURE"}, + ] + } + }, + ) + ) + assert coverage_retry.action == "wait" + assert coverage_retry.reason == SKIP_REASON + + original = make_pr(headRefOid="old-head") + monkeypatch.setattr( + sched, + "wait_for_updated_branch_head", + lambda repo, pr: make_pr( + headRefOid="new-head", + statusCheckRollup={"contexts": {"nodes": [strix_check()]}}, + ), + ) + followup_reason = sched.post_update_branch_followup( + "owner/repo", + original, + dry_run=False, + trigger_reviews=True, + review_dispatch_allowed=True, + workflow="OpenCode Review", + security_workflow="Strix Security Scan", + stale_opencode_minutes=45, + ) + assert SKIP_REASON in followup_reason From 49eb9e7035a6994fffb5b24bf943156be27a02fb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 02:08:57 +0900 Subject: [PATCH 33/80] test(actions): anchor cancel-in-progress contracts so a flipped flag fails (#1979) strix.yml's workflow-level cancel-in-progress was guarded by nothing: its own test asserted the string as a substring, and the list in test_required_pull_request_workflows_cancel_superseded_runs did not include it. Authored and verified by separate sessions. Merger verification, my own runs: head agreed across three paths (local ref, ls-remote, PR head) after the author hit a push/PR-creation mismatch on this branch; 0 behind main; merge tree identical to the branch tree; 13 files, all under tests/. Assertion forms: 24 helper calls, 4 line-anchored regexes, 19 assertions replaced. Two-way control, comment out strix.yml's flag and set it false: main reports 2968 passed and does not catch it; this branch fails 2 tests. Gate on the branch tree: 2968 passed, 1 skipped, coverage 100% (13196 statements, 0 missed), interrogate 100%. Four workflows are deliberately left uncontracted because no test states their intended value, and writing one would invent policy. One of them, scheduled-security-scan.yml, does produce the required context 'Detect CodeQL languages' -- which strengthens rather than weakens that choice: its cancel-in-progress: true with a github.ref-shared group is what starved it to 228 cancellations and 0 completions over 2026-09-01..09-05, so the correct value is an open policy question on #1800, not a contract to fix here. Authorization is this session's standing instruction to clear the queue, which lives outside this repository's text. Corroborating, not authorizing: on this head none of the 12 required contexts can pass -- the CodeQL pair is unreachable while codeql-scan-dispatch has never succeeded (#1929). Co-Authored-By: Claude Opus 5 --- .../test_agent_mention_downstream_idempotency.py | 5 ++++- tests/test_agent_mention_queue_isolation.py | 8 +++++++- ..._agent_review_runtime_quality_consolidation.py | 6 +++++- tests/test_bootstrap_codeql_pull_requests.py | 6 +++++- tests/test_close_empty_pr_queue_pressure.py | 3 ++- .../test_exact_artifact_quality_single_runner.py | 6 +++++- tests/test_noema_review_gate.py | 6 +++++- ...est_noema_token_lifetime_stale_run_contract.py | 5 ++++- tests/test_opencode_required_rerun_capacity.py | 5 ++++- .../test_opencode_required_verdict_regression.py | 10 +++++++--- tests/test_pr_review_fix_scheduler_source_pin.py | 6 +++++- tests/test_repository_metadata_workflow_pages.py | 8 +++++++- tests/test_required_workflow_queue_contract.py | 15 ++++++++++----- 13 files changed, 70 insertions(+), 19 deletions(-) diff --git a/tests/test_agent_mention_downstream_idempotency.py b/tests/test_agent_mention_downstream_idempotency.py index 85974a5719..cb3c31763e 100644 --- a/tests/test_agent_mention_downstream_idempotency.py +++ b/tests/test_agent_mention_downstream_idempotency.py @@ -1,5 +1,8 @@ """Static contracts for downstream review-agent invocation idempotency.""" +from tests.test_required_workflow_queue_contract import ( + workflow_level_cancels_in_progress, +) from pathlib import Path ROOT = Path(__file__).resolve().parents[1] @@ -54,7 +57,7 @@ def test_downstream_workflows_claim_artifacts_and_coalesce_by_pull_request() -> f"group: {workflow_name}-${{{{ github.event.client_payload.target_repository }}}}-${{{{ github.event.client_payload.pr_number || github.run_id }}}}" in concurrency ) - assert "cancel-in-progress: true" in concurrency + assert workflow_level_cancels_in_progress(text) assert "queue: max" not in text assert "^[0-9a-f]{64}$" in text assert "^[1-9][0-9]*$" in text diff --git a/tests/test_agent_mention_queue_isolation.py b/tests/test_agent_mention_queue_isolation.py index e93ae61aed..d751076661 100644 --- a/tests/test_agent_mention_queue_isolation.py +++ b/tests/test_agent_mention_queue_isolation.py @@ -2,6 +2,9 @@ from __future__ import annotations +import re + + from pathlib import Path ROOT = Path(__file__).resolve().parents[1] @@ -68,4 +71,7 @@ def test_interactive_queue_retires_older_requests_for_only_the_same_pr() -> None concurrency = _concurrency_block(local_job) assert "github.event.issue.number || github.run_id" in concurrency - assert "cancel-in-progress: true" in concurrency + # Anchored on the JOB block, not the workflow-level helper: this router + # declares no workflow-level concurrency, so the sibling helper would raise + # rather than read the block this test is about. + assert re.search(r"(?m)^[ \t]+cancel-in-progress:[ \t]+true[ \t]*$", concurrency) diff --git a/tests/test_agent_review_runtime_quality_consolidation.py b/tests/test_agent_review_runtime_quality_consolidation.py index b0c90eb707..4592cfd166 100644 --- a/tests/test_agent_review_runtime_quality_consolidation.py +++ b/tests/test_agent_review_runtime_quality_consolidation.py @@ -2,6 +2,10 @@ from __future__ import annotations +from tests.test_required_workflow_queue_contract import ( + workflow_level_cancels_in_progress, +) + import re import subprocess from pathlib import Path @@ -56,7 +60,7 @@ def test_pr_concurrency_cancels_only_the_same_workflow_repository_and_pr() -> No "${{ github.repository }}-${{ github.event.pull_request.number }}" in concurrency_contract ) - assert "cancel-in-progress: true" in concurrency_contract + assert workflow_level_cancels_in_progress(workflow) assert "github.sha" not in concurrency_contract assert "head.sha" not in concurrency_contract assert "github.ref" not in concurrency_contract diff --git a/tests/test_bootstrap_codeql_pull_requests.py b/tests/test_bootstrap_codeql_pull_requests.py index eb20c3d1e0..12fd1d8b52 100644 --- a/tests/test_bootstrap_codeql_pull_requests.py +++ b/tests/test_bootstrap_codeql_pull_requests.py @@ -2,6 +2,10 @@ from __future__ import annotations +from tests.test_required_workflow_queue_contract import ( + workflow_level_cancels_in_progress, +) + from io import StringIO import json import subprocess @@ -67,7 +71,7 @@ def test_rendered_workflow_redetects_stacks_and_pins_every_action() -> None: assert "pull_request:" not in workflow assert "github.event.pull_request" not in workflow assert "github.event_name == 'push' && github.ref || github.event_name" in workflow - assert "cancel-in-progress: true" in workflow + assert workflow_level_cancels_in_progress(workflow) assert workflow.count("@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9") == 2 assert "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0" in workflow diff --git a/tests/test_close_empty_pr_queue_pressure.py b/tests/test_close_empty_pr_queue_pressure.py index 331a604631..6da88f63f1 100644 --- a/tests/test_close_empty_pr_queue_pressure.py +++ b/tests/test_close_empty_pr_queue_pressure.py @@ -1,5 +1,6 @@ """Regression contracts for close-event runner admission pressure.""" +import re from pathlib import Path import pytest @@ -29,7 +30,7 @@ def test_closed_pull_request_does_not_allocate_a_noop_runner( assert "closed" in workflow assert "github.event.pull_request.number" in concurrency assert "github.event.pull_request.head.sha" not in concurrency - assert "cancel-in-progress:" in concurrency + assert re.search(r"(?m)^[ \t]+cancel-in-progress:[ \t]+\S", concurrency) assert "cancel-closed-pr-runs:" not in workflow assert "github.event.action != 'closed'" in workflow assert evidence_job in workflow diff --git a/tests/test_exact_artifact_quality_single_runner.py b/tests/test_exact_artifact_quality_single_runner.py index b8711ab7a5..9378e2232c 100644 --- a/tests/test_exact_artifact_quality_single_runner.py +++ b/tests/test_exact_artifact_quality_single_runner.py @@ -2,6 +2,10 @@ from __future__ import annotations +from tests.test_required_workflow_queue_contract import ( + workflow_level_cancels_in_progress, +) + import re from pathlib import Path @@ -68,7 +72,7 @@ def test_pr_concurrency_uses_workflow_repository_and_pr_identity() -> None: "${{ github.event.pull_request.number }}" in concurrency ) - assert "cancel-in-progress: true" in concurrency + assert workflow_level_cancels_in_progress(workflow) assert "github.sha" not in concurrency assert "pull_request.head.sha" not in concurrency diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index 5fa23dec53..e8a0dd6f59 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -1,3 +1,7 @@ + +from tests.test_required_workflow_queue_contract import ( + workflow_level_cancels_in_progress, +) import base64 import hashlib import http.client @@ -60,7 +64,7 @@ def test_noema_concurrency_and_live_head_cleanup_preserve_current_review(): workflow = Path(".github/workflows/noema-review.yml").read_text(encoding="utf-8") concurrency = workflow.split("concurrency:", 1)[1].split("permissions:", 1)[0] assert "github.event.workflow_run" not in concurrency - assert "cancel-in-progress: true" in concurrency + assert workflow_level_cancels_in_progress(workflow) admission = workflow.split("\n admit-current-head:\n", 1)[1].split( "\n cancel-closed-pr-runs:", 1 )[0] diff --git a/tests/test_noema_token_lifetime_stale_run_contract.py b/tests/test_noema_token_lifetime_stale_run_contract.py index 77a64cabdb..108647907a 100644 --- a/tests/test_noema_token_lifetime_stale_run_contract.py +++ b/tests/test_noema_token_lifetime_stale_run_contract.py @@ -1,5 +1,8 @@ """Regression contract for consolidated Noema quality-run retirement.""" +from tests.test_required_workflow_queue_contract import ( + workflow_level_cancels_in_progress, +) from pathlib import Path @@ -26,4 +29,4 @@ def test_noema_token_lifetime_quality_ci_retires_superseded_pr_runs() -> None: assert "github.event.pull_request.head.sha" not in concurrency_contract assert "github.sha" not in concurrency_contract assert "github.ref" not in concurrency_contract - assert "cancel-in-progress: true" in concurrency_contract + assert workflow_level_cancels_in_progress(workflow) diff --git a/tests/test_opencode_required_rerun_capacity.py b/tests/test_opencode_required_rerun_capacity.py index 431d3a8bc2..c85bc24e3c 100644 --- a/tests/test_opencode_required_rerun_capacity.py +++ b/tests/test_opencode_required_rerun_capacity.py @@ -1,5 +1,8 @@ """Capacity contract for Required OpenCode dispatch and exact-run wakeup.""" +from tests.test_required_workflow_queue_contract import ( + workflow_level_cancels_in_progress, +) import json import os from pathlib import Path @@ -48,7 +51,7 @@ def test_native_cancellation_runs_before_runner_admission() -> None: assert "required-opencode-review-${{" in concurrency assert "github.event.pull_request.number || github.run_id" in concurrency - assert "cancel-in-progress: true" in concurrency + assert workflow_level_cancels_in_progress(required) assert "live_head_matches()" in required diff --git a/tests/test_opencode_required_verdict_regression.py b/tests/test_opencode_required_verdict_regression.py index f29b97a663..5c5325d1aa 100644 --- a/tests/test_opencode_required_verdict_regression.py +++ b/tests/test_opencode_required_verdict_regression.py @@ -2,6 +2,10 @@ from __future__ import annotations +from tests.test_required_workflow_queue_contract import ( + workflow_level_cancels_in_progress, +) + import json import os import re @@ -86,7 +90,7 @@ def test_opencode_dispatch_uses_the_same_target_repo_pr_group() -> None: assert "opencode-review-${{" in dispatched assert "needs.validate-pr-metadata.outputs.target_repository" in dispatched assert "needs.validate-pr-metadata.outputs.pr_number || github.run_id" in dispatched - assert "cancel-in-progress: true" in dispatched + assert workflow_level_cancels_in_progress(dispatched) assert dispatched.index("validate-pr-metadata:") < dispatched.index(" concurrency:") @@ -621,7 +625,7 @@ def test_opencode_review_trigger_reacts_to_draft_conversion() -> None: "types: [opened, synchronize, reopened, ready_for_review, " "converted_to_draft, closed]" ) in trigger_block - assert "cancel-in-progress: true" in workflow.split("\npermissions:\n", 1)[0] + assert workflow_level_cancels_in_progress(workflow) def test_opencode_review_concurrency_group_is_workflow_level_repo_and_pr() -> None: @@ -637,7 +641,7 @@ def test_opencode_review_concurrency_group_is_workflow_level_repo_and_pr() -> No assert "required-opencode-review-${{" in concurrency_block assert "github.event.pull_request.head.sha || github.run_id" not in concurrency_block assert "github.event.pull_request.number || github.run_id" in concurrency_block - assert "cancel-in-progress: true" in concurrency_block + assert workflow_level_cancels_in_progress(workflow) assert " concurrency:" not in target_job.split(" permissions:", 1)[0] admission = workflow.split("\n admit-current-head:\n", 1)[1].split( "\n coverage-source-tree:", 1 diff --git a/tests/test_pr_review_fix_scheduler_source_pin.py b/tests/test_pr_review_fix_scheduler_source_pin.py index 7958ba5163..0f9e0adb1c 100644 --- a/tests/test_pr_review_fix_scheduler_source_pin.py +++ b/tests/test_pr_review_fix_scheduler_source_pin.py @@ -2,6 +2,10 @@ from __future__ import annotations +from tests.test_required_workflow_queue_contract import ( + workflow_level_cancels_in_progress, +) + from pathlib import Path @@ -88,7 +92,7 @@ def test_reusable_scheduler_retains_least_privilege_and_bounded_dispatch() -> No assert "pull-requests: write" not in workflow assert "MAX_DISPATCHES:" in workflow assert "RETRY_HOURS:" in workflow - assert "cancel-in-progress: true" in workflow + assert workflow_level_cancels_in_progress(workflow) def test_reusable_scheduler_bounds_both_oidc_exchange_requests() -> None: diff --git a/tests/test_repository_metadata_workflow_pages.py b/tests/test_repository_metadata_workflow_pages.py index 82aa4462f7..5c05dfbe3d 100644 --- a/tests/test_repository_metadata_workflow_pages.py +++ b/tests/test_repository_metadata_workflow_pages.py @@ -2,6 +2,8 @@ from __future__ import annotations +import re + import importlib.util import json from pathlib import Path @@ -40,7 +42,11 @@ def test_metadata_pr_validation_cancels_superseded_head_runs() -> None: concurrency = workflow.split("concurrency:", 1)[1].split("jobs:", 1)[0] assert "group: repository-metadata-reconcile-${{ github.ref }}" in concurrency - assert "cancel-in-progress: ${{ github.event_name == 'pull_request' }}" in concurrency + assert re.search( + r"(?m)^[ \t]+cancel-in-progress:[ \t]+\$\{\{ github\.event_name == 'pull_request' \}\}" + r"[ \t]*$", + concurrency, + ) assert "github.event.pull_request.head.sha" not in concurrency diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 512cb6bef5..8bc0791631 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -402,7 +402,12 @@ def test_agent_mention_router_keeps_its_two_distinct_job_groups() -> None: assert "group: review-agent-mention-router-sweep-${{ github.repository }}" in workflow sweep = workflow.split("sweep-organization-agent-mentions:", 1)[1] - assert "cancel-in-progress: false" in sweep.split("steps:", 1)[0] + # Anchored on the sweep JOB block: this router declares no workflow-level + # concurrency, so the sibling helper would raise rather than read it. + assert re.search( + r"(?m)^[ \t]+cancel-in-progress:[ \t]+false[ \t]*$", + sweep.split("steps:", 1)[0], + ) def test_concurrency_group_slice_ignores_the_comment_that_documents_it() -> None: """A comment quoting the key must not satisfy an assertion about the key. @@ -721,7 +726,7 @@ def test_strix_serializes_provider_evidence_per_repository_and_pr() -> None: assert "github.run_id" in group_value assert "github.event.pull_request.head.sha" not in concurrency_contract assert "github.event.client_payload.pr_head_sha" not in concurrency_contract - assert "cancel-in-progress: true" in concurrency_contract + assert workflow_level_cancels_in_progress(workflow) assert " concurrency:" not in strix_job.split(" permissions:", 1)[0] assert "queue: max" not in workflow assert workflow.index("admit-current-head:") < workflow.index("\n strix:\n") @@ -981,7 +986,7 @@ def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() - # outside that queue so synchronize and close events can retire old work. assert "admit-current-head:" in strix_workflow assert "skipping stale evidence" in strix_workflow - assert "cancel-in-progress: true" in strix_workflow + assert workflow_level_cancels_in_progress(strix_workflow) def test_merge_scheduler_owns_empty_pr_cleanup_without_checkout() -> None: @@ -1043,7 +1048,7 @@ def test_noema_triggers_preserve_standalone_pull_request_review() -> None: assert "github.event_name" not in concurrency_contract.split( "cancel-in-progress:", 1 )[0] - assert "cancel-in-progress: true" in concurrency_contract + assert workflow_level_cancels_in_progress(workflow) assert re.search(r"(?m)^concurrency:", workflow) assert not re.search(r"(?m)^ concurrency:", workflow) assert "needs.admit-current-head.outputs.admitted == 'true'" in noema_job @@ -1360,7 +1365,7 @@ def test_fix_scheduler_cancels_superseded_cron_runs() -> None: workflow = workflow_text("pr-review-fix-scheduler.yml") assert "central-pr-review-fix-scheduler-" in workflow - assert "cancel-in-progress: true" in workflow + assert workflow_level_cancels_in_progress(workflow) def test_security_scan_fails_closed_when_dependency_review_is_unavailable() -> None: From 5c60b5d9e8fb8461480d6a30b0b5f149753afb12 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 03:16:17 +0900 Subject: [PATCH 34/80] test(concurrency): pin the cancel flag as a value for the PR-keyed scan workflows (#1980) python-security.yml and sast-semgrep.yml had only the presence of cancel-in-progress asserted, so flipping it to false passed the whole suite. This pins the value. Co-Authored-By: Claude Opus 5 --- .../test_required_workflow_queue_contract.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 8bc0791631..4ab09b1d0c 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -905,6 +905,32 @@ def test_strix_draft_transition_cancels_current_scan(tmp_path: Path) -> None: assert "/actions/runs/100/cancel" in calls +def test_pr_keyed_scan_workflows_pin_cancellation_as_a_value() -> None: + """Pin `cancel-in-progress` for the two PR-keyed scans that only had presence. + + Both appear in ``test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs``, + but in the branch that asserts the key is *present* rather than what it says. + That branch is shaped by ``pr-review-merge-scheduler.yml``, whose value is + deliberately an expression over ``github.event_name``, so the loop cannot + assert a constant for everyone in it. Nothing else read the flag: flipping + either to ``false`` left the whole suite green (2968 passed, 0 failed, + measured 2026-09-06). + + Kept out of ``test_required_pull_request_workflows_cancel_superseded_runs`` + because that loop ends by requiring a ``github.event_name`` discriminator in + the group, and these two key on + ``pull_request.number || github.ref`` with no event-name term. Adding them + there would need a branch that asserts nothing. + """ + for filename in ("python-security.yml", "sast-semgrep.yml"): + workflow = workflow_text(filename) + group_value = workflow_level_concurrency_group(workflow) + + assert workflow_level_cancels_in_progress(workflow) + assert "github.event.pull_request.number" in group_value + assert "github.event_name" not in group_value + + def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() -> None: """Close events should cancel old runs without starting expensive jobs.""" workflows = ( From 74224b2091701a754b2fd1a539833c7a6d50b79e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 03:34:54 +0900 Subject: [PATCH 35/80] fix(scheduler): match the run name GitHub actually sends for central review runs (#1983) `active_review_run_refs` matched a workflow run's `name` exactly against the review workflow aliases. But eight workflows in this repository define `run-name:`, and that set contains every workflow whose runs this matcher looks for -- `opencode-review.yml` ("Required OpenCode Review"), `opencode-review-dispatch.yml` ("OpenCode Review Dispatch") and `strix.yml` ("Strix Security Scan"). For such a workflow GitHub reports the *rendered* run name in `name` -- the same string as `display_title`, e.g. OpenCode Review Dispatch ContextualWisdomLab/.github#834@e748ee61... Sampled 2026-09-07: 100 of 100 opencode-review-dispatch runs carry that form and none carries the bare workflow name. So the exact match dropped every production dispatch run at this line, before the `event == "repository_dispatch"` branch immediately below that exists to read them. Two consequences: * `already_running` never suppressed a same-head repeat. .github#1529 took 27 dispatches on one unchanged head over 100.8 hours; each new run's creation preceded the previous run's cancellation by about three seconds, so the previous run was demonstrably still active when the check ran and did not see it. * `stale` never populated, so older-head central runs were never cancelled. A first count of the live queue said "20 duplicates of 45 active runs" and was wrong: it grouped by repository and PR without the workflow, so runs of different dispatch workflows on one PR were counted as duplicates of each other. Regrouped by (workflow, repository, PR): active repository_dispatch runs, queued + in_progress 33 codeql-scan-dispatch.yml 25 runs / 12 keys / 13 same-head duplicates opencode-review-dispatch.yml 5 runs / 5 keys / 0 duplicates pr-review-autofix.yml 3 runs / 3 keys / 0 duplicates So the workflows this matcher governs show no live duplication at this instant. The harm this fix addresses is the historical chain on .github#1529 and a suppression that has never once fired, not a backlog visible right now. The 13 duplicates all belong to CodeQL Scan Dispatch, which this matcher does not govern; that workflow also defines `run-name:`, which makes it a separate lead rather than evidence for this change. Reviving stale cancellation is separately safe: of 163 non-terminal central runs, 49 are review or dispatch kind and 4 become cancellable, all of them subjects that no longer exist (3 closed or merged PRs, 1 moved head). The fix is confined to the run comparison. `OPENCODE_WORKFLOW_NAMES` is unchanged, because its other consumer compares a *workflow* object's name, which is genuinely bare. `active_review_run_refs` has exactly two call sites, OpenCode's and Strix's, so both are fixed here; the Strix side is pinned by its own test so a later narrowing to the OpenCode aliases cannot silently reopen half of it. This is one instance of a class, and the file already contains the stable form. `run.name` is compared as an identifier at four places -- `:1250`, `:3198`, `:3254` (this one) and `:3783` -- while `:3060` keys on `run.get("workflow_id") or run.get("path") or run.get("name")`, which cannot be rewritten by a `run-name:`. `:1250` in particular feeds the REST fallback's workflow-level policy boundary and would see a rendered title where it expects a workflow name. Fixing the whole class means moving the callers from display names to paths, which also touches how `dispatch_title_prefixes` is built, so it is deliberately left out of this change; .github#1941 is the same root seen from the `display_title` side. Recorded here so the next reader does not rediscover it as a fifth instance. Note the new behaviour this enables: while a same-head central run is active, a repeat is now suppressed. A run that never terminates would therefore hold the PR, where before the check simply never fired. The existing fixture sets a bare `name` beside a rendered `display_title`, a payload GitHub never emits for a `run-name:` workflow, which is why 100 percent line coverage of that branch never revealed that production could not reach it. Developer experience: the scheduler's same-head suppression and stale-run cancellation work against real payloads instead of a shape only the tests produce. User experience: a pull request stops accumulating duplicate concurrent review runs that cancel each other, so a review that starts can finish. Co-authored-by: Claude Opus 5 --- scripts/ci/pr_review_merge_scheduler_core.py | 19 +++- tests/test_pr_review_merge_scheduler.py | 95 ++++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/scripts/ci/pr_review_merge_scheduler_core.py b/scripts/ci/pr_review_merge_scheduler_core.py index cffb52cb53..e535025a86 100644 --- a/scripts/ci/pr_review_merge_scheduler_core.py +++ b/scripts/ci/pr_review_merge_scheduler_core.py @@ -3254,7 +3254,24 @@ def active_review_run_refs( for run_repo in (dispatch_repo,): for run_data in active_workflow_runs(run_repo, statuses): run_name = str(run_data.get("name") or "") - if run_name != workflow and run_name not in workflow_aliases: + # GitHub reports the *rendered* ``run-name:`` in a run's ``name``, + # not the workflow name, and eight workflows here define one -- + # every workflow whose runs this matcher looks for + # (``opencode-review.yml`` = "Required OpenCode Review", + # ``opencode-review-dispatch.yml`` = "OpenCode Review Dispatch", + # ``strix.yml`` = "Strix Security Scan") is among them. Exact + # matching therefore dropped every production dispatch run here, + # before the ``repository_dispatch`` branch below that exists to + # handle it: ``already_running`` never suppressed a same-head + # repeat and ``stale`` never populated, so .github#1529 took 27 + # dispatches on one unchanged head and older-head central runs were + # never cancelled. Sampled 2026-09-07: 100 of 100 + # opencode-review-dispatch runs carry the rendered form, 0 bare. + # Accept it -- the workflow name, then a space, then the suffix. + if not any( + run_name == candidate or run_name.startswith(f"{candidate} ") + for candidate in (workflow, *workflow_aliases) + ): continue run_id = run_data.get("id") if not run_id: diff --git a/tests/test_pr_review_merge_scheduler.py b/tests/test_pr_review_merge_scheduler.py index 8b924291a5..b653efea18 100644 --- a/tests/test_pr_review_merge_scheduler.py +++ b/tests/test_pr_review_merge_scheduler.py @@ -6178,6 +6178,101 @@ def test_dispatch_strix_waits_for_active_target_repository_run(monkeypatch, caps assert "target repository already has active run(s) ContextualWisdomLab/.github@9350" in capsys.readouterr().out +def test_central_run_filter_accepts_the_run_name_github_actually_sends(monkeypatch): + """A ``run-name:`` workflow reports the rendered title in ``name``. + + ``opencode-review-dispatch.yml``, ``strix.yml`` and ``noema-review.yml`` all + define ``run-name:``, so GitHub sets each run's ``name`` to the rendered + string, identical to ``display_title`` -- sampled 2026-09-07, 100 of 100 + opencode-review-dispatch runs carry that form and none carries the bare + workflow name. Matching ``name`` exactly against the aliases dropped every + one of them before the ``repository_dispatch`` branch that exists to read + them, so ``already_running`` never suppressed a same-head repeat and + ``stale`` never populated: .github#1529 took 27 dispatches on one unchanged + head, and older-head central runs were never cancelled. + + The neighbouring fixture below sets a bare ``name`` alongside a rendered + ``display_title``, which is why 100% coverage of that branch never showed + that production could not reach it. + """ + head_sha = "a" * 40 + stale_sha = "b" * 40 + current_title = f"Required OpenCode Review owner/repo#1@{head_sha}" + stale_title = f"Required OpenCode Review owner/repo#1@{stale_sha}" + central_runs = [ + { + "id": 9500, + "name": current_title, + "display_title": current_title, + "event": "repository_dispatch", + }, + { + "id": 9501, + "name": stale_title, + "display_title": stale_title, + "event": "repository_dispatch", + }, + ] + + def fake_active_runs(repo, statuses=("queued", "in_progress")): + del statuses + return central_runs if repo == "ContextualWisdomLab/.github" else [] + + monkeypatch.setattr(sched, "active_workflow_runs", fake_active_runs) + monkeypatch.setenv( + "SCHEDULER_REQUIRED_WORKFLOW_REPOSITORY", + "ContextualWisdomLab/.github", + ) + + assert sched.active_opencode_run_refs( + "owner/repo", + "OpenCode Review", + make_pr(headRefOid=head_sha), + ) == ( + [("ContextualWisdomLab/.github", "9500")], + [("ContextualWisdomLab/.github", "9501")], + ) + + +def test_central_run_filter_reads_the_rendered_strix_run_name_too(monkeypatch): + """Strix shares the matcher, and ``strix.yml`` also defines ``run-name:``. + + ``active_review_run_refs`` has exactly two call sites -- OpenCode's and + ``dispatch_strix_evidence``'s -- so the exact-``name`` match blinded both. + Pinning the Strix side here keeps a later narrowing of the fix to the + OpenCode aliases from silently reopening the Strix half. + """ + head_sha = "c" * 40 + current_title = f"Strix Security Scan owner/repo#1@{head_sha}" + + def fake_active_runs(repo, statuses=("queued", "in_progress")): + del statuses + if repo != "ContextualWisdomLab/.github": + return [] + return [ + { + "id": 9600, + "name": current_title, + "display_title": current_title, + "event": "repository_dispatch", + } + ] + + monkeypatch.setattr(sched, "active_workflow_runs", fake_active_runs) + monkeypatch.setenv( + "SCHEDULER_REQUIRED_WORKFLOW_REPOSITORY", + "ContextualWisdomLab/.github", + ) + + assert sched.active_review_run_refs( + "owner/repo", + "Strix Security Scan", + make_pr(headRefOid=head_sha), + run_title="Strix Security Scan", + workflow_aliases=frozenset({"Strix Security Scan"}), + ) == ([("ContextualWisdomLab/.github", "9600")], []) + + def test_central_run_filter_ignores_malformed_and_non_dispatch_titles(monkeypatch): head_sha = "a" * 40 central_runs = [ From 2396ddcaf4b6c50785548b313ccddfa1784915a2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 03:39:28 +0900 Subject: [PATCH 36/80] Revert "fix(scheduler): skip review dispatch for a head whose merge tree cannot materialize (#1973)" (#1985) This reverts commit ad0779bee66624c3997947d7691f4b0dbb973be1. I wrote that guard and its justification is false. It blocked every OpenCode dispatch on a DIRTY/CONFLICTING head on the grounds that "a conflicting head can only produce a failed dispatch". The run object does conclude failure, but the review is published anyway: the reviewer reads the pull request diff, not a merge tree -- "Coverage is a separate gate", in the review's own words -- and only `coverage-source-tree` needs the merge commit. Measured on .github#1529, the PR that motivated the guard: the last of its 27 dispatches published a 2404-character review at 2026-09-05T19:40:58Z. That review body carries its own run id, 33969161561, which concluded failure at the receipt gate nine seconds later. It is the only OpenCode review that head has (4 reviews total, 1 by opencode-agent[bot]) and it is why the PR now reads as reviewed. The guard would have discarded it. The pattern is not unique to #1529: .github#1555 is CONFLICTING right now and its current head carries a 2019-character CHANGES_REQUESTED review from the same reviewer. I reached "produced nothing" by reading run conclusions, which are roll-ups that cannot name what a run did. The cost the guard claimed to protect is also wrong by two orders of magnitude. Across all 27 dispatches of that head, 109 jobs: 27 were allocated a runner for 0.24 h in total, 82 never were, and the rest of the elapsed time was queue residency. Blocking the whole chain would have saved fourteen minutes of runner time. The repeat itself is a real problem, and it is fixed at its cause rather than here. `active_review_run_refs` matched a run's `name` exactly against the review workflow aliases, but the central review workflows define `run-name:`, so GitHub sends the rendered title in that field and every dispatch run was filtered out before the check could see it -- `already_running` never fired. With that repaired, a conflicting head receives one dispatch and the next is suppressed while it runs, which is the outcome this guard was reaching for without discarding the review. Developer experience: the scheduler no longer prints a skip reason that asserts an outcome contradicted by the runs it cites. User experience: a conflicting pull request receives a review and repair guidance, instead of repair guidance alone. Co-authored-by: Claude Opus 5 --- scripts/ci/pr_review_merge_scheduler_core.py | 29 --- tests/test_pr_review_merge_scheduler.py | 177 ------------------- 2 files changed, 206 deletions(-) diff --git a/scripts/ci/pr_review_merge_scheduler_core.py b/scripts/ci/pr_review_merge_scheduler_core.py index e535025a86..4df4dac3de 100644 --- a/scripts/ci/pr_review_merge_scheduler_core.py +++ b/scripts/ci/pr_review_merge_scheduler_core.py @@ -2927,8 +2927,6 @@ def post_update_branch_followup( if wait_reason: return f"{head_note}; {wait_reason}" dispatch_result = dispatch_opencode_review(repo, workflow, updated_pr, dry_run=dry_run) - if dispatch_result == "merge_conflict": - return f"{head_note}; PR merge tree cannot be materialized while the head conflicts; review dispatch skipped" if dispatch_result == "admission_deferred": return f"{head_note}; bounded admission budget is exhausted" if dispatch_result == "already_running": @@ -3705,23 +3703,6 @@ def dispatch_opencode_review(repo: str, workflow: str, pr: dict[str, Any], *, dr return "already_running" if dry_run: return "dry_run" - if effective_merge_state(pr) in {"DIRTY", "CONFLICTING"}: - # Materializing the PR merge tree is a hard precondition of - # coverage-source-tree, so a conflicting head can only produce a failed - # dispatch. Returning before review_dispatch_admitted keeps the bounded - # admission budget for a PR a review could actually finish: measured on - # .github#1529, one conflicting head consumed 27 dispatches across 100.8 - # hours with zero successes (20 cancelled, 7 failed, and all 7 that - # reached coverage-source-tree died there; 2026-09-01T08:46Z..09-05T13:31Z). - # UNKNOWN is deliberately not blocked -- an - # uncomputed mergeability must not starve a reviewable PR. - print( - "OpenCode review dispatch skipped: GitHub reports the current head as " - f"{effective_merge_state(pr)}, so the PR merge tree cannot be materialized " - "and the review would fail. Repair the branch and push it, then the review " - "runs on the new head." - ) - return "merge_conflict" if not review_dispatch_admitted("opencode", repo, pr): return "admission_deferred" base_ref, base_sha, head_sha = validated_pr_dispatch_fields(pr) @@ -4170,8 +4151,6 @@ def dispatch_draft_review_only( f"draft PR review-only dispatch; current head has completed Strix evidence; {wait_reason}", ) dispatch_result = dispatch_opencode_review(repo, workflow, pr, dry_run=dry_run) - if dispatch_result == "merge_conflict": - return Decision(number, "wait", "draft PR review-only dispatch; PR merge tree cannot be materialized while the head conflicts; review dispatch skipped") if dispatch_result == "admission_deferred": return Decision(number, "wait", "draft PR review-only dispatch; bounded admission budget is exhausted") if dispatch_result == "already_running": @@ -4284,8 +4263,6 @@ def inspect_pr( if wait_reason: return Decision(number, "wait", f"stacked PR onto {base_ref}; {wait_reason}") dispatch_result = dispatch_opencode_review(repo, workflow, pr, dry_run=dry_run) - if dispatch_result == "merge_conflict": - return Decision(number, "wait", f"stacked PR onto {base_ref}; PR merge tree cannot be materialized while the head conflicts; review dispatch skipped") if dispatch_result == "admission_deferred": return Decision(number, "wait", f"stacked PR onto {base_ref}; bounded admission budget is exhausted") if dispatch_result == "already_running": @@ -4504,8 +4481,6 @@ def request_branch_update(freshness_reason: str, *, suffix: str = "") -> Decisio if wait_reason: return decide("wait", wait_reason) dispatch_result = dispatch_opencode_review(repo, workflow, pr, dry_run=dry_run) - if dispatch_result == "merge_conflict": - return decide("wait", "PR merge tree cannot be materialized while the head conflicts; review dispatch skipped") if dispatch_result == "admission_deferred": return decide("wait", "bounded admission budget is exhausted") if dispatch_result == "already_running": @@ -4921,8 +4896,6 @@ def request_branch_update(freshness_reason: str, *, suffix: str = "") -> Decisio f"OpenCode review exceeded {stale_opencode_minutes} minute retry threshold; review dispatch limit reached", ) dispatch_result = dispatch_opencode_review(repo, workflow, pr, dry_run=dry_run) - if dispatch_result == "merge_conflict": - return decide("wait", "PR merge tree cannot be materialized while the head conflicts; review dispatch skipped") if dispatch_result == "admission_deferred": return decide("wait", "bounded admission budget is exhausted") if dispatch_result == "already_running": @@ -4973,8 +4946,6 @@ def request_branch_update(freshness_reason: str, *, suffix: str = "") -> Decisio if wait_reason: return decide("wait", f"current head has completed Strix evidence; {wait_reason}") dispatch_result = dispatch_opencode_review(repo, workflow, pr, dry_run=dry_run) - if dispatch_result == "merge_conflict": - return decide("wait", "PR merge tree cannot be materialized while the head conflicts; review dispatch skipped") if dispatch_result == "admission_deferred": return decide("wait", "bounded admission budget is exhausted") if dispatch_result == "already_running": diff --git a/tests/test_pr_review_merge_scheduler.py b/tests/test_pr_review_merge_scheduler.py index b653efea18..ba47b89c8d 100644 --- a/tests/test_pr_review_merge_scheduler.py +++ b/tests/test_pr_review_merge_scheduler.py @@ -2330,75 +2330,6 @@ def test_dispatch_opencode_review_falls_back_to_bounded_discovery(monkeypatch): assert json.loads(dispatch_calls[0])["client_payload"]["required_run_id"] == 999 -def _dispatch_with_merge_state(monkeypatch, **overrides): - """Run the OpenCode dispatch funnel and report what it did.""" - monkeypatch.setenv("GITHUB_ACTIONS", "true") - monkeypatch.setenv("GH_TOKEN", "opencode-app-token") - monkeypatch.setattr( - sched, "active_opencode_run_refs", lambda repo, workflow, pr: ([], []) - ) - monkeypatch.setattr(sched, "discover_opencode_required_run_id", lambda repo, head_sha: None) - dispatched: list[str | None] = [] - monkeypatch.setattr( - sched, "run_github_dispatch", lambda args, stdin=None: dispatched.append(stdin) - ) - admitted: list[str] = [] - - def record_admission(component, repo, pr): - admitted.append(component) - return True - - monkeypatch.setattr(sched, "review_dispatch_admitted", record_admission) - pr = make_pr(headRefOid="a" * 40, baseRefOid="b" * 40, **overrides) - monkeypatch.setattr(sched, "fetch_pr", lambda *_args: [pr]) - result = sched.dispatch_opencode_review("owner/repo", "OpenCode Review", pr, dry_run=False) - return result, dispatched, admitted - - -def test_review_dispatch_skips_a_head_whose_merge_tree_cannot_materialize(monkeypatch): - """A conflicting head is skipped before it can spend the admission budget. - - coverage-source-tree must materialize the PR merge tree, which git cannot do - while the head conflicts, so the dispatch could only fail. Measured on - .github#1529: one conflicting head took 27 dispatches over 100.8 hours and - produced no review; the 7 that reached coverage-source-tree all died there, - and the other 20 were cancelled before they ever started it. - """ - for graph_state in ("DIRTY", "CONFLICTING"): - result, dispatched, admitted = _dispatch_with_merge_state( - monkeypatch, mergeStateStatus=graph_state - ) - assert result == "merge_conflict" - assert dispatched == [], f"{graph_state} must not reach the dispatch API" - assert admitted == [], f"{graph_state} must not consume the admission budget" - - -def test_review_dispatch_reads_the_rest_merge_state_not_only_graphql(monkeypatch): - """The skip honours REST mergeability, which outranks a stale GraphQL value.""" - result, dispatched, admitted = _dispatch_with_merge_state( - monkeypatch, mergeStateStatus="CLEAN", restMergeableState="DIRTY" - ) - assert result == "merge_conflict" - assert dispatched == [] - assert admitted == [] - - -def test_review_dispatch_still_runs_when_mergeability_is_not_yet_known(monkeypatch): - """UNKNOWN mergeability must not starve a reviewable PR. - - Negative control for the conflict skip: GitHub reports UNKNOWN while it is - still computing a merge commit, so blocking on it would defer every PR the - scheduler reached first. - """ - for graph_state in ("UNKNOWN", "BEHIND", "BLOCKED", "CLEAN"): - result, dispatched, admitted = _dispatch_with_merge_state( - monkeypatch, mergeStateStatus=graph_state - ) - assert result == "dispatched", f"{graph_state} must still dispatch" - assert len(dispatched) == 1 - assert admitted == ["opencode"] - - def test_central_progress_ignores_required_workflow_checkrun_placeholder( monkeypatch, ): @@ -10859,111 +10790,3 @@ def behind_with(nodes): assert "checks are still queued or running" not in resumed.reason assert sched.has_in_flight_check_runs(behind_with([])) is False - - -def _skip_opencode_dispatch(monkeypatch): - """Make the OpenCode dispatch funnel report an unmaterializable merge tree.""" - monkeypatch.setattr( - sched, "dispatch_opencode_review", lambda repo, workflow, pr, dry_run: "merge_conflict" - ) - monkeypatch.setattr( - sched, "dispatch_strix_evidence", lambda repo, workflow, pr, dry_run: "dispatched" - ) - - -SKIP_REASON = "PR merge tree cannot be materialized while the head conflicts; review dispatch skipped" - - -def test_every_review_dispatch_caller_reports_the_conflict_skip_truthfully(monkeypatch): - """No dispatch path may report a skipped conflicting head as a dispatch. - - Each caller's fall-through says the review was dispatched, so a new funnel - result that a caller does not handle would be reported as work that never - happened -- and that is the same telemetry used to find the treadmill this - skip removes. - """ - _skip_opencode_dispatch(monkeypatch) - - stacked = inspect(make_pr(baseRefName="develop")) - assert stacked.action == "wait" - assert stacked.reason == f"stacked PR onto develop; {SKIP_REASON}" - - draft = inspect( - make_pr(isDraft=True, statusCheckRollup={"contexts": {"nodes": [strix_check()]}}), - allow_draft_review_dispatch=True, - ) - assert draft.action == "wait" - assert draft.reason == f"draft PR review-only dispatch; {SKIP_REASON}" - - strix_done = inspect(make_pr(statusCheckRollup={"contexts": {"nodes": [strix_check()]}})) - assert strix_done.action == "wait" - assert strix_done.reason == SKIP_REASON - - stale = inspect( - make_pr( - statusCheckRollup={ - "contexts": { - "nodes": [ - opencode_check(started_at="2026-06-25T07:00:00Z"), - strix_check(), - ] - } - } - ) - ) - assert stale.action == "wait" - assert stale.reason == SKIP_REASON - - coverage_retry = inspect( - make_pr( - reviews={ - "nodes": [ - { - **opencode_review("CHANGES_REQUESTED", "head"), - "body": ( - "OpenCode cannot approve yet because required coverage evidence " - "did not pass. The coverage-evidence gate reported that required " - "test/docstring evidence was not proven." - ), - } - ] - }, - statusCheckRollup={ - "contexts": { - "nodes": [ - strix_check(), - { - "__typename": "CheckRun", - "name": "coverage-evidence", - "status": "COMPLETED", - "conclusion": "SUCCESS", - }, - {**opencode_check(status="COMPLETED"), "conclusion": "FAILURE"}, - ] - } - }, - ) - ) - assert coverage_retry.action == "wait" - assert coverage_retry.reason == SKIP_REASON - - original = make_pr(headRefOid="old-head") - monkeypatch.setattr( - sched, - "wait_for_updated_branch_head", - lambda repo, pr: make_pr( - headRefOid="new-head", - statusCheckRollup={"contexts": {"nodes": [strix_check()]}}, - ), - ) - followup_reason = sched.post_update_branch_followup( - "owner/repo", - original, - dry_run=False, - trigger_reviews=True, - review_dispatch_allowed=True, - workflow="OpenCode Review", - security_workflow="Strix Security Scan", - stale_opencode_minutes=45, - ) - assert SKIP_REASON in followup_reason From bf0bf0ab0c9ebcf4cea05f8c9219dc093f9ab351 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 04:08:50 +0900 Subject: [PATCH 37/80] fix(audit): stop a ruleset drift from disabling the CodeQL coverage detector (#1987) Two unrelated audits share one job in audit-central-ruleset.yml, and the ruleset step runs first. It has exited 1 since at least 2026-09-04 on owner-configured governance drift: ERROR: exactly two approving reviews are not required ERROR: last-push approval protection is disabled FAIL: ruleset 18156473 has 2 governance drift reason(s) Live values on ruleset 18156473 today are `required_approving_review_count: 1` and `require_last_push_approval: false`, against the 2 and true the audit asserts. Both are owner-configured settings, and this change does not touch either them or the audit's expectations: with every session sharing one GitHub identity and unable to approve another's pull request, a two-approval requirement may well have been relaxed deliberately, in which case the stale side is the assertion rather than the configuration. Deciding that is an owner call. What is not an owner call is the collateral damage. Because the failure exits a shared job, the two steps below it never ran: the CodeQL coverage detector and the backlog-38 bootstrap that opens CodeQL setup pull requests. So the detector that would have reported a coverage gap has been dead for days, and its workflow was red the whole time for an unrelated reason -- red status, wrong subject, and no signal about coverage either way. The coverage step now carries `if: always()`. It builds its own repository list into its own temp file and the step above exports nothing to GITHUB_ENV or GITHUB_OUTPUT, so it has no data dependency to lose; the job still fails overall. The bootstrap step deliberately does not get the same guard, because it opens pull requests and running a mutation after an unexplained upstream failure is a different decision from running a read-only detector. A contract test pins both halves. The detector also needed a correction of its own. It accepted `default_setup_state == "configured"` as coverage, but a repository can report `configured` with an **empty** `languages` list, which scans nothing. Measured 2026-09-07: life-os, aFIPC and inkspan all report that shape, and life-os has zero CodeQL analyses of any language while codeql-pr.yml still runs on every pull request head. The control holds in both directions -- html4tree, naruon and wardnet have non-empty language lists and do have `dynamic/` analyses for exactly those languages. The audit workflow now collects `languages` alongside `state`, the predicate requires a non-empty list, and a payload missing the new key fails closed rather than falling back to the state alone. Gaps of the two kinds are reported as different sentences, because they need different fixes: enable languages on an existing setup, versus set coverage up at all. A reviewer then asked whether the newly-unblocked step could run and audit zero repositories while still passing, and one layer below the fix it could: $ echo '[]' | python3 scripts/ci/audit_org_codeql_coverage.py PASS: all 0 repositories have real CodeQL coverage exit 0 The calling workflow already refuses that -- its sentinel check requires known private repositories to appear in the enumeration, which an empty list fails -- but the script is directly runnable against a JSON path or stdin, so the guard did not cover every entry point. `main` now refuses an empty payload. This is the same vacuous-pass shape as the `configured`-with-no-languages case above, one level down, which is where it was found: a pass that examines nothing is not a pass. The step's independence from the failing step above it is established mechanically rather than by reading the YAML. Steps can only share state through `GITHUB_ENV`, `GITHUB_OUTPUT`, `GITHUB_PATH` or files, since each `run:` is a separate shell. The ruleset step uses none of those channels, and the two steps' `$RUNNER_TEMP` paths are disjoint: `central-required-workflow-*`, `ruleset-probe-*` and `stacked-opencode-ruleset.*` against `codeql-coverage-*`, `codeql-analysis-*` and `codeql-default-setup-*`. Developer experience: a governance-drift failure no longer hides an unrelated security-coverage result, and the audit says which of the two problems a repository has. User experience: a repository whose CodeQL setup is switched on but scanning nothing is reported as uncovered instead of counted as covered. Co-authored-by: Claude Opus 5 --- .github/workflows/audit-central-ruleset.yml | 33 +++++-- scripts/ci/audit_org_codeql_coverage.py | 62 ++++++++++++- tests/test_audit_org_codeql_coverage.py | 90 +++++++++++++++++++ ...central_required_workflow_ruleset_audit.py | 39 +++++++- 4 files changed, 212 insertions(+), 12 deletions(-) diff --git a/.github/workflows/audit-central-ruleset.yml b/.github/workflows/audit-central-ruleset.yml index bf24e36c7c..2b72f21aab 100644 --- a/.github/workflows/audit-central-ruleset.yml +++ b/.github/workflows/audit-central-ruleset.yml @@ -105,6 +105,18 @@ jobs: python3 scripts/ci/audit_central_required_workflows.py --stacked "$stacked_ruleset_json" - name: Audit organization CodeQL coverage + # Runs even when the ruleset step above failed. Those two audits share a + # job but not a subject: the ruleset step exits 1 on owner-configured + # governance drift, and on 2026-09-06 it did exactly that ("exactly two + # approving reviews are not required", "last-push approval protection is + # disabled"), which silently took this CodeQL coverage detector down with + # it -- every run since 2026-09-04 failed there and never reached this + # step. This step builds its own repository list into its own temp file + # and the step above exports nothing to GITHUB_ENV or GITHUB_OUTPUT, so + # it has no data dependency to lose. The job still fails overall; what + # changes is that a coverage gap is reported instead of hidden behind an + # unrelated failure. + if: always() env: ORG_LOGIN: ContextualWisdomLab ORG_WIDE_CREDENTIAL_AVAILABLE: ${{ secrets.PR_REVIEW_MERGE_TOKEN != '' || secrets.OPENCODE_APPROVE_TOKEN != '' }} @@ -165,13 +177,21 @@ jobs: printf '[]\n' >"$coverage_json" while IFS=$'\t' read -r repository archived; do default_setup_state=null + # `state` alone is not coverage: a repository can report + # "configured" with an empty `languages` list, which scans nothing + # and produces no analyses (measured 2026-09-07 on life-os, aFIPC + # and inkspan). Collect both fields so the audit can tell those + # apart from a setup that actually covers a language. + default_setup_languages=null if [ "$archived" != "true" ]; then - default_setup_state_json="$RUNNER_TEMP/codeql-default-setup-${repository//[^A-Za-z0-9_.-]/_}.json" - if gh api "repos/${ORG_LOGIN}/${repository}/code-scanning/default-setup" --jq .state \ - >"$default_setup_state_json" 2>/dev/null; then - default_setup_state=$(jq -R '.' "$default_setup_state_json") + default_setup_json="$RUNNER_TEMP/codeql-default-setup-${repository//[^A-Za-z0-9_.-]/_}.json" + if gh api "repos/${ORG_LOGIN}/${repository}/code-scanning/default-setup" \ + >"$default_setup_json" 2>/dev/null; then + default_setup_state=$(jq '.state // null' "$default_setup_json") + default_setup_languages=$(jq '.languages // []' "$default_setup_json") else default_setup_state=null + default_setup_languages=null fi fi @@ -187,12 +207,13 @@ jobs: fi fi - echo "CODEQL_COVERAGE repository=${repository} archived=${archived} default_setup_state=${default_setup_state} latest_codeql_analysis=${latest_codeql_analysis}" + echo "CODEQL_COVERAGE repository=${repository} archived=${archived} default_setup_state=${default_setup_state} default_setup_languages=${default_setup_languages} latest_codeql_analysis=${latest_codeql_analysis}" jq --arg name "$repository" \ --argjson archived "$archived" \ --argjson default_setup_state "$default_setup_state" \ + --argjson default_setup_languages "$default_setup_languages" \ --argjson latest_codeql_analysis "$latest_codeql_analysis" \ - '. + [{name: $name, archived: $archived, default_setup_state: $default_setup_state, latest_codeql_analysis: $latest_codeql_analysis}]' \ + '. + [{name: $name, archived: $archived, default_setup_state: $default_setup_state, default_setup_languages: $default_setup_languages, latest_codeql_analysis: $latest_codeql_analysis}]' \ "$coverage_json" >"${coverage_json}.next" mv "${coverage_json}.next" "$coverage_json" done < <(jq -r '.[] | [.name, (.archived | tostring)] | @tsv' "$repositories_json") diff --git a/scripts/ci/audit_org_codeql_coverage.py b/scripts/ci/audit_org_codeql_coverage.py index f9fb2eaf17..9051c67568 100644 --- a/scripts/ci/audit_org_codeql_coverage.py +++ b/scripts/ci/audit_org_codeql_coverage.py @@ -66,6 +66,27 @@ def _is_analysis_fresh_and_successful( return parsed >= now - timedelta(days=CODEQL_ANALYSIS_FRESHNESS_DAYS) +def _default_setup_scans_a_language(repository: dict[str, Any]) -> bool: + """Return True when default-setup is configured AND has languages enabled. + + ``state == "configured"`` alone is not coverage. Measured 2026-09-07: + ``life-os``, ``aFIPC`` and ``inkspan`` all report ``configured`` with an + **empty** ``languages`` list and no ``schedule``; ``life-os`` has zero CodeQL + analyses of any language as a result, while still satisfying the + configured-state check this function replaces. A default setup with nothing + enabled is a commitment to scan nothing. + + A missing ``default_setup_languages`` key fails closed rather than falling + back to the state alone, which would silently restore that gap. The audit + workflow collects the field in the same change that introduced this check, + so the key is absent only when the payload predates them both. + """ + if repository.get("default_setup_state") != "configured": + return False + languages = repository.get("default_setup_languages") + return isinstance(languages, list) and bool(languages) + + def repositories_without_codeql( repositories: list[dict[str, Any]], now: datetime | None = None ) -> list[dict[str, Any]]: @@ -88,8 +109,10 @@ def repositories_without_codeql( # CodeQL going forward (like a scheduled cron guarantee), not a # one-time historical scan that can go stale -- so it does not need # the same freshness check as latest_codeql_analysis below. Do not - # "fix" this into requiring a completed scan. - has_default_setup = repository.get("default_setup_state") == "configured" + # "fix" this into requiring a completed scan. It does need the + # commitment to cover at least one language: see + # _default_setup_scans_a_language. + has_default_setup = _default_setup_scans_a_language(repository) has_fresh_analysis = _is_analysis_fresh_and_successful( repository.get("latest_codeql_analysis"), current ) @@ -98,13 +121,30 @@ def repositories_without_codeql( return uncovered +def _coverage_gap_reason(repository: dict[str, Any]) -> str: + """Return the gap description that tells the operator what to change. + + "Default setup is on but scans nothing" and "there is no coverage at all" + need different fixes -- enable languages on the existing setup, versus set + coverage up -- so they are reported as different sentences. + """ + if repository.get("default_setup_state") == "configured": + return ( + f"{repository.get('name')} has CodeQL default-setup configured with no " + "languages enabled, so it scans nothing and produces no analyses" + ) + return ( + f"{repository.get('name')} has no CodeQL coverage from any source " + "(no default-setup, no recent analysis)" + ) + + def audit_codeql_coverage( repositories: list[dict[str, Any]], now: datetime | None = None ) -> list[str]: """Return one human-readable error per repository with zero CodeQL coverage.""" return [ - f"{repository.get('name')} has no CodeQL coverage from any source " - "(no default-setup, no recent analysis)" + _coverage_gap_reason(repository) for repository in repositories_without_codeql(repositories, now) ] @@ -137,6 +177,20 @@ def main(argv: list[str] | None = None) -> int: print(f"ERROR: unable to load repository JSON: {exc}", file=sys.stderr) return 2 + if not repositories: + # An empty payload is not a clean organization: it is an audit that + # examined nothing, and "PASS: all 0 repositories have real CodeQL + # coverage" reads as success. The calling workflow already refuses an + # enumeration missing its known-private sentinel repositories, which + # an empty list would also fail -- this closes the same hole for any + # other entry point, because the script is directly runnable against a + # JSON path or stdin. + print( + "ERROR: repository payload is empty, so this run audited nothing", + file=sys.stderr, + ) + return 2 + errors = audit_codeql_coverage(repositories) if errors: for error in errors: diff --git a/tests/test_audit_org_codeql_coverage.py b/tests/test_audit_org_codeql_coverage.py index ccd2cd9c42..20d623aca5 100644 --- a/tests/test_audit_org_codeql_coverage.py +++ b/tests/test_audit_org_codeql_coverage.py @@ -14,6 +14,23 @@ def covered_by_default_setup(name: str) -> dict: "name": name, "archived": False, "default_setup_state": "configured", + "default_setup_languages": ["actions", "python"], + "latest_codeql_analysis": None, + } + + +def default_setup_scanning_nothing(name: str) -> dict: + """Return a repository whose default-setup is on but has no languages enabled. + + The live shape measured on 2026-09-07 for ``life-os``, ``aFIPC`` and + ``inkspan``: ``state`` is ``configured``, ``languages`` is empty and + ``schedule`` is null. ``life-os`` had zero CodeQL analyses of any language. + """ + return { + "name": name, + "archived": False, + "default_setup_state": "configured", + "default_setup_languages": [], "latest_codeql_analysis": None, } @@ -89,6 +106,60 @@ def test_default_setup_alone_counts_as_coverage() -> None: assert audit.audit_codeql_coverage(repositories, now=NOW) == [] +def test_default_setup_with_no_languages_enabled_is_not_coverage() -> None: + """A setup that scans nothing must not satisfy the configured-state check. + + Measured 2026-09-07: ``life-os`` reports ``configured`` with an empty + ``languages`` list and has zero CodeQL analyses of any language, while + ``codeql-pr.yml`` still runs on every pull request head. Before this check + the audit passed it on the state alone. + """ + repositories = [default_setup_scanning_nothing("life-os")] + + assert audit.audit_codeql_coverage(repositories, now=NOW) == [ + "life-os has CodeQL default-setup configured with no languages enabled, " + "so it scans nothing and produces no analyses" + ] + + +def test_default_setup_scanning_nothing_still_passes_on_a_fresh_analysis() -> None: + """The empty-language setup is only a gap when nothing else covers the repo. + + ``aFIPC`` and ``inkspan`` both report the empty-language shape yet receive + analyses from a repository-local ``codeql.yml``, so flagging them would be a + false alarm. + """ + repository = default_setup_scanning_nothing("aFIPC") + repository["latest_codeql_analysis"] = covered_by_recent_analysis("aFIPC")[ + "latest_codeql_analysis" + ] + + assert audit.audit_codeql_coverage([repository], now=NOW) == [] + + +def test_payload_without_the_languages_key_fails_closed() -> None: + """A payload predating the workflow change must not pass on state alone. + + Falling back to ``default_setup_state`` when the key is missing would + silently restore the gap this check exists to close. + """ + repository = covered_by_default_setup("PolicyWeave") + del repository["default_setup_languages"] + + assert audit.audit_codeql_coverage([repository], now=NOW) == [ + "PolicyWeave has CodeQL default-setup configured with no languages " + "enabled, so it scans nothing and produces no analyses" + ] + + +def test_non_list_languages_value_fails_closed() -> None: + """A malformed ``languages`` value is not evidence that anything is scanned.""" + repository = covered_by_default_setup("PolicyWeave") + repository["default_setup_languages"] = "python" + + assert len(audit.audit_codeql_coverage([repository], now=NOW)) == 1 + + def test_recent_analysis_alone_counts_as_coverage() -> None: repositories = [covered_by_recent_analysis("TEPP")] @@ -269,3 +340,22 @@ def test_parse_args_defaults_to_none() -> None: args = audit.parse_args([]) assert args.repositories_json is None + + +def test_main_refuses_an_empty_payload_instead_of_passing_vacuously( + monkeypatch, capsys +) -> None: + """An audit that examined nothing must not print PASS. + + ``audit_codeql_coverage([])`` returning no gaps is correct -- there are no + repositories to have gaps. What is wrong is ``main`` turning that into + "PASS: all 0 repositories have real CodeQL coverage" and exiting 0, which + is the same vacuous-pass shape as a default setup that is configured with + no languages enabled. + """ + monkeypatch.setattr("sys.stdin", StringIO("[]")) + + assert audit.main([]) == 2 + captured = capsys.readouterr() + assert "audited nothing" in captured.err + assert "PASS" not in captured.out diff --git a/tests/test_central_required_workflow_ruleset_audit.py b/tests/test_central_required_workflow_ruleset_audit.py index 77bbf53305..cec0d2aead 100644 --- a/tests/test_central_required_workflow_ruleset_audit.py +++ b/tests/test_central_required_workflow_ruleset_audit.py @@ -517,13 +517,21 @@ def test_audit_organization_codeql_coverage_step_has_freshness_and_credential_gu " exit 1\n" " fi" ) in workflow + # This pinned `--jq .state` until 2026-09-07. What it protects is that the + # audit reads default-setup per repository, not that it reads only the + # state: `state == "configured"` with an empty `languages` list scans + # nothing and produces no analyses (live on life-os, aFIPC and inkspan), + # so the step now fetches the whole object and extracts both fields. + assert 'repos/${ORG_LOGIN}/${repository}/code-scanning/default-setup"' in workflow + assert """default_setup_state=$(jq '.state // null' "$default_setup_json")""" in workflow assert ( - 'repos/${ORG_LOGIN}/${repository}/code-scanning/default-setup" --jq .state' + """default_setup_languages=$(jq '.languages // []' "$default_setup_json")""" in workflow ) + assert "default_setup_languages: $default_setup_languages" in workflow assert ( 'if [ "$archived" != "true" ]; then\n' - ' default_setup_state_json="$RUNNER_TEMP/codeql-default-setup-' + ' default_setup_json="$RUNNER_TEMP/codeql-default-setup-' '${repository//[^A-Za-z0-9_.-]/_}.json"' ) in workflow assert ( @@ -540,6 +548,33 @@ def test_audit_organization_codeql_coverage_step_has_freshness_and_credential_gu assert "python3 scripts/ci/audit_org_codeql_coverage.py" in workflow +def test_codeql_coverage_audit_survives_a_ruleset_drift_failure() -> None: + """An owner-configured ruleset drift must not disable the coverage detector. + + Both audits live in one job, and the ruleset step exits 1 on governance + drift. It did on 2026-09-06 ("exactly two approving reviews are not + required", "last-push approval protection is disabled"), so every run since + 2026-09-04 failed before reaching the CodeQL coverage step. The subjects are + unrelated and the coverage step has no data dependency on the one above it, + so it is guarded by ``if: always()``. + + The bootstrap steps below it are deliberately *not* given the same guard: + they open pull requests, and running a mutation after an unexplained + upstream failure is a different decision from running a read-only detector. + """ + workflow = (REPO_ROOT / ".github/workflows/audit-central-ruleset.yml").read_text( + encoding="utf-8" + ) + coverage_step = workflow.split("- name: Audit organization CodeQL coverage\n", 1)[1] + before_next_step = coverage_step.split(" - name: ", 1)[0] + + assert "\n if: always()\n" in before_next_step + bootstrap_step = workflow.split( + "- name: Create missing CodeQL setup pull requests\n", 1 + )[1].split(" - name: ", 1)[0] + assert "if: always()" not in bootstrap_step + + def test_codeql_gap_bootstrap_uses_trusted_opencode_identity_without_pr_head_execution() -> None: """Backlog item 38 stays on trusted main and treats installation tokens as opaque.""" workflow = (REPO_ROOT / ".github/workflows/audit-central-ruleset.yml").read_text( From c9052e607e5f3cc76e73207e7786b21500721b79 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 04:20:08 +0900 Subject: [PATCH 38/80] fix(audit): count the repositories examined, not the ones supplied (#1989) #1987 taught the CodeQL coverage audit to refuse an empty payload, because "PASS: all 0 repositories have real CodeQL coverage" reads as success over a run that examined nothing. Reviewing that change, host 2 fed it a payload of a single *archived* repository: PASS: all 1 repositories have real CodeQL coverage exit 0 Non-empty, so it clears the new guard, and archived repositories are then legitimately skipped by the coverage loop. The subject set is empty again, by a different route -- the same defect the guard was added to close, one input shape sideways from the one it checks. The count that matters is what the audit examined. `auditable_repositories()` is now the single place that decides which repositories are in scope, shared by the loop and by the guard, so the two cannot drift apart when the archived rule changes. The guard refuses an empty examined set however it became empty, and the PASS line reports the examined count rather than the supplied one, so an organization of nothing but archived repositories can no longer be reported as fully covered. Developer experience: the pass line's number is the number of repositories the run actually checked. User experience: an audit that examined nothing says so instead of passing. Co-authored-by: Claude Opus 5 --- scripts/ci/audit_org_codeql_coverage.py | 44 +++++++++++++++++-------- tests/test_audit_org_codeql_coverage.py | 38 +++++++++++++++++++++ 2 files changed, 69 insertions(+), 13 deletions(-) diff --git a/scripts/ci/audit_org_codeql_coverage.py b/scripts/ci/audit_org_codeql_coverage.py index 9051c67568..bdbc835491 100644 --- a/scripts/ci/audit_org_codeql_coverage.py +++ b/scripts/ci/audit_org_codeql_coverage.py @@ -87,6 +87,22 @@ def _default_setup_scans_a_language(repository: dict[str, Any]) -> bool: return isinstance(languages, list) and bool(languages) +def auditable_repositories( + repositories: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Return the repositories this audit actually examines. + + Archived repositories are excluded: they cannot run workflows or code + scanning, so a lack of coverage there is not a product gap. Counting them + as examined is what let ``main`` report success over an empty subject set. + """ + return [ + repository + for repository in repositories + if not repository.get("archived") + ] + + def repositories_without_codeql( repositories: list[dict[str, Any]], now: datetime | None = None ) -> list[dict[str, Any]]: @@ -102,9 +118,7 @@ def repositories_without_codeql( """ current = now or datetime.now(timezone.utc) uncovered: list[dict[str, Any]] = [] - for repository in repositories: - if repository.get("archived"): - continue + for repository in auditable_repositories(repositories): # "configured" is GitHub's own forward-looking commitment to run # CodeQL going forward (like a scheduled cron guarantee), not a # one-time historical scan that can go stale -- so it does not need @@ -177,16 +191,20 @@ def main(argv: list[str] | None = None) -> int: print(f"ERROR: unable to load repository JSON: {exc}", file=sys.stderr) return 2 - if not repositories: - # An empty payload is not a clean organization: it is an audit that - # examined nothing, and "PASS: all 0 repositories have real CodeQL - # coverage" reads as success. The calling workflow already refuses an - # enumeration missing its known-private sentinel repositories, which - # an empty list would also fail -- this closes the same hole for any - # other entry point, because the script is directly runnable against a - # JSON path or stdin. + audited = auditable_repositories(repositories) + if not audited: + # An audit that examined nothing is not a clean organization, and + # "PASS: all 0 repositories have real CodeQL coverage" reads as + # success. The count that matters is what was examined, not what was + # supplied: an empty payload and a payload of nothing but archived + # repositories both reach zero subjects, and only the first was caught + # when this guard counted `repositories`. The calling workflow refuses + # an enumeration missing its known-private sentinel repositories, but + # the script is directly runnable against a JSON path or stdin, so the + # guard has to live here too. print( - "ERROR: repository payload is empty, so this run audited nothing", + f"ERROR: this run audited nothing " + f"(0 of {len(repositories)} repositories were eligible)", file=sys.stderr, ) return 2 @@ -201,7 +219,7 @@ def main(argv: list[str] | None = None) -> int: ) return 1 - print(f"PASS: all {len(repositories)} repositories have real CodeQL coverage") + print(f"PASS: all {len(audited)} repositories have real CodeQL coverage") return 0 diff --git a/tests/test_audit_org_codeql_coverage.py b/tests/test_audit_org_codeql_coverage.py index 20d623aca5..0ffc8fa749 100644 --- a/tests/test_audit_org_codeql_coverage.py +++ b/tests/test_audit_org_codeql_coverage.py @@ -359,3 +359,41 @@ def test_main_refuses_an_empty_payload_instead_of_passing_vacuously( captured = capsys.readouterr() assert "audited nothing" in captured.err assert "PASS" not in captured.out + + +def test_main_refuses_a_payload_of_only_archived_repositories(monkeypatch, capsys) -> None: + """Counting what was supplied, not what was examined, left the hole open. + + An archived-only payload is non-empty, so it passed the first version of + this guard, and archived repositories are then legitimately skipped -- the + run reported "PASS: all 1 repositories have real CodeQL coverage" having + examined none of them. Found in review, one layer out from the empty-payload + case it replaces. + """ + monkeypatch.setattr( + "sys.stdin", StringIO(json.dumps([uncovered("trivy-sarif-repro", archived=True)])) + ) + + assert audit.main([]) == 2 + captured = capsys.readouterr() + assert "0 of 1 repositories were eligible" in captured.err + assert "PASS" not in captured.out + + +def test_pass_line_counts_examined_repositories_not_supplied_ones( + tmp_path, capsys +) -> None: + """The PASS line must not credit archived repositories it never examined.""" + payload = tmp_path / "repositories.json" + payload.write_text( + json.dumps( + [ + covered_by_default_setup("PolicyWeave"), + uncovered("trivy-sarif-repro", archived=True), + ] + ), + encoding="utf-8", + ) + + assert audit.main([str(payload)]) == 0 + assert "PASS: all 1 repositories" in capsys.readouterr().out From dd7ea8133e90653e3baf748315ad08367239388b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 16:36:19 +0900 Subject: [PATCH 39/80] fix(codeql): dispatch one current-head scan per pull request (#2008) Per-language codeql-scan-dispatch runs were occupying the 60-job ceiling: 149 queued handler runs across 60 PR@SHA tuples, and duplicate cancel could not collapse them because the language is absent from the run name, job name, and REST payload. analyze-head shards still publish the required check name and still fail pending to release the runner, but they no longer POST. dispatch-current-head collects this run's language job ids and sends one codeql-scan payload (matrix + required_jobs). The handler concurrency group is {workflow}-{repository}-{PR}; language independence is fail-fast: false on that run's job matrix. ADR-0025's 2026-09-05 rejection of a full-matrix dispatch is superseded. The sibling-cancel failure from contextual-orchestrator#1049 / run 33938784437 is gone because siblings are jobs in one run. --- .github/workflows/codeql-pr.yml | 219 +++++++--- .github/workflows/codeql-scan-dispatch.yml | 60 ++- ...required-workflow-dispatch-architecture.md | 97 +++-- tests/test_codeql_pr_workflow_contract.py | 394 +++++++++++++++++- ..._codeql_scan_dispatch_workflow_contract.py | 116 +++++- tests/test_docs_only_pr_runner_admission.py | 14 +- ...d_codeql_dispatch_runner_image_contract.py | 4 +- 7 files changed, 754 insertions(+), 150 deletions(-) diff --git a/.github/workflows/codeql-pr.yml b/.github/workflows/codeql-pr.yml index cb07ad2fab..db06b7c4ba 100644 --- a/.github/workflows/codeql-pr.yml +++ b/.github/workflows/codeql-pr.yml @@ -2,12 +2,12 @@ # refuses to admit it, 0/43+ across every sampled repository # (docs/doctoring/codeql-pr-required-workflow-always-fails.md). This file # stays required-workflow-safe by never calling codeql-action itself: it -# detects languages, dispatches the actual scan via repository_dispatch to -# codeql-scan-dispatch.yml (which runs natively, unrestricted, in -# ContextualWisdomLab/.github). The shard then fails intentionally to release -# its runner; the handler publishes codeql-dispatch/ and reruns only -# that exact failed job. On rerun the shard reads the terminal status once. -# Design: +# detects languages, fails each analyze-head shard pending to release its +# runner, then one coordinator POSTs repository_dispatch to +# codeql-scan-dispatch.yml (native, unrestricted, in +# ContextualWisdomLab/.github) with the remaining language matrix. The +# handler publishes codeql-dispatch/ and reruns only that exact +# failed job. On rerun the shard reads the terminal status once. Design: # docs/adr/0025-codeql-required-workflow-dispatch-architecture.md. The # merge-preview scan (analyze-merge) is required nowhere (PR #1766) and was # dropped, not migrated. @@ -156,28 +156,21 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.detect-languages.outputs.matrix) }} steps: - - name: Request current-head CodeQL scan dispatch - # Each shard dispatches only its own language and passes its exact - # run/job identity. The shard intentionally fails after dispatch so - # its runner is released; the trusted handler later reruns that one - # failed job after publishing a terminal current-head verdict. + - name: Read current-head CodeQL dispatch verdict + # Shards never dispatch. They re-check the live head, consume an + # authenticated codeql-dispatch/ verdict when one exists, + # and otherwise fail pending so the runner is released. One + # coordinator job POSTs the remaining language matrix after every + # shard has a job id. id: dispatch if: needs.detect-languages.outputs.code == 'true' env: GH_TOKEN: ${{ github.token }} - OIDC_AUDIENCE: opencode-github-action - OPENCODE_API_BASE_URL: https://api.opencode.ai TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.repository }} PR_NUMBER: ${{ github.event.pull_request.number }} - PR_BASE_REF: ${{ github.event.pull_request.base.ref }} - PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - PR_HEAD_REF: ${{ github.event.pull_request.head.ref }} PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} LANGUAGE: ${{ matrix.language }} - BUILD_MODE: ${{ matrix.build-mode }} RUN_ATTEMPT: ${{ github.run_attempt }} - REQUIRED_RUN_ID: ${{ github.run_id }} - REQUIRED_JOB_ID: ${{ job.check_run_id }} run: | set -euo pipefail live_pr="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}")" @@ -219,43 +212,6 @@ jobs: echo "::error::Exact CodeQL job was rerun without an authenticated terminal verdict." exit 1 fi - if ! [[ "$REQUIRED_RUN_ID" =~ ^[1-9][0-9]*$ ]] || - ! [[ "$REQUIRED_JOB_ID" =~ ^[1-9][0-9]*$ ]]; then - echo "::error::CodeQL dispatch requires canonical current run and job ids." - exit 1 - fi - - if [ -z "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:-}" ] || [ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]; then - echo "::error::CodeQL scan dispatch requires GitHub OIDC." - exit 1 - fi - separator='&' - [[ "$ACTIONS_ID_TOKEN_REQUEST_URL" == *\?* ]] || separator='?' - oidc_token="$(curl -fsS -H "Authorization: Bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" "${ACTIONS_ID_TOKEN_REQUEST_URL}${separator}audience=${OIDC_AUDIENCE}" | jq -r '.value // empty')" - if [ -z "$oidc_token" ]; then - echo "::error::CodeQL scan dispatch could not obtain its OIDC token." - exit 1 - fi - app_token="$(curl -fsS -X POST -H "Authorization: Bearer ${oidc_token}" "${OPENCODE_API_BASE_URL}/exchange_github_app_token" | jq -r '.token // empty')" - if [ -z "$app_token" ]; then - echo "::error::CodeQL scan dispatch could not obtain its repository-scoped app token." - exit 1 - fi - echo "::add-mask::$app_token" - jq -cn \ - --arg target_repository "$TARGET_REPOSITORY" \ - --arg pr_number "$PR_NUMBER" \ - --arg pr_base_ref "$PR_BASE_REF" \ - --arg pr_base_sha "$PR_BASE_SHA" \ - --arg pr_head_ref "$PR_HEAD_REF" \ - --arg pr_head_sha "$PR_HEAD_SHA" \ - --arg language "$LANGUAGE" \ - --arg build_mode "$BUILD_MODE" \ - --arg required_run_id "$REQUIRED_RUN_ID" \ - --arg required_job_id "$REQUIRED_JOB_ID" \ - --arg required_language "$LANGUAGE" \ - '{event_type:"codeql-scan",client_payload:{target_repository:$target_repository,pr_number:$pr_number,pr_base_ref:$pr_base_ref,pr_base_sha:$pr_base_sha,pr_head_ref:$pr_head_ref,pr_head_sha:$pr_head_sha,matrix:[{language:$language,"build-mode":$build_mode}],required_run_id:$required_run_id,required_job_id:$required_job_id,required_language:$required_language}}' | - GH_TOKEN="$app_token" gh api -X POST repos/ContextualWisdomLab/.github/dispatches --input - echo "verdict=pending" >>"$GITHUB_OUTPUT" - name: Release runner or enforce current-head CodeQL verdict @@ -287,3 +243,154 @@ jobs: exit 1 ;; esac + + dispatch-current-head: + name: Dispatch current-head CodeQL scan + needs: [detect-languages, analyze-head] + if: >- + always() + && github.event.action != 'closed' + && github.event.pull_request.state != 'closed' + && github.run_attempt == 1 + && needs.detect-languages.result == 'success' + && needs.detect-languages.outputs.code == 'true' + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + actions: read + steps: + - name: Dispatch current-head CodeQL scan + env: + GH_TOKEN: ${{ github.token }} + OIDC_AUDIENCE: opencode-github-action + OPENCODE_API_BASE_URL: https://api.opencode.ai + TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_BASE_REF: ${{ github.event.pull_request.base.ref }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_HEAD_REF: ${{ github.event.pull_request.head.ref }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + REQUIRED_RUN_ID: ${{ github.run_id }} + MATRIX: ${{ needs.detect-languages.outputs.matrix }} + run: | + set -euo pipefail + live_pr="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}")" + live_head="$(printf '%s' "$live_pr" | jq -r '.head.sha // empty')" + live_state="$(printf '%s' "$live_pr" | jq -r 'if (.state | type) == "string" then .state else empty end')" + if [ -z "$live_head" ] || [ -z "$live_state" ]; then + echo "::error::Could not validate live pull request state before CodeQL dispatch." + exit 1 + fi + if [ "$live_state" = "closed" ]; then + echo "PR is closed on the live exact head; a current-head CodeQL scan is not requested." + exit 0 + fi + if [ "${live_head,,}" != "${PR_HEAD_SHA,,}" ]; then + echo "Pull request head moved on the live open PR; a fresh dispatch will fire for the current head." + exit 0 + fi + if ! [[ "$REQUIRED_RUN_ID" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::CodeQL dispatch requires a canonical current run id." + exit 1 + fi + + include_json="$(printf '%s' "$MATRIX" | jq -c '.include // empty' 2>/dev/null || true)" + if [ -z "$include_json" ] || + [ "$(printf '%s' "$include_json" | jq 'type == "array" and length >= 1')" != "true" ]; then + echo "::error::CodeQL coordinator received an empty or malformed language matrix." + exit 1 + fi + + jobs_json="$( + gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${REQUIRED_RUN_ID}/jobs" --jq '.jobs[]' | + jq -s '{jobs:.}' + )" + required_jobs='[]' + while IFS= read -r entry; do + language="$(printf '%s' "$entry" | jq -r '.language // empty')" + expected_name="CodeQL compatibility analysis (${language})" + job_id="$(printf '%s' "$jobs_json" | jq -r --arg name "$expected_name" ' + [.jobs[]? | select(.name == $name) | .id] + | if length == 1 then .[0] | tostring else empty end + ')" + if ! [[ "$job_id" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::CodeQL coordinator missing current-head job id for ${language}." + exit 1 + fi + required_jobs="$( + jq -c --arg language "$language" --argjson job_id "$job_id" \ + '. + [{language:$language,job_id:$job_id}]' <<<"$required_jobs" + )" + done < <(printf '%s' "$include_json" | jq -c '.[]') + + statuses="$(gh api "repos/${TARGET_REPOSITORY}/commits/${PR_HEAD_SHA}/statuses")" + pending_matrix='[]' + while IFS= read -r entry; do + language="$(printf '%s' "$entry" | jq -r '.language // empty')" + verdict_state="$(printf '%s' "$statuses" | jq -r --arg ctx "codeql-dispatch/${language}" ' + [ + .[] + | select(.context == $ctx) + | select( + (.creator.login // "" | ascii_downcase) as $creator + | $creator == "opencode-agent" or $creator == "opencode-agent[bot]" + ) + ] + | first // {} | .state // empty + ')" + case "$verdict_state" in + success|failure|error) + echo "Found authenticated current-head CodeQL verdict for ${language}: ${verdict_state}." + ;; + *) + pending_matrix="$(jq -c --argjson entry "$entry" '. + [$entry]' <<<"$pending_matrix")" + ;; + esac + done < <(printf '%s' "$include_json" | jq -c '.[]') + + if [ "$(printf '%s' "$pending_matrix" | jq 'length')" -eq 0 ]; then + echo "All detected CodeQL languages already have authenticated terminal verdicts; skipping dispatch." + exit 0 + fi + + required_jobs="$( + jq -nc --argjson pending "$pending_matrix" --argjson jobs "$required_jobs" ' + ($pending | map(.language)) as $langs + | [$jobs[] | select(.language as $l | $langs | index($l) != null)] + ' + )" + if [ "$(printf '%s' "$required_jobs" | jq 'length')" != "$(printf '%s' "$pending_matrix" | jq 'length')" ]; then + echo "::error::CodeQL coordinator could not bind a job id to every pending language." + exit 1 + fi + + if [ -z "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:-}" ] || [ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]; then + echo "::error::CodeQL scan dispatch requires GitHub OIDC." + exit 1 + fi + separator='&' + [[ "$ACTIONS_ID_TOKEN_REQUEST_URL" == *\?* ]] || separator='?' + oidc_token="$(curl -fsS -H "Authorization: Bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" "${ACTIONS_ID_TOKEN_REQUEST_URL}${separator}audience=${OIDC_AUDIENCE}" | jq -r '.value // empty')" + if [ -z "$oidc_token" ]; then + echo "::error::CodeQL scan dispatch could not obtain its OIDC token." + exit 1 + fi + app_token="$(curl -fsS -X POST -H "Authorization: Bearer ${oidc_token}" "${OPENCODE_API_BASE_URL}/exchange_github_app_token" | jq -r '.token // empty')" + if [ -z "$app_token" ]; then + echo "::error::CodeQL scan dispatch could not obtain its repository-scoped app token." + exit 1 + fi + echo "::add-mask::$app_token" + jq -cn \ + --arg target_repository "$TARGET_REPOSITORY" \ + --arg pr_number "$PR_NUMBER" \ + --arg pr_base_ref "$PR_BASE_REF" \ + --arg pr_base_sha "$PR_BASE_SHA" \ + --arg pr_head_ref "$PR_HEAD_REF" \ + --arg pr_head_sha "$PR_HEAD_SHA" \ + --argjson matrix "$pending_matrix" \ + --arg required_run_id "$REQUIRED_RUN_ID" \ + --argjson required_jobs "$required_jobs" \ + '{event_type:"codeql-scan",client_payload:{target_repository:$target_repository,pr_number:$pr_number,pr_base_ref:$pr_base_ref,pr_base_sha:$pr_base_sha,pr_head_ref:$pr_head_ref,pr_head_sha:$pr_head_sha,matrix:$matrix,required_run_id:$required_run_id,required_jobs:$required_jobs}}' | + GH_TOKEN="$app_token" gh api -X POST repos/ContextualWisdomLab/.github/dispatches --input - diff --git a/.github/workflows/codeql-scan-dispatch.yml b/.github/workflows/codeql-scan-dispatch.yml index 1c9dda3e45..23c47f6abc 100644 --- a/.github/workflows/codeql-scan-dispatch.yml +++ b/.github/workflows/codeql-scan-dispatch.yml @@ -26,8 +26,7 @@ concurrency: group: >- codeql-scan-dispatch-${{ github.event.client_payload.target_repository || github.repository }}-${{ - github.event.client_payload.pr_number || github.run_id }}-${{ - github.event.client_payload.required_language || 'unknown-language' }} + github.event.client_payload.pr_number || github.run_id }} cancel-in-progress: true permissions: @@ -50,8 +49,7 @@ jobs: head_sha: ${{ steps.validate.outputs.head_sha }} matrix: ${{ steps.validate.outputs.matrix }} required_run_id: ${{ steps.validate.outputs.required_run_id }} - required_job_id: ${{ steps.validate.outputs.required_job_id }} - required_language: ${{ steps.validate.outputs.required_language }} + required_jobs: ${{ steps.validate.outputs.required_jobs }} steps: - name: Exchange OpenCode app token for target repository metadata reads id: metadata_read_app_token @@ -148,8 +146,7 @@ jobs: SUPPLIED_HEAD_SHA: ${{ github.event.client_payload.pr_head_sha || '' }} SUPPLIED_MATRIX: ${{ toJSON(github.event.client_payload.matrix) }} SUPPLIED_REQUIRED_RUN_ID: ${{ github.event.client_payload.required_run_id || '' }} - SUPPLIED_REQUIRED_JOB_ID: ${{ github.event.client_payload.required_job_id || '' }} - SUPPLIED_REQUIRED_LANGUAGE: ${{ github.event.client_payload.required_language || '' }} + SUPPLIED_REQUIRED_JOBS: ${{ toJSON(github.event.client_payload.required_jobs) }} run: | set -euo pipefail # ALLOWED_DISPATCH_ACTOR is a comma-separated allowlist shared with @@ -180,19 +177,37 @@ jobs: fi matrix_json="$(printf '%s' "$SUPPLIED_MATRIX" | jq -c '.' 2>/dev/null || true)" + jobs_json="$(printf '%s' "$SUPPLIED_REQUIRED_JOBS" | jq -c '.' 2>/dev/null || true)" if [ -z "$matrix_json" ] || - [ "$(printf '%s' "$matrix_json" | jq 'type == "array" and length == 1')" != "true" ] || - [ "$(printf '%s' "$matrix_json" | jq '[.[] | select((.language | type == "string") and (.language | test("^[a-z0-9-]+$")) and (."build-mode" | type == "string"))] | length == ($ARGS.positional[0] | tonumber)' --args "$(printf '%s' "$matrix_json" | jq 'length')")" != "true" ]; then - printf '::error::CodeQL scan dispatch matrix must contain exactly one valid language/build-mode shard. matrix=%s\n' "${SUPPLIED_MATRIX:-}" + [ "$(printf '%s' "$matrix_json" | jq 'type == "array" and length >= 1')" != "true" ] || + [ "$(printf '%s' "$matrix_json" | jq '[.[] | select((.language | type == "string") and (.language | test("^[a-z0-9-]+$")) and (."build-mode" | type == "string"))] | length == ($ARGS.positional[0] | tonumber)' --args "$(printf '%s' "$matrix_json" | jq 'length')")" != "true" ] || + [ "$(printf '%s' "$matrix_json" | jq '(map(.language) | unique | length) == (map(.language) | length)')" != "true" ]; then + printf '::error::CodeQL scan dispatch matrix must contain at least one valid language/build-mode shard with unique languages. matrix=%s\n' "${SUPPLIED_MATRIX:-}" exit 1 fi - matrix_language="$(printf '%s' "$matrix_json" | jq -r '.[0].language // empty')" - if ! [[ "$SUPPLIED_REQUIRED_RUN_ID" =~ ^[1-9][0-9]*$ ]] || - ! [[ "$SUPPLIED_REQUIRED_JOB_ID" =~ ^[1-9][0-9]*$ ]] || - [ "$SUPPLIED_REQUIRED_LANGUAGE" != "$matrix_language" ]; then - printf '::error::CodeQL wake identity is missing, non-canonical, or does not match the dispatched language.\n' + if [ -z "$jobs_json" ] || + [ "$(jq -n --argjson matrix "$matrix_json" --argjson jobs "$jobs_json" ' + ($jobs | type == "array") + and (($jobs | length) == ($matrix | length)) + and ($jobs | all( + (.language | type == "string") + and (.language | test("^[a-z0-9-]+$")) + and ( + ((.job_id | type == "number") and (.job_id == (.job_id | floor)) and (.job_id >= 1)) + or ((.job_id | type == "string") and (.job_id | test("^[1-9][0-9]*$"))) + ) + )) + and (($jobs | map(.language) | sort) == ($matrix | map(.language) | sort)) + and (($jobs | map(.language) | unique | length) == ($jobs | length)) + ')" != "true" ]; then + printf '::error::CodeQL wake identity is missing, non-canonical, or does not match the dispatched languages one-to-one.\n' exit 1 fi + if ! [[ "$SUPPLIED_REQUIRED_RUN_ID" =~ ^[1-9][0-9]*$ ]]; then + printf '::error::CodeQL wake identity is missing, non-canonical, or does not match the dispatched languages one-to-one.\n' + exit 1 + fi + jobs_json="$(printf '%s' "$jobs_json" | jq -c 'map({language, job_id: (.job_id | tonumber)})')" pull_request_json="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}")" live_base_repository="$(jq -r '.base.repo.full_name // empty' <<<"$pull_request_json")" @@ -235,8 +250,9 @@ jobs: printf '%s\n' "$matrix_json" echo "EOF" printf 'required_run_id=%s\n' "$SUPPLIED_REQUIRED_RUN_ID" - printf 'required_job_id=%s\n' "$SUPPLIED_REQUIRED_JOB_ID" - printf 'required_language=%s\n' "$SUPPLIED_REQUIRED_LANGUAGE" + echo "required_jobs<>"$GITHUB_OUTPUT" printf 'Validated current live metadata for %s#%s: base=%s/%s head=%s/%s.\n' "$TARGET_REPOSITORY" "$PR_NUMBER" "$live_base_ref" "$live_base_sha" "$live_head_ref" "$live_head_sha" @@ -484,16 +500,16 @@ jobs: && needs.validate-dispatch.outputs.target_repository != '' && needs.validate-dispatch.outputs.pr_number != '' && needs.validate-dispatch.outputs.head_sha != '' - && github.event.client_payload.required_run_id != '' - && github.event.client_payload.required_job_id != '' + && needs.validate-dispatch.outputs.required_run_id != '' + && needs.validate-dispatch.outputs.required_jobs != '' env: GH_TOKEN: ${{ needs.validate-dispatch.outputs.target_repository == github.repository && github.token || secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN }} TARGET_REPOSITORY: ${{ needs.validate-dispatch.outputs.target_repository }} PR_NUMBER: ${{ needs.validate-dispatch.outputs.pr_number }} HEAD_SHA: ${{ needs.validate-dispatch.outputs.head_sha }} REQUIRED_RUN_ID: ${{ needs.validate-dispatch.outputs.required_run_id }} - REQUIRED_JOB_ID: ${{ needs.validate-dispatch.outputs.required_job_id }} - REQUIRED_LANGUAGE: ${{ needs.validate-dispatch.outputs.required_language }} + REQUIRED_JOBS: ${{ needs.validate-dispatch.outputs.required_jobs }} + REQUIRED_LANGUAGE: ${{ matrix.language }} WAKE_TOKEN_SOURCE: ${{ needs.validate-dispatch.outputs.target_repository == github.repository && 'github-token' || secrets.PR_REVIEW_MERGE_TOKEN != '' && 'PR_REVIEW_MERGE_TOKEN' || secrets.OPENCODE_APPROVE_TOKEN != '' && 'OPENCODE_APPROVE_TOKEN' || 'unavailable' }} run: | set -euo pipefail @@ -501,6 +517,10 @@ jobs: echo "::error::Actions-capable CodeQL wake credential is unavailable." exit 1 fi + REQUIRED_JOB_ID="$(printf '%s' "$REQUIRED_JOBS" | jq -r --arg lang "$REQUIRED_LANGUAGE" ' + [.[] | select(.language == $lang) | .job_id | tostring] + | if length == 1 and (.[0] | test("^[1-9][0-9]*$")) then .[0] else empty end + ')" if ! [[ "$REQUIRED_RUN_ID" =~ ^[1-9][0-9]*$ ]] || ! [[ "$REQUIRED_JOB_ID" =~ ^[1-9][0-9]*$ ]] || ! [[ "$REQUIRED_LANGUAGE" =~ ^[a-z0-9-]+$ ]]; then diff --git a/docs/adr/0025-codeql-required-workflow-dispatch-architecture.md b/docs/adr/0025-codeql-required-workflow-dispatch-architecture.md index 065a9d4d0f..5a11894767 100644 --- a/docs/adr/0025-codeql-required-workflow-dispatch-architecture.md +++ b/docs/adr/0025-codeql-required-workflow-dispatch-architecture.md @@ -1,6 +1,6 @@ # 0025 — Restore central CodeQL as a required workflow via repository_dispatch -**Status:** Proposed · **Date:** 2026-09-03 · **Owner intent recorded:** loop-brief item 41 +**Status:** Proposed, amended 2026-09-07 (one dispatch per pull request; language independence is the handler job matrix) · **Date:** 2026-09-03 · **Owner intent recorded:** loop-brief item 41 ## Problem @@ -96,14 +96,29 @@ codeql-pr.yml (required workflow, runs in target repo context) does) before dispatching. analyze-head (matrix) -- SAME REQUIRED-CHECK NAME: "CodeQL compatibility analysis (${{ matrix.language }})". - No codeql-action reference. On attempt one it - dispatches its exact run id, job id, language, - and head, then fails intentionally to release - the runner. The trusted handler publishes the - terminal status and reruns only that failed - job. On attempt two the shard reads the - authenticated current-head status once and - reflects it as this job's own exit code. + No codeql-action reference and no + repository_dispatch. On attempt one it + re-checks the live head, consumes an + authenticated codeql-dispatch/ + status when one exists, and otherwise fails + pending to release the runner. The trusted + handler publishes the terminal status and + reruns only that failed job. On the woken + attempt the shard reads the authenticated + current-head status once and reflects it as + this job's own exit code. + dispatch-current-head -- NEW: needs analyze-head, runs on attempt one + of an open current-head PR after the shards + have job ids. Collects those ids from this + run's jobs API, POSTs event_type codeql-scan + once with the remaining language matrix and + required_jobs: [{language, job_id}, ...], and + fails closed if any shard job id is missing. + Skips the POST when every language already + has a terminal verdict. github.run_attempt == 1 + is required: a single-job wake re-runs + dependents, and a second POST would cancel + the in-flight multi-language handler. .github/workflows/codeql-scan-dispatch.yml (NEW, runs natively in .github, NOT admitted through the ruleset, so codeql-action is unrestricted here) @@ -151,24 +166,49 @@ NOT admitted through the ruleset, so codeql-action is unrestricted here) closed and leaves the required job failed. ``` -### Concurrency identity is per pull request and language shard +### Concurrency identity is per pull request; language independence is the job matrix -Each required `analyze-head` matrix job dispatches one language and supplies a -matching `required_language`. The native handler therefore serializes only the -same repository, pull request, and language tuple. A newer dispatch for that -tuple cancels its stale predecessor, while Python, JavaScript/TypeScript, and -Actions scans for the same head remain independent. +The required `analyze-head` matrix still publishes one named check per +language. It no longer POSTs. One `dispatch-current-head` job sends every +still-pending language in a single `codeql-scan` payload (`matrix` plus +`required_jobs`). The native handler's concurrency group is +`codeql-scan-dispatch-${target_repository}-${pr_number}` with +`cancel-in-progress: true`, so a newer HEAD of the same pull request cancels +its predecessor and other repositories or pull requests stay independent. -This distinction is required by the exact-job wake contract. On 2026-09-05, -contextual-orchestrator PR #1049 dispatched all three current-head language -jobs, but central run `33938784437` was the sole survivor because the handler's -group omitted `required_language`. The sibling runs cancelled one another, -leaving their required jobs failed in the documented `pending` handoff state. -The chosen key adds the already validated language to the existing workflow, -repository, and pull-request identity. Sending the full language matrix in one -dispatch was rejected because the handler validates one shard and wakes one -exact required job per run; changing that contract would enlarge the security -and recovery surface without solving another observed need. +Language independence is `strategy.fail-fast: false` on that one run's job +matrix. Each scan job still publishes `codeql-dispatch/` and wakes +only its own required job. One language's failure cannot cancel or skip a +sibling. + +#### 2026-09-07 amendment: one dispatch per pull request, adopted for the 60-job ceiling + +The 2026-09-05 per-language run was the right fix for the accident it +recorded. contextual-orchestrator PR #1049 dispatched three current-head +language jobs, and central run `33938784437` was the sole survivor because +the handler's group omitted `required_language`. Sibling runs cancelled one +another and left their required jobs failed in the `pending` handoff state. +Sending the full language matrix in one dispatch was rejected then because +the handler validated one shard and woke one exact required job per run; +enlarging that surface had no observed need. + +That need now exists. On 2026-09-07 the organization job ceiling (60 jobs) +was saturated by this fan-out: ContextualWisdomLab/.github had ~300 queued +runs, 149 of them `codeql-scan-dispatch.yml`, covering 60 PR@SHA tuples +(n=2:29, n=3:27, n=4:2). Duplicate cancellation could not collapse them: +the language is not present on the run name, the job name, or the REST +payload. The user-facing concurrency contract for pull-request workflows is +`{workflow}-{repository}-{PR}` with `cancel-in-progress: true` only for a +superseded HEAD of the same pull request, and a language suffix is +forbidden. + +The 2026-09-05 rejection of "full matrix in one dispatch" is therefore +superseded. The sibling-cancel failure mode is gone because siblings are +jobs in one run, not runs in one concurrency group. The exact-job wake +contract is preserved: `required_jobs` is a 1:1 map of language to canonical +job id, each scan shard looks up only its own id, and a missing, stale, or +mismatched identity still fails closed. The old scalar +`required_job_id`/`required_language` payload is retired. ## Scope decision: `analyze-merge` is dropped, not migrated @@ -240,9 +280,10 @@ blocker for this one. inline Python between `analyze-head`/`analyze-merge` today. exact run/job wake-up follows the OpenCode runner-release pattern while avoiding one occupied runner per language for the scan's full duration. -- A repository and pull request can now have one active native handler per - language. This modest concurrency increase is bounded by the detected CodeQL - matrix and prevents valid sibling evidence from being treated as stale work. +- A repository and pull request have one active native handler run. Language + parallelism is bounded by the detected CodeQL matrix inside that run, and a + superseded HEAD of the same pull request cancels the in-flight handler + instead of queuing another copy per language. - Re-admitting `codeql-pr.yml` to ruleset `18156473` must happen only after this design is implemented, tested, and its `detect-languages`/ `dispatch-analysis`/`analyze-head` jobs are confirmed free of any diff --git a/tests/test_codeql_pr_workflow_contract.py b/tests/test_codeql_pr_workflow_contract.py index 90612e9bc8..2d11ca0141 100644 --- a/tests/test_codeql_pr_workflow_contract.py +++ b/tests/test_codeql_pr_workflow_contract.py @@ -59,25 +59,49 @@ def test_codeql_pr_workflow_structure() -> None: assert "commits/${PR_HEAD_SHA}/statuses" in workflow -def test_codeql_pr_dispatches_one_language_per_shard_not_the_full_matrix() -> None: - """Every shard dispatches, but only its own language, not the full matrix. - - Each shard carries its own run, job, language, and head identity so the - trusted dispatcher can wake only that intentionally failed job. +def test_codeql_pr_shards_do_not_dispatch_and_coordinator_sends_the_full_matrix_once() -> None: + """Shards consume verdicts; one coordinator POSTs the remaining language matrix. + + Per-language repository_dispatch runs were the 60-job ceiling: live + 2026-09-07 queued ~149 ``codeql-scan-dispatch.yml`` runs across 60 PR@SHA + tuples because each analyze-head shard POSTed its own ``codeql-scan``. + Language independence now lives in the handler's job matrix, so the + required workflow may send every still-pending language in one payload. """ workflow = WORKFLOW_PATH.read_text(encoding="utf-8") - - assert "id: dispatch" in workflow - assert 'matrix:[{language:$language,"build-mode":$build_mode}]' in workflow - assert "needs.detect-languages.outputs.matrix).include[0]" not in workflow - assert "DISPATCH_OUTCOME: ${{ steps.dispatch.outcome }}" in workflow - assert workflow.count("- name: Request current-head CodeQL scan dispatch") == 1 + analyze_head = workflow.split(" analyze-head:\n", 1)[1].split( + " dispatch-current-head:\n", 1 + )[0] + coordinator = workflow.split(" dispatch-current-head:\n", 1)[1] + + assert "id: dispatch" in analyze_head + assert "repos/ContextualWisdomLab/.github/dispatches" not in analyze_head + assert 'event_type:"codeql-scan"' not in analyze_head + assert 'matrix:[{language:$language,"build-mode":$build_mode}]' not in workflow + assert "required_job_id:$required_job_id" not in analyze_head + assert "required_language:$required_language" not in analyze_head + assert "DISPATCH_OUTCOME: ${{ steps.dispatch.outcome }}" in analyze_head + assert workflow.count("- name: Read current-head CodeQL dispatch verdict") == 1 assert workflow.count("- name: Release runner or enforce current-head CodeQL verdict") == 1 + assert workflow.count("- name: Dispatch current-head CodeQL scan") == 1 + assert "needs: [detect-languages, analyze-head]" in coordinator + assert "always()" in coordinator.split("\n runs-on:", 1)[0] + assert "github.event.action != 'closed'" in coordinator.split("\n runs-on:", 1)[0] + assert "github.run_attempt == 1" in coordinator.split("\n runs-on:", 1)[0] + assert coordinator.count("repos/ContextualWisdomLab/.github/dispatches") == 1 + assert 'event_type:"codeql-scan"' in coordinator + assert "required_jobs:$required_jobs" in coordinator + assert "required_run_id:$required_run_id" in coordinator + assert "required_job_id:$required_job_id" not in coordinator + assert "required_language:$required_language" not in coordinator + assert "actions/runs/${REQUIRED_RUN_ID}/jobs" in coordinator + assert "CodeQL compatibility analysis (" in coordinator RUN_BLOCK_STEP_NAMES = ( - "Request current-head CodeQL scan dispatch", + "Read current-head CodeQL dispatch verdict", "Release runner or enforce current-head CodeQL verdict", + "Dispatch current-head CodeQL scan", ) @@ -103,8 +127,9 @@ def test_codeql_pr_dispatch_and_release_run_blocks_are_valid_bash() -> None: assert result.returncode == 0, f"{step_name}: {result.stderr}" -DISPATCH_STEP_NAME = "Request current-head CodeQL scan dispatch" +DISPATCH_STEP_NAME = "Read current-head CodeQL dispatch verdict" VERDICT_STEP_NAME = "Release runner or enforce current-head CodeQL verdict" +COORDINATOR_STEP_NAME = "Dispatch current-head CodeQL scan" def _run_verdict_read( @@ -238,19 +263,21 @@ def test_codeql_action_steps_use_one_version_per_workflow() -> None: assert len(refs) == 1, f"scheduled-security-scan.yml mixes CodeQL action refs: {sorted(refs)}" -def test_codeql_shard_releases_runner_and_dispatches_exact_wake_identity() -> None: +def test_codeql_shard_releases_runner_and_reads_exact_head_verdict() -> None: workflow = WORKFLOW_PATH.read_text(encoding="utf-8") - shard = workflow.split(" analyze-head:\n", 1)[1] + shard = workflow.split(" analyze-head:\n", 1)[1].split( + " dispatch-current-head:\n", 1 + )[0] assert "while :; do" not in shard assert "poll_interval_seconds" not in shard assert "sleep " not in shard - assert "job.check_run_id" in shard - assert "required_run_id:$required_run_id" in shard - assert "required_job_id:$required_job_id" in shard - assert "required_language:$required_language" in shard + assert "job.check_run_id" not in shard + assert "required_job_id:$required_job_id" not in shard + assert "required_language:$required_language" not in shard assert "The dispatch workflow will rerun this exact failed CodeQL job" in shard assert "commits/${PR_HEAD_SHA}/statuses" in shard + assert "repos/ContextualWisdomLab/.github/dispatches" not in shard def test_codeql_required_workflow_does_not_gain_actions_write() -> None: @@ -259,6 +286,335 @@ def test_codeql_required_workflow_does_not_gain_actions_write() -> None: shard_permissions = workflow.split(" analyze-head:\n", 1)[1].split( " strategy:\n", 1 )[0] + coordinator_permissions = workflow.split(" dispatch-current-head:\n", 1)[1].split( + " steps:\n", 1 + )[0] assert "actions: write" not in permissions assert "actions: write" not in shard_permissions + assert "actions: write" not in coordinator_permissions + + +def test_codeql_pr_attempt_one_without_verdict_fails_pending_without_dispatch( + tmp_path: Path, +) -> None: + """Attempt 1 with no authenticated status releases the runner and does not POST.""" + bash = shutil.which("bash") + jq = shutil.which("jq") + assert bash is not None and jq is not None, "bash and jq are required to run this test" + + workflow_text = WORKFLOW_PATH.read_text(encoding="utf-8") + dispatch_script = _extract_run_block(workflow_text, DISPATCH_STEP_NAME) + verdict_script = _extract_run_block(workflow_text, VERDICT_STEP_NAME) + head_sha = "b" * 40 + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + post_log = tmp_path / "posts" + fake_gh = fake_bin / "gh" + fake_gh.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + 'test "$1" = api\n' + 'if [ "${2:-}" = "-X" ]; then\n' + ' printf \'%s\\n\' "$4" >>"$FAKE_POST_LOG"\n' + " exit 0\n" + "fi\n" + 'case "$2" in\n' + " */pulls/*) printf '%s\\n' \"$FAKE_PULL_JSON\" ;;\n" + " */statuses) printf '%s\\n' \"$FAKE_STATUSES_JSON\" ;;\n" + " *) exit 1 ;;\n" + "esac\n", + encoding="utf-8", + ) + fake_gh.chmod(0o755) + output = tmp_path / "github-output" + env = { + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "FAKE_PULL_JSON": json.dumps({"head": {"sha": head_sha}, "state": "open"}), + "FAKE_STATUSES_JSON": json.dumps([]), + "FAKE_POST_LOG": str(post_log), + "GH_TOKEN": "fake-token", + "TARGET_REPOSITORY": "ContextualWisdomLab/naruon", + "PR_NUMBER": "42", + "PR_HEAD_SHA": head_sha, + "LANGUAGE": "python", + "BUILD_MODE": "none", + "RUN_ATTEMPT": "1", + "REQUIRED_RUN_ID": "42", + "GITHUB_OUTPUT": str(output), + } + dispatch_result = subprocess.run( + [bash], input=dispatch_script, text=True, capture_output=True, check=False, + env=env, timeout=60, + ) + assert dispatch_result.returncode == 0, dispatch_result.stderr + assert "verdict=pending" in output.read_text(encoding="utf-8") + assert not post_log.exists() + verdict_result = subprocess.run( + [bash], + input=verdict_script, + text=True, + capture_output=True, + check=False, + env={ + **os.environ, + "LANGUAGE": "python", + "DISPATCH_OUTCOME": "success", + "VERDICT_STATE": "pending", + }, + timeout=60, + ) + assert verdict_result.returncode == 1 + assert "CodeQL scan dispatched" in verdict_result.stdout + + +def _write_coordinator_fakes( + tmp_path: Path, + *, + pull: dict, + jobs: dict, + statuses: list[dict], +) -> tuple[Path, Path, Path]: + """Install fake gh/curl binaries and return (bin, post_log, post_body).""" + fake_bin = tmp_path / "bin" + fake_bin.mkdir(parents=True) + post_log = tmp_path / "posts" + post_body = tmp_path / "post-body" + fake_gh = fake_bin / "gh" + fake_gh.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + 'test "$1" = api\n' + "shift\n" + "method=GET\n" + "path=\n" + "jq_filter=\n" + "while [ $# -gt 0 ]; do\n" + ' case "$1" in\n' + " -X) shift; method=$1 ;;\n" + " --input) shift; input=$1 ;;\n" + " --jq|-q) shift; jq_filter=$1 ;;\n" + " --paginate) ;;\n" + ' repos/*) path=$1 ;;\n' + " esac\n" + " shift || true\n" + "done\n" + 'if [ "$method" = POST ]; then\n' + ' printf \'%s\\n\' "$path" >>"$FAKE_POST_LOG"\n' + ' if [ "${input:-}" = "-" ]; then cat >>"$FAKE_POST_BODY"; fi\n' + " exit 0\n" + "fi\n" + "body=\n" + 'case "$path" in\n' + " */pulls/*) body=$FAKE_PULL_JSON ;;\n" + " */statuses) body=$FAKE_STATUSES_JSON ;;\n" + " */actions/runs/*/jobs) body=$FAKE_JOBS_JSON ;;\n" + " *) exit 1 ;;\n" + "esac\n" + 'if [ -n "${jq_filter}" ]; then printf \'%s\\n\' "$body" | jq -c "$jq_filter"; else printf \'%s\\n\' "$body"; fi\n', + encoding="utf-8", + ) + fake_gh.chmod(0o755) + fake_curl = fake_bin / "curl" + fake_curl.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + 'printf \'%s\\n\' "$*" >>"$FAKE_CURL_LOG"\n' + 'if [[ " $* " == *"exchange_github_app_token"* ]]; then\n' + " printf '%s\\n' '{\"token\":\"fake-app-token\"}'\n" + " exit 0\n" + "fi\n" + "printf '%s\\n' '{\"value\":\"fake-oidc-token\"}'\n", + encoding="utf-8", + ) + fake_curl.chmod(0o755) + (tmp_path / "pull.json").write_text(json.dumps(pull), encoding="utf-8") + (tmp_path / "jobs.json").write_text(json.dumps(jobs), encoding="utf-8") + (tmp_path / "statuses.json").write_text(json.dumps(statuses), encoding="utf-8") + return fake_bin, post_log, post_body + + +def _run_coordinator( + tmp_path: Path, + *, + pull: dict | None = None, + jobs: dict | None = None, + statuses: list[dict] | None = None, + env_overrides: dict[str, str] | None = None, +) -> tuple[subprocess.CompletedProcess[str], Path, Path]: + """Execute the coordinator dispatch block against fixture-backed APIs.""" + bash = shutil.which("bash") + jq = shutil.which("jq") + assert bash is not None and jq is not None, "bash and jq are required to run this test" + + head_sha = "b" * 40 + pull = pull or { + "state": "open", + "head": {"sha": head_sha, "ref": "feature"}, + "base": {"sha": "a" * 40, "ref": "main"}, + } + jobs = jobs or { + "total_count": 2, + "jobs": [ + { + "id": 101, + "name": "CodeQL compatibility analysis (python)", + "status": "completed", + "conclusion": "failure", + }, + { + "id": 102, + "name": "CodeQL compatibility analysis (actions)", + "status": "completed", + "conclusion": "failure", + }, + ], + } + statuses = statuses if statuses is not None else [] + fake_bin, post_log, post_body = _write_coordinator_fakes( + tmp_path, pull=pull, jobs=jobs, statuses=statuses + ) + script = _extract_run_block( + WORKFLOW_PATH.read_text(encoding="utf-8"), COORDINATOR_STEP_NAME + ) + env = { + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "FAKE_PULL_JSON": json.dumps(pull), + "FAKE_JOBS_JSON": json.dumps(jobs), + "FAKE_STATUSES_JSON": json.dumps(statuses), + "FAKE_POST_LOG": str(post_log), + "FAKE_POST_BODY": str(post_body), + "FAKE_CURL_LOG": str(tmp_path / "curl.log"), + "GH_TOKEN": "fake-token", + "TARGET_REPOSITORY": "ContextualWisdomLab/naruon", + "GITHUB_REPOSITORY": "ContextualWisdomLab/naruon", + "PR_NUMBER": "42", + "PR_BASE_REF": "main", + "PR_BASE_SHA": "a" * 40, + "PR_HEAD_REF": "feature", + "PR_HEAD_SHA": head_sha, + "REQUIRED_RUN_ID": "99", + "MATRIX": json.dumps( + { + "include": [ + {"language": "python", "build-mode": "none"}, + {"language": "actions", "build-mode": "none"}, + ] + } + ), + "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "oidc-request-token", + "ACTIONS_ID_TOKEN_REQUEST_URL": "https://example.test/oidc", + "OIDC_AUDIENCE": "opencode-github-action", + "OPENCODE_API_BASE_URL": "https://api.opencode.ai", + **(env_overrides or {}), + } + result = subprocess.run( + [bash], input=script, text=True, capture_output=True, check=False, env=env, + timeout=60, + ) + return result, post_log, post_body + + +def test_codeql_coordinator_posts_one_dispatch_for_every_pending_language( + tmp_path: Path, +) -> None: + """One repository_dispatch carries every language that still needs a scan.""" + result, post_log, post_body = _run_coordinator(tmp_path) + + assert result.returncode == 0, result.stderr + result.stdout + assert post_log.read_text(encoding="utf-8").splitlines() == [ + "repos/ContextualWisdomLab/.github/dispatches" + ] + payload = json.loads(post_body.read_text(encoding="utf-8")) + assert payload["event_type"] == "codeql-scan" + client = payload["client_payload"] + assert client["target_repository"] == "ContextualWisdomLab/naruon" + assert client["pr_number"] == "42" + assert client["required_run_id"] == "99" + assert "required_job_id" not in client + assert "required_language" not in client + languages = [entry["language"] for entry in client["matrix"]] + assert languages == ["python", "actions"] + jobs_by_language = { + entry["language"]: entry["job_id"] for entry in client["required_jobs"] + } + assert jobs_by_language == {"python": 101, "actions": 102} + + +def test_codeql_coordinator_skips_dispatch_when_every_language_has_a_verdict( + tmp_path: Path, +) -> None: + """A rerun that already has terminal statuses must not enqueue another scan.""" + result, post_log, post_body = _run_coordinator( + tmp_path, + statuses=[ + { + "context": "codeql-dispatch/python", + "state": "success", + "creator": {"login": "opencode-agent[bot]"}, + }, + { + "context": "codeql-dispatch/actions", + "state": "failure", + "creator": {"login": "opencode-agent[bot]"}, + }, + ], + ) + + assert result.returncode == 0, result.stderr + result.stdout + assert not post_log.exists() + assert not post_body.exists() or post_body.read_text(encoding="utf-8") == "" + assert "already have authenticated terminal verdicts" in result.stdout + + +def test_codeql_coordinator_fails_closed_when_a_shard_job_id_is_missing( + tmp_path: Path, +) -> None: + """A matrix language with no analyze-head job cannot be woken later.""" + result, post_log, _post_body = _run_coordinator( + tmp_path, + jobs={ + "total_count": 1, + "jobs": [ + { + "id": 101, + "name": "CodeQL compatibility analysis (python)", + "status": "completed", + "conclusion": "failure", + } + ], + }, + ) + + assert result.returncode == 1 + assert "missing current-head job id" in result.stdout + assert not post_log.exists() + + +def test_codeql_coordinator_does_not_dispatch_a_closed_or_stale_pull_request( + tmp_path: Path, +) -> None: + """Live-head revalidation remains fail-closed before the single POST.""" + closed, closed_log, _closed_body = _run_coordinator( + tmp_path / "closed", + pull={ + "state": "closed", + "head": {"sha": "b" * 40, "ref": "feature"}, + "base": {"sha": "a" * 40, "ref": "main"}, + }, + ) + stale, stale_log, _stale_body = _run_coordinator( + tmp_path / "stale", + pull={ + "state": "open", + "head": {"sha": "c" * 40, "ref": "feature"}, + "base": {"sha": "a" * 40, "ref": "main"}, + }, + ) + + assert closed.returncode == 0, closed.stderr + assert stale.returncode == 0, stale.stderr + assert not closed_log.exists() + assert not stale_log.exists() diff --git a/tests/test_codeql_scan_dispatch_workflow_contract.py b/tests/test_codeql_scan_dispatch_workflow_contract.py index 71fa43541f..2b3208f6a0 100644 --- a/tests/test_codeql_scan_dispatch_workflow_contract.py +++ b/tests/test_codeql_scan_dispatch_workflow_contract.py @@ -95,18 +95,29 @@ def test_codeql_scan_dispatch_workflow_structure(): def test_codeql_scan_dispatch_keeps_current_head_language_shards_independent(): - """A current-head language scan cannot cancel its sibling language scans.""" + """Sibling languages stay independent as jobs in one run, not as separate runs. + + The 60-job ceiling was one queued handler run per language. Putting + ``required_language`` in the concurrency group was the 2026-09-05 + workaround after contextual-orchestrator#1049 / run 33938784437 cancelled + sibling scans. Independence now comes from ``strategy.fail-fast: false`` + on this run's language matrix, so the group can be + ``{workflow}-{repository}-{PR}`` and ``cancel-in-progress: true`` only + drops a superseded HEAD of the same pull request. + """ workflow = WORKFLOW_PATH.read_text(encoding="utf-8") group_value = workflow_level_concurrency_group(workflow) + header = workflow.split("\non:", 1)[0] + scan = workflow.split(" scan:\n", 1)[1] + strategy = scan.split(" strategy:\n", 1)[1].split(" steps:\n", 1)[0] - # The language segment is what keeps sibling language shards in separate groups, so it is - # asserted on the group's own value: a comment naming it would otherwise satisfy the check - # while the key had lost it, silently letting one language's scan cancel another's. assert "github.event.client_payload.target_repository" in group_value assert "github.event.client_payload.pr_number" in group_value - assert "github.event.client_payload.required_language" in group_value - # Same reasoning as the group above, applied to the flag: the substring form - # is satisfied by a comment quoting it while the key beside it reads false. + assert "github.event.client_payload.required_language" not in group_value + assert "unknown-language" not in group_value + assert "required_language" not in header + assert "fail-fast: false" in strategy + assert "include: ${{ fromJSON(needs.validate-dispatch.outputs.matrix) }}" in strategy assert workflow_level_cancels_in_progress(workflow) @@ -148,8 +159,7 @@ def _run_validate_step(tmp_path: Path, env_overrides: dict[str, str], pull_reque "SUPPLIED_HEAD_SHA": "b" * 40, "SUPPLIED_MATRIX": json.dumps([{"language": "python", "build-mode": "none"}]), "SUPPLIED_REQUIRED_RUN_ID": "42", - "SUPPLIED_REQUIRED_JOB_ID": "43", - "SUPPLIED_REQUIRED_LANGUAGE": "python", + "SUPPLIED_REQUIRED_JOBS": json.dumps([{"language": "python", "job_id": 43}]), **env_overrides, } result = subprocess.run([bash], input=script, text=True, capture_output=True, check=False, env=env) @@ -177,8 +187,9 @@ def test_codeql_scan_dispatch_validate_step_accepts_matching_live_metadata(tmp_p assert "head_sha=" + "b" * 40 in output_text assert '[{"language":"python","build-mode":"none"}]' in output_text assert "required_run_id=42" in output_text - assert "required_job_id=43" in output_text - assert "required_language=python" in output_text + assert '"job_id":43' in output_text.replace(" ", "") + assert "required_job_id=" not in output_text + assert "required_language=" not in output_text def test_codeql_scan_dispatch_validate_step_rejects_actor_mismatch(tmp_path): @@ -270,15 +281,78 @@ def test_codeql_scan_dispatch_validate_step_rejects_non_org_target(tmp_path): def test_codeql_scan_dispatch_validate_step_rejects_malformed_matrix(tmp_path): - """A matrix entry missing a valid language/build-mode fails closed.""" + """Empty, invalid, or job-map-mismatched matrices fail closed; a multi-language payload is valid.""" + missing_build_mode = _run_validate_step( + tmp_path / "missing-build-mode", + {"SUPPLIED_MATRIX": json.dumps([{"language": "python"}])}, + _matching_pull_request(), + ) + empty_matrix = _run_validate_step( + tmp_path / "empty", + { + "SUPPLIED_MATRIX": "[]", + "SUPPLIED_REQUIRED_JOBS": "[]", + }, + _matching_pull_request(), + ) + invalid_language = _run_validate_step( + tmp_path / "invalid-language", + { + "SUPPLIED_MATRIX": json.dumps([{"language": "PYTHON", "build-mode": "none"}]), + "SUPPLIED_REQUIRED_JOBS": json.dumps([{"language": "PYTHON", "job_id": 43}]), + }, + _matching_pull_request(), + ) + mismatched_jobs = _run_validate_step( + tmp_path / "mismatched-jobs", + { + "SUPPLIED_MATRIX": json.dumps( + [ + {"language": "python", "build-mode": "none"}, + {"language": "actions", "build-mode": "none"}, + ] + ), + "SUPPLIED_REQUIRED_JOBS": json.dumps([{"language": "python", "job_id": 43}]), + }, + _matching_pull_request(), + ) + + assert missing_build_mode.returncode == 1 + assert empty_matrix.returncode == 1 + assert invalid_language.returncode == 1 + assert mismatched_jobs.returncode == 1 + assert "at least one valid language/build-mode shard" in missing_build_mode.stdout + assert "at least one valid language/build-mode shard" in empty_matrix.stdout + assert "at least one valid language/build-mode shard" in invalid_language.stdout + assert "does not match the dispatched languages one-to-one" in mismatched_jobs.stdout + + +def test_codeql_scan_dispatch_validate_step_accepts_multi_language_payload(tmp_path): + """One dispatch may carry every remaining language for the current head.""" result = _run_validate_step( tmp_path, - {"SUPPLIED_MATRIX": json.dumps([{"language": "python"}])}, + { + "SUPPLIED_MATRIX": json.dumps( + [ + {"language": "python", "build-mode": "none"}, + {"language": "javascript-typescript", "build-mode": "none"}, + ] + ), + "SUPPLIED_REQUIRED_JOBS": json.dumps( + [ + {"language": "javascript-typescript", "job_id": "55"}, + {"language": "python", "job_id": 43}, + ] + ), + }, _matching_pull_request(), ) - assert result.returncode == 1 - assert "matrix must contain exactly one valid language/build-mode shard" in result.stdout + assert result.returncode == 0, result.stderr + result.stdout + output_text = result.output_path.read_text(encoding="utf-8") + assert "javascript-typescript" in output_text + assert '"job_id":55' in output_text.replace(" ", "") + assert '"job_id":43' in output_text.replace(" ", "") def test_codeql_scan_dispatch_validate_step_rejects_stale_head_sha(tmp_path): @@ -347,8 +421,9 @@ def test_dispatch_wake_has_only_trusted_actions_write_boundary() -> None: assert "actions: write" in scan_permissions assert "pull_request:" not in workflow assert "pull_request_target:" not in workflow - assert "github.event.client_payload.required_run_id != ''" in scan - assert "github.event.client_payload.required_job_id != ''" in scan + assert "needs.validate-dispatch.outputs.required_run_id != ''" in scan + assert "needs.validate-dispatch.outputs.required_jobs != ''" in scan + assert "github.event.client_payload.required_job_id" not in scan def _run_wake_step( @@ -419,7 +494,12 @@ def _run_wake_step( "PR_NUMBER": "42", "HEAD_SHA": head_sha, "REQUIRED_RUN_ID": "42", - "REQUIRED_JOB_ID": "43", + "REQUIRED_JOBS": json.dumps( + [ + {"language": "python", "job_id": 43}, + {"language": "actions", "job_id": 44}, + ] + ), "REQUIRED_LANGUAGE": "python", } result = subprocess.run( diff --git a/tests/test_docs_only_pr_runner_admission.py b/tests/test_docs_only_pr_runner_admission.py index 8b2e6e8ee2..49631d2a19 100644 --- a/tests/test_docs_only_pr_runner_admission.py +++ b/tests/test_docs_only_pr_runner_admission.py @@ -186,13 +186,13 @@ def test_codeql_pr_gates_analyze_head_at_step_level_not_job_level(): the required `CodeQL compatibility analysis (actions|python)` contexts, so those required checks never appear. Gating the steps instead lets the job run (~20s), succeed, and publish the correctly expanded names. Since - the dispatch+poll rewrite (docs/adr/0025-codeql-required-workflow-dispatch-architecture.md), - `analyze-head` has two steps: the dispatch step's `if:` additionally - restricts it to the first matrix shard (see - tests/test_codeql_pr_workflow_contract.py::test_codeql_pr_dispatches_once_not_once_per_matrix_shard), - while the poll step runs unconditionally on `code == 'true'` alone -- both - still gate at step level, never at job level. `analyze-merge` no longer - exists: it was required nowhere (PR #1766) and was dropped, not migrated. + the dispatch+exact-job-wake rewrite + (docs/adr/0025-codeql-required-workflow-dispatch-architecture.md), + `analyze-head` has two steps: the verdict-read step and the runner-release + step. Both still gate at step level on `code == 'true'`, never at job + level. The one-shot coordinator that POSTs the remaining language matrix + is a separate job. `analyze-merge` no longer exists: it was required + nowhere (PR #1766) and was dropped, not migrated. """ workflow = _read("codeql-pr.yml") diff --git a/tests/test_scheduler_and_codeql_dispatch_runner_image_contract.py b/tests/test_scheduler_and_codeql_dispatch_runner_image_contract.py index dd7af43427..ba0b2598a9 100644 --- a/tests/test_scheduler_and_codeql_dispatch_runner_image_contract.py +++ b/tests/test_scheduler_and_codeql_dispatch_runner_image_contract.py @@ -45,10 +45,10 @@ def test_hourly_review_repair_uses_explicit_supported_image(self) -> None: self.assert_explicit_supported_image(HOURLY_REVIEW_REPAIR) def test_codeql_pr_uses_explicit_supported_image(self) -> None: - """Require both CodeQL PR compatibility-analysis jobs to pin Ubuntu 24.04.""" + """Require detect-languages, analyze-head, and the coordinator to pin Ubuntu 24.04.""" workflow = CODEQL_PR.read_text(encoding="utf-8") self.assertNotIn("runs-on: ubuntu-latest", workflow) - self.assertEqual(workflow.count("runs-on: ubuntu-24.04"), 2) + self.assertEqual(workflow.count("runs-on: ubuntu-24.04"), 3) def test_codeql_scan_dispatch_uses_explicit_supported_image(self) -> None: """Require both CodeQL Scan Dispatch jobs to pin Ubuntu 24.04.""" From 78a4937c684a54ca8e415822c913742f41c6efc4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 16:47:06 +0900 Subject: [PATCH 40/80] fix(codeql): accept queued pre-cutover scan-dispatch payloads (#2009) #2008 made required_jobs mandatory on the default-branch handler. repository_dispatch runs that file, so payloads that lined up before the merge still carry required_language + required_job_id and a one-shard matrix. Validate now synthesizes required_jobs from those scalars when the array is empty, the matrix length is 1, and the language matches. A present required_jobs array still wins; missing or mismatched identity still fails closed. Concurrency stays {workflow}-{repository}-{PR} with no language suffix. Developer experience: queued single-language dispatches validate instead of all failing closed after the cutover. User experience: required CodeQL compatibility checks for those in-flight PRs can still complete. --- .github/workflows/codeql-scan-dispatch.yml | 13 ++ ..._codeql_scan_dispatch_workflow_contract.py | 126 ++++++++++++++++++ 2 files changed, 139 insertions(+) diff --git a/.github/workflows/codeql-scan-dispatch.yml b/.github/workflows/codeql-scan-dispatch.yml index 23c47f6abc..521ceeb167 100644 --- a/.github/workflows/codeql-scan-dispatch.yml +++ b/.github/workflows/codeql-scan-dispatch.yml @@ -147,6 +147,11 @@ jobs: SUPPLIED_MATRIX: ${{ toJSON(github.event.client_payload.matrix) }} SUPPLIED_REQUIRED_RUN_ID: ${{ github.event.client_payload.required_run_id || '' }} SUPPLIED_REQUIRED_JOBS: ${{ toJSON(github.event.client_payload.required_jobs) }} + # Pre-#2008 payloads still send scalar required_job_id + + # required_language with a one-shard matrix. Synthesize + # required_jobs from those only when the array is empty. + SUPPLIED_REQUIRED_JOB_ID: ${{ github.event.client_payload.required_job_id || '' }} + SUPPLIED_REQUIRED_LANGUAGE: ${{ github.event.client_payload.required_language || '' }} run: | set -euo pipefail # ALLOWED_DISPATCH_ACTOR is a comma-separated allowlist shared with @@ -185,6 +190,14 @@ jobs: printf '::error::CodeQL scan dispatch matrix must contain at least one valid language/build-mode shard with unique languages. matrix=%s\n' "${SUPPLIED_MATRIX:-}" exit 1 fi + if [ -z "$jobs_json" ] || + [ "$(printf '%s' "$jobs_json" | jq '(. == null) or (. == [])')" = "true" ]; then + if [ "$(printf '%s' "$matrix_json" | jq 'type == "array" and length == 1')" = "true" ] && + [[ "$SUPPLIED_REQUIRED_JOB_ID" =~ ^[1-9][0-9]*$ ]] && + [ "$SUPPLIED_REQUIRED_LANGUAGE" = "$(printf '%s' "$matrix_json" | jq -r '.[0].language // empty')" ]; then + jobs_json="$(jq -nc --arg language "$SUPPLIED_REQUIRED_LANGUAGE" --arg job_id "$SUPPLIED_REQUIRED_JOB_ID" '[{language: $language, job_id: ($job_id | tonumber)}]')" + fi + fi if [ -z "$jobs_json" ] || [ "$(jq -n --argjson matrix "$matrix_json" --argjson jobs "$jobs_json" ' ($jobs | type == "array") diff --git a/tests/test_codeql_scan_dispatch_workflow_contract.py b/tests/test_codeql_scan_dispatch_workflow_contract.py index 2b3208f6a0..dea1326494 100644 --- a/tests/test_codeql_scan_dispatch_workflow_contract.py +++ b/tests/test_codeql_scan_dispatch_workflow_contract.py @@ -160,6 +160,8 @@ def _run_validate_step(tmp_path: Path, env_overrides: dict[str, str], pull_reque "SUPPLIED_MATRIX": json.dumps([{"language": "python", "build-mode": "none"}]), "SUPPLIED_REQUIRED_RUN_ID": "42", "SUPPLIED_REQUIRED_JOBS": json.dumps([{"language": "python", "job_id": 43}]), + "SUPPLIED_REQUIRED_JOB_ID": "", + "SUPPLIED_REQUIRED_LANGUAGE": "", **env_overrides, } result = subprocess.run([bash], input=script, text=True, capture_output=True, check=False, env=env) @@ -355,6 +357,118 @@ def test_codeql_scan_dispatch_validate_step_accepts_multi_language_payload(tmp_p assert '"job_id":43' in output_text.replace(" ", "") +def test_codeql_scan_dispatch_validate_step_accepts_legacy_single_language_payload(tmp_path): + """A queued pre-cutover payload still validates after required_jobs became mandatory. + + repository_dispatch always runs the default-branch file. Payloads that + lined up before #2008 carry required_language + required_job_id and a + one-shard matrix, with required_jobs absent (JSON null) or empty. Those + fields synthesize required_jobs=[{language, job_id}] and must be accepted. + """ + for empty_jobs, case_name in (("null", "missing"), ("[]", "empty-array")): + result = _run_validate_step( + tmp_path / case_name, + { + "SUPPLIED_REQUIRED_JOBS": empty_jobs, + "SUPPLIED_REQUIRED_LANGUAGE": "python", + "SUPPLIED_REQUIRED_JOB_ID": "43", + }, + _matching_pull_request(), + ) + + assert result.returncode == 0, result.stderr + result.stdout + output_text = result.output_path.read_text(encoding="utf-8") + compact = output_text.replace(" ", "") + assert '"language":"python"' in compact + assert '"job_id":43' in compact + assert "required_job_id=" not in output_text + assert "required_language=" not in output_text + + +def test_codeql_scan_dispatch_validate_step_ignores_legacy_fields_when_required_jobs_present( + tmp_path, +): + """A current required_jobs array wins; leftover scalar fields are ignored.""" + result = _run_validate_step( + tmp_path, + { + "SUPPLIED_MATRIX": json.dumps( + [ + {"language": "python", "build-mode": "none"}, + {"language": "javascript-typescript", "build-mode": "none"}, + ] + ), + "SUPPLIED_REQUIRED_JOBS": json.dumps( + [ + {"language": "javascript-typescript", "job_id": "55"}, + {"language": "python", "job_id": 43}, + ] + ), + "SUPPLIED_REQUIRED_LANGUAGE": "actions", + "SUPPLIED_REQUIRED_JOB_ID": "999", + }, + _matching_pull_request(), + ) + + assert result.returncode == 0, result.stderr + result.stdout + compact = result.output_path.read_text(encoding="utf-8").replace(" ", "") + assert '"job_id":55' in compact + assert '"job_id":43' in compact + assert '"job_id":999' not in compact + assert "actions" not in compact + + +def test_codeql_scan_dispatch_validate_step_rejects_unusable_legacy_payload(tmp_path): + """Empty required_jobs still fail closed when the scalar identity cannot be synthesized.""" + missing_both = _run_validate_step( + tmp_path / "missing-both", + {"SUPPLIED_REQUIRED_JOBS": "null"}, + _matching_pull_request(), + ) + language_mismatch = _run_validate_step( + tmp_path / "language-mismatch", + { + "SUPPLIED_REQUIRED_JOBS": "[]", + "SUPPLIED_REQUIRED_LANGUAGE": "javascript-typescript", + "SUPPLIED_REQUIRED_JOB_ID": "43", + }, + _matching_pull_request(), + ) + multi_language_legacy = _run_validate_step( + tmp_path / "multi-language-legacy", + { + "SUPPLIED_MATRIX": json.dumps( + [ + {"language": "python", "build-mode": "none"}, + {"language": "javascript-typescript", "build-mode": "none"}, + ] + ), + "SUPPLIED_REQUIRED_JOBS": "null", + "SUPPLIED_REQUIRED_LANGUAGE": "python", + "SUPPLIED_REQUIRED_JOB_ID": "43", + }, + _matching_pull_request(), + ) + invalid_job_id = _run_validate_step( + tmp_path / "invalid-job-id", + { + "SUPPLIED_REQUIRED_JOBS": "null", + "SUPPLIED_REQUIRED_LANGUAGE": "python", + "SUPPLIED_REQUIRED_JOB_ID": "0", + }, + _matching_pull_request(), + ) + + assert missing_both.returncode == 1 + assert language_mismatch.returncode == 1 + assert multi_language_legacy.returncode == 1 + assert invalid_job_id.returncode == 1 + assert "does not match the dispatched languages one-to-one" in missing_both.stdout + assert "does not match the dispatched languages one-to-one" in language_mismatch.stdout + assert "does not match the dispatched languages one-to-one" in multi_language_legacy.stdout + assert "does not match the dispatched languages one-to-one" in invalid_job_id.stdout + + def test_codeql_scan_dispatch_validate_step_rejects_stale_head_sha(tmp_path): """A dispatch whose supplied head SHA no longer matches the live PR head is rejected.""" stale_pull_request = _matching_pull_request() @@ -602,3 +716,15 @@ def test_codeql_scan_dispatch_serialises_the_matrix_payload() -> None: assert ( "SUPPLIED_MATRIX: ${{ github.event.client_payload.matrix" not in workflow ), "SUPPLIED_MATRIX must not assign the raw client_payload array to env:" + assert ( + "SUPPLIED_REQUIRED_JOBS: ${{ toJSON(github.event.client_payload.required_jobs) }}" + in workflow + ), "SUPPLIED_REQUIRED_JOBS must be serialised with toJSON(); a bare array breaks template validation" + assert ( + "SUPPLIED_REQUIRED_JOB_ID: ${{ github.event.client_payload.required_job_id || '' }}" + in workflow + ), "Queued pre-cutover payloads still supply required_job_id as a scalar" + assert ( + "SUPPLIED_REQUIRED_LANGUAGE: ${{ github.event.client_payload.required_language || '' }}" + in workflow + ), "Queued pre-cutover payloads still supply required_language as a scalar" From 7fd571dbcdbae6acf29d8f4ee704d7ba6297e4db Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:07:12 +0900 Subject: [PATCH 41/80] fix(codeql): keep a clean dispatch scan when status publish 403s (#2028) * fix(codeql): keep a clean dispatch scan when status publish 403s opencode-agent is installed with statuses:read, so POST /statuses to a target repo returns HTTP 403 after the SARIF gate already passed. Treat the completed dispatch scan job as terminal evidence and let the required shard consume that public run on rerun instead of fail-closing a clean scan. * fix(codeql): dispatch remaining languages on workflow reruns Attempt 2 of .github#2028 skipped Dispatch current-head because the coordinator required github.run_attempt == 1, so no codeql-scan was posted. Later attempts still skip when every language already has a terminal opencode-agent verdict. * test(codeql): require paginated dispatch evidence lookup * test(codeql): require paginated dispatch job lookup * fix(codeql): paginate exact dispatch evidence * test(codeql): parse paginated gh endpoint options * style(codeql): normalize pagination fixture spacing * test(codeql): match paginated empty dispatch fixtures Signed-off-by: Seongho Bae * fix(codeql): bind dispatch fallback to live base and required run The completed-scan fallback matched only repo#PR@head plus language, so a same-head retarget could consume a predecessor-base scan and a different waiting required run could satisfy this shard. Encode live base SHA and required_run_id in the public dispatch run-name, look up that identity from the required shard, and POST the live base on later attempts. Concurrency stays repository+PR per #2008/#2009. --------- Signed-off-by: Seongho Bae --- .github/workflows/codeql-pr.yml | 56 +++- .github/workflows/codeql-scan-dispatch.yml | 9 +- tests/test_codeql_pr_workflow_contract.py | 283 +++++++++++++++++- ..._codeql_scan_dispatch_workflow_contract.py | 39 +++ 4 files changed, 368 insertions(+), 19 deletions(-) diff --git a/.github/workflows/codeql-pr.yml b/.github/workflows/codeql-pr.yml index db06b7c4ba..c21c8446df 100644 --- a/.github/workflows/codeql-pr.yml +++ b/.github/workflows/codeql-pr.yml @@ -171,10 +171,12 @@ jobs: PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} LANGUAGE: ${{ matrix.language }} RUN_ATTEMPT: ${{ github.run_attempt }} + REQUIRED_RUN_ID: ${{ github.run_id }} run: | set -euo pipefail live_pr="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}")" live_head="$(printf '%s' "$live_pr" | jq -r '.head.sha // empty')" + live_base="$(printf '%s' "$live_pr" | jq -r '.base.sha // empty')" live_state="$(printf '%s' "$live_pr" | jq -r 'if (.state | type) == "string" then .state else empty end')" if [ -z "$live_head" ] || [ -z "$live_state" ]; then echo "::error::Could not validate live pull request state before CodeQL dispatch." @@ -188,6 +190,14 @@ jobs: echo "Pull request head moved on the live open PR; a fresh dispatch will fire for the current head." exit 0 fi + if ! [[ "$live_base" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "::error::Could not validate live pull request base SHA before CodeQL verdict read." + exit 1 + fi + if ! [[ "$REQUIRED_RUN_ID" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::CodeQL shard requires a canonical current run id." + exit 1 + fi statuses="$(gh api "repos/${TARGET_REPOSITORY}/commits/${PR_HEAD_SHA}/statuses")" verdict_state="$(printf '%s' "$statuses" | jq -r --arg ctx "codeql-dispatch/${LANGUAGE}" ' @@ -208,6 +218,36 @@ jobs: exit 0 ;; esac + + expected_title="CodeQL Scan Dispatch ${TARGET_REPOSITORY}#${PR_NUMBER}@${PR_HEAD_SHA}/${live_base}/${REQUIRED_RUN_ID}" + expected_job="CodeQL dispatch scan (${LANGUAGE})" + runs_json="$(gh api --paginate --slurp "repos/ContextualWisdomLab/.github/actions/workflows/codeql-scan-dispatch.yml/runs")" + run_id="$(printf '%s' "$runs_json" | jq -r --arg title "$expected_title" --arg path ".github/workflows/codeql-scan-dispatch.yml" ' + [ + .[] | .workflow_runs[] + | select(.path == $path) + | select(.event == "repository_dispatch") + | select(.status == "completed") + | select(.display_title == $title or .name == $title) + ] + | first + | .id // empty + ')" + if [[ "$run_id" =~ ^[1-9][0-9]*$ ]]; then + jobs_json="$(gh api --paginate --slurp "repos/ContextualWisdomLab/.github/actions/runs/${run_id}/jobs")" + job_conclusion="$(printf '%s' "$jobs_json" | jq -r --arg name "$expected_job" ' + [.[] | .jobs[] | select(.name == $name)] + | if length == 1 then .[0].conclusion else empty end + ')" + case "$job_conclusion" in + success|failure) + echo "verdict=${job_conclusion}" >>"$GITHUB_OUTPUT" + echo "Found completed CodeQL dispatch scan job for ${LANGUAGE}: ${job_conclusion}." + exit 0 + ;; + esac + fi + if [ "$RUN_ATTEMPT" != "1" ]; then echo "::error::Exact CodeQL job was rerun without an authenticated terminal verdict." exit 1 @@ -251,7 +291,6 @@ jobs: always() && github.event.action != 'closed' && github.event.pull_request.state != 'closed' - && github.run_attempt == 1 && needs.detect-languages.result == 'success' && needs.detect-languages.outputs.code == 'true' runs-on: ubuntu-24.04 @@ -277,6 +316,9 @@ jobs: set -euo pipefail live_pr="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}")" live_head="$(printf '%s' "$live_pr" | jq -r '.head.sha // empty')" + live_base="$(printf '%s' "$live_pr" | jq -r '.base.sha // empty')" + live_base_ref="$(printf '%s' "$live_pr" | jq -r '.base.ref // empty')" + live_head_ref="$(printf '%s' "$live_pr" | jq -r '.head.ref // empty')" live_state="$(printf '%s' "$live_pr" | jq -r 'if (.state | type) == "string" then .state else empty end')" if [ -z "$live_head" ] || [ -z "$live_state" ]; then echo "::error::Could not validate live pull request state before CodeQL dispatch." @@ -294,6 +336,10 @@ jobs: echo "::error::CodeQL dispatch requires a canonical current run id." exit 1 fi + if ! [[ "$live_base" =~ ^[0-9a-fA-F]{40}$ ]] || [ -z "$live_base_ref" ] || [ -z "$live_head_ref" ]; then + echo "::error::Could not validate live pull request base identity before CodeQL dispatch." + exit 1 + fi include_json="$(printf '%s' "$MATRIX" | jq -c '.include // empty' 2>/dev/null || true)" if [ -z "$include_json" ] || @@ -385,10 +431,10 @@ jobs: jq -cn \ --arg target_repository "$TARGET_REPOSITORY" \ --arg pr_number "$PR_NUMBER" \ - --arg pr_base_ref "$PR_BASE_REF" \ - --arg pr_base_sha "$PR_BASE_SHA" \ - --arg pr_head_ref "$PR_HEAD_REF" \ - --arg pr_head_sha "$PR_HEAD_SHA" \ + --arg pr_base_ref "$live_base_ref" \ + --arg pr_base_sha "$live_base" \ + --arg pr_head_ref "$live_head_ref" \ + --arg pr_head_sha "$live_head" \ --argjson matrix "$pending_matrix" \ --arg required_run_id "$REQUIRED_RUN_ID" \ --argjson required_jobs "$required_jobs" \ diff --git a/.github/workflows/codeql-scan-dispatch.yml b/.github/workflows/codeql-scan-dispatch.yml index 521ceeb167..c94fdf55c2 100644 --- a/.github/workflows/codeql-scan-dispatch.yml +++ b/.github/workflows/codeql-scan-dispatch.yml @@ -16,7 +16,9 @@ run-name: >- CodeQL Scan Dispatch ${{ github.event.client_payload.target_repository || github.repository }}#${{ github.event.client_payload.pr_number || 'event' }}@${{ - github.event.client_payload.pr_head_sha || github.sha }} + github.event.client_payload.pr_head_sha || github.sha }}/${{ + github.event.client_payload.pr_base_sha || 'none' }}/${{ + github.event.client_payload.required_run_id || github.run_id }} on: repository_dispatch: @@ -503,6 +505,11 @@ jobs: exit 0 fi + if [ "$GATE_OUTCOME" = "success" ]; then + echo "::notice::Could not publish the CodeQL dispatch status after all configured credentials failed. The completed dispatch scan job remains the evidence for this head." + exit 0 + fi + echo "::error::Could not publish the CodeQL dispatch status after all configured credentials failed; the exact required job will remain failed and will not be woken with stale or missing evidence." exit 1 diff --git a/tests/test_codeql_pr_workflow_contract.py b/tests/test_codeql_pr_workflow_contract.py index 2d11ca0141..dc67eef258 100644 --- a/tests/test_codeql_pr_workflow_contract.py +++ b/tests/test_codeql_pr_workflow_contract.py @@ -87,8 +87,28 @@ def test_codeql_pr_shards_do_not_dispatch_and_coordinator_sends_the_full_matrix_ assert "needs: [detect-languages, analyze-head]" in coordinator assert "always()" in coordinator.split("\n runs-on:", 1)[0] assert "github.event.action != 'closed'" in coordinator.split("\n runs-on:", 1)[0] - assert "github.run_attempt == 1" in coordinator.split("\n runs-on:", 1)[0] + coordinator_if = coordinator.split("\n runs-on:", 1)[0] + assert "github.run_attempt == 1" not in coordinator_if assert coordinator.count("repos/ContextualWisdomLab/.github/dispatches") == 1 + + +def test_codeql_coordinator_dispatches_later_attempts_when_no_terminal_verdict() -> None: + """A rerun must still POST codeql-scan if attempt 1 never dispatched. + + Live ContextualWisdomLab/.github#2028 run 34175742278 was attempt 2. + ``github.run_attempt == 1`` skipped Dispatch current-head, so no + codeql-scan-dispatch.yml run existed and compatibility stayed pending. + The coordinator script already skips when every language has a terminal + opencode-agent verdict, so later attempts are safe. + """ + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + coordinator_if = workflow.split(" dispatch-current-head:\n", 1)[1].split( + "\n runs-on:", 1 + )[0] + coordinator = workflow.split(" dispatch-current-head:\n", 1)[1] + + assert "github.run_attempt == 1" not in coordinator_if + assert "All detected CodeQL languages already have authenticated terminal verdicts" in coordinator assert 'event_type:"codeql-scan"' in coordinator assert "required_jobs:$required_jobs" in coordinator assert "required_run_id:$required_run_id" in coordinator @@ -130,10 +150,47 @@ def test_codeql_pr_dispatch_and_release_run_blocks_are_valid_bash() -> None: DISPATCH_STEP_NAME = "Read current-head CodeQL dispatch verdict" VERDICT_STEP_NAME = "Release runner or enforce current-head CodeQL verdict" COORDINATOR_STEP_NAME = "Dispatch current-head CodeQL scan" +_TEST_HEAD_SHA = "b" * 40 +_TEST_BASE_SHA = "a" * 40 +_TEST_REQUIRED_RUN_ID = "42" + + +def _dispatch_scan_title( + *, + head_sha: str = _TEST_HEAD_SHA, + base_sha: str = _TEST_BASE_SHA, + required_run_id: str = _TEST_REQUIRED_RUN_ID, +) -> str: + """Return the immutable CodeQL dispatch run-name for one required shard.""" + return ( + "CodeQL Scan Dispatch ContextualWisdomLab/naruon#42@" + f"{head_sha}/{base_sha}/{required_run_id}" + ) + + +def _completed_dispatch_run( + *, + title: str, + run_id: int = 34173910106, +) -> dict: + """Return one completed central CodeQL dispatch workflow-run fixture.""" + return { + "id": run_id, + "event": "repository_dispatch", + "path": ".github/workflows/codeql-scan-dispatch.yml", + "status": "completed", + "display_title": title, + "name": title, + } def _run_verdict_read( - tmp_path: Path, statuses: list[dict] + tmp_path: Path, + statuses: list[dict], + *, + dispatch_runs: dict | list[dict] | None = None, + dispatch_jobs: dict | list[dict] | None = None, + run_attempt: str = "2", ) -> tuple[subprocess.CompletedProcess[str], subprocess.CompletedProcess[str]]: """Execute the real one-shot status read and verdict enforcement blocks.""" bash = shutil.which("bash") @@ -144,8 +201,12 @@ def _run_verdict_read( dispatch_script = _extract_run_block(workflow_text, DISPATCH_STEP_NAME) verdict_script = _extract_run_block(workflow_text, VERDICT_STEP_NAME) - head_sha = "b" * 40 - live_pr = {"head": {"sha": head_sha}, "state": "open"} + head_sha = _TEST_HEAD_SHA + live_pr = { + "head": {"sha": head_sha}, + "base": {"sha": _TEST_BASE_SHA}, + "state": "open", + } fake_bin = tmp_path / "bin" fake_bin.mkdir() @@ -154,9 +215,12 @@ def _run_verdict_read( "#!/usr/bin/env bash\n" "set -euo pipefail\n" 'test "$1" = api\n' - 'case "$2" in\n' + 'endpoint="${@: -1}"\n' + 'case "$endpoint" in\n' " */pulls/*) printf '%s\\n' \"$FAKE_PULL_JSON\" ;;\n" " */statuses) printf '%s\\n' \"$FAKE_STATUSES_JSON\" ;;\n" + " */codeql-scan-dispatch.yml/runs*) printf '%s\\n' \"$FAKE_DISPATCH_RUNS_JSON\" ;;\n" + " */actions/runs/*/jobs*) printf '%s\\n' \"$FAKE_DISPATCH_JOBS_JSON\" ;;\n" " *) exit 1 ;;\n" "esac\n", encoding="utf-8", @@ -169,6 +233,16 @@ def _run_verdict_read( "PATH": f"{fake_bin}:{os.environ['PATH']}", "FAKE_PULL_JSON": json.dumps(live_pr), "FAKE_STATUSES_JSON": json.dumps(statuses), + "FAKE_DISPATCH_RUNS_JSON": json.dumps( + dispatch_runs + if isinstance(dispatch_runs, list) + else [dispatch_runs if dispatch_runs is not None else {"workflow_runs": []}] + ), + "FAKE_DISPATCH_JOBS_JSON": json.dumps( + dispatch_jobs + if isinstance(dispatch_jobs, list) + else [dispatch_jobs if dispatch_jobs is not None else {"jobs": []}] + ), "GH_TOKEN": "fake-token", "TARGET_REPOSITORY": "ContextualWisdomLab/naruon", "PR_NUMBER": "42", @@ -176,10 +250,10 @@ def _run_verdict_read( "LANGUAGE": "python", "BUILD_MODE": "none", "BASE_REF": "main", - "BASE_SHA": "a" * 40, + "BASE_SHA": _TEST_BASE_SHA, "HEAD_REF": "feature", - "RUN_ATTEMPT": "2", - "REQUIRED_RUN_ID": "42", + "RUN_ATTEMPT": run_attempt, + "REQUIRED_RUN_ID": _TEST_REQUIRED_RUN_ID, "REQUIRED_JOB_ID": "43", "GITHUB_OUTPUT": str(output), } @@ -187,9 +261,16 @@ def _run_verdict_read( [bash], input=dispatch_script, text=True, capture_output=True, check=False, env=dispatch_env, timeout=60, ) - output_values = dict( - line.split("=", 1) for line in output.read_text(encoding="utf-8").splitlines() - ) + output_values = {} + if output.exists(): + output_values = dict( + line.split("=", 1) for line in output.read_text(encoding="utf-8").splitlines() + if "=" in line + ) + if "verdict" not in output_values: + return dispatch_result, subprocess.CompletedProcess( + args=[bash], returncode=1, stdout="", stderr="" + ) verdict_env = { **os.environ, "LANGUAGE": "python", @@ -248,6 +329,145 @@ def test_codeql_pr_one_shot_read_accepts_the_opencode_agent_creator(tmp_path: Pa assert "Current-head CodeQL dispatch verdict for python: success." in verdict_result.stdout +def test_codeql_pr_one_shot_read_accepts_completed_dispatch_scan_job_when_status_unpublishable( + tmp_path: Path, +) -> None: + """A completed dispatch scan job is terminal evidence when statuses:write 403s. + + Live 2026-09-08 naruon#1596 dispatch run 34173910106 scanned clean, then + POST /statuses returned HTTP 403 for opencode-agent (statuses:read only) + and github.token (cross-repo). The required shard must consume that + completed scan job instead of staying fail-closed on a missing status. + """ + head_sha = _TEST_HEAD_SHA + title = _dispatch_scan_title(head_sha=head_sha) + dispatch_result, verdict_result = _run_verdict_read( + tmp_path, + statuses=[], + dispatch_runs={"workflow_runs": [_completed_dispatch_run(title=title)]}, + dispatch_jobs={ + "jobs": [ + { + "name": "CodeQL dispatch scan (python)", + "conclusion": "success", + } + ] + }, + ) + assert dispatch_result.returncode == 0, dispatch_result.stderr + dispatch_result.stdout + assert verdict_result.returncode == 0, verdict_result.stderr + verdict_result.stdout + assert "completed CodeQL dispatch scan job for python: success" in dispatch_result.stdout + assert "Current-head CodeQL dispatch verdict for python: success." in verdict_result.stdout + + +def test_codeql_pr_finds_completed_dispatch_scan_beyond_first_results_page( + tmp_path: Path, +) -> None: + """The exact completed dispatch remains discoverable on later API pages.""" + head_sha = _TEST_HEAD_SHA + expected_title = _dispatch_scan_title(head_sha=head_sha) + dispatch_result, verdict_result = _run_verdict_read( + tmp_path, + statuses=[], + dispatch_runs=[ + {"workflow_runs": []}, + {"workflow_runs": [_completed_dispatch_run(title=expected_title)]}, + ], + dispatch_jobs=[ + {"jobs": []}, + { + "jobs": [ + { + "name": "CodeQL dispatch scan (python)", + "conclusion": "success", + } + ] + }, + ], + ) + + assert dispatch_result.returncode == 0, dispatch_result.stderr + dispatch_result.stdout + assert verdict_result.returncode == 0, verdict_result.stderr + verdict_result.stdout + assert "completed CodeQL dispatch scan job for python: success" in dispatch_result.stdout + + +def test_codeql_pr_rejects_completed_dispatch_scan_from_a_stale_base( + tmp_path: Path, +) -> None: + """Same head and language after a base retarget must not reuse the prior scan. + + A PR can keep its head SHA while the base moves. The native handler already + binds receipts to the live base SHA; the required shard must not accept a + completed dispatch whose run-name still names the predecessor base. + """ + stale_title = _dispatch_scan_title(base_sha="c" * 40) + dispatch_result, _verdict_result = _run_verdict_read( + tmp_path, + statuses=[], + dispatch_runs={"workflow_runs": [_completed_dispatch_run(title=stale_title)]}, + dispatch_jobs={ + "jobs": [ + { + "name": "CodeQL dispatch scan (python)", + "conclusion": "success", + } + ] + }, + ) + + assert dispatch_result.returncode == 1, dispatch_result.stderr + dispatch_result.stdout + assert "without an authenticated terminal verdict" in dispatch_result.stdout + assert "completed CodeQL dispatch scan job for python: success" not in dispatch_result.stdout + + +def test_codeql_pr_rejects_completed_dispatch_scan_from_a_different_required_run( + tmp_path: Path, +) -> None: + """A same-PR/head/language scan for another required run cannot wake this shard. + + Language plus repository/PR/head is not enough: each waiting required job + lives in one required-workflow run. Binding required_run_id in the + dispatch run-name, together with the language job name, is the job + identity the shard can observe without reading client_payload. + """ + other_run_title = _dispatch_scan_title(required_run_id="99") + dispatch_result, _verdict_result = _run_verdict_read( + tmp_path, + statuses=[], + dispatch_runs={ + "workflow_runs": [_completed_dispatch_run(title=other_run_title)] + }, + dispatch_jobs={ + "jobs": [ + { + "name": "CodeQL dispatch scan (python)", + "conclusion": "success", + } + ] + }, + ) + + assert dispatch_result.returncode == 1, dispatch_result.stderr + dispatch_result.stdout + assert "without an authenticated terminal verdict" in dispatch_result.stdout + assert "completed CodeQL dispatch scan job for python: success" not in dispatch_result.stdout + + +def test_codeql_pr_fallback_binds_live_base_and_required_run_identity() -> None: + """The required shard looks up the public dispatch run by immutable identity.""" + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + shard = workflow.split(" analyze-head:\n", 1)[1].split( + " dispatch-current-head:\n", 1 + )[0] + + assert "REQUIRED_RUN_ID: ${{ github.run_id }}" in shard + assert 'live_base="$(printf' in shard + assert ( + 'expected_title="CodeQL Scan Dispatch ${TARGET_REPOSITORY}#${PR_NUMBER}' + '@${PR_HEAD_SHA}/${live_base}/${REQUIRED_RUN_ID}"' + ) in shard + assert "Could not validate live pull request base SHA before CodeQL verdict read." in shard + + def test_codeql_action_steps_use_one_version_per_workflow() -> None: """Prevent CodeQL init/analyze version splits from failing the scheduled scan.""" workflow = (REPO_ROOT / ".github/workflows/scheduled-security-scan.yml").read_text( @@ -319,9 +539,12 @@ def test_codeql_pr_attempt_one_without_verdict_fails_pending_without_dispatch( ' printf \'%s\\n\' "$4" >>"$FAKE_POST_LOG"\n' " exit 0\n" "fi\n" - 'case "$2" in\n' + 'endpoint="${@: -1}"\n' + 'case "$endpoint" in\n' " */pulls/*) printf '%s\\n' \"$FAKE_PULL_JSON\" ;;\n" " */statuses) printf '%s\\n' \"$FAKE_STATUSES_JSON\" ;;\n" + " */codeql-scan-dispatch.yml/runs*) printf '%s\\n' \"$FAKE_DISPATCH_RUNS_JSON\" ;;\n" + " */actions/runs/*/jobs*) printf '%s\\n' \"$FAKE_DISPATCH_JOBS_JSON\" ;;\n" " *) exit 1 ;;\n" "esac\n", encoding="utf-8", @@ -331,8 +554,16 @@ def test_codeql_pr_attempt_one_without_verdict_fails_pending_without_dispatch( env = { **os.environ, "PATH": f"{fake_bin}:{os.environ['PATH']}", - "FAKE_PULL_JSON": json.dumps({"head": {"sha": head_sha}, "state": "open"}), + "FAKE_PULL_JSON": json.dumps( + { + "head": {"sha": head_sha}, + "base": {"sha": _TEST_BASE_SHA}, + "state": "open", + } + ), "FAKE_STATUSES_JSON": json.dumps([]), + "FAKE_DISPATCH_RUNS_JSON": json.dumps([{"workflow_runs": []}]), + "FAKE_DISPATCH_JOBS_JSON": json.dumps([{"jobs": []}]), "FAKE_POST_LOG": str(post_log), "GH_TOKEN": "fake-token", "TARGET_REPOSITORY": "ContextualWisdomLab/naruon", @@ -593,6 +824,32 @@ def test_codeql_coordinator_fails_closed_when_a_shard_job_id_is_missing( assert not post_log.exists() +def test_codeql_coordinator_dispatches_the_live_base_after_a_same_head_retarget( + tmp_path: Path, +) -> None: + """A retargeted PR must dispatch against the live base, not the event snapshot.""" + live_base = "c" * 40 + result, post_log, post_body = _run_coordinator( + tmp_path, + pull={ + "state": "open", + "head": {"sha": "b" * 40, "ref": "feature"}, + "base": {"sha": live_base, "ref": "release"}, + }, + env_overrides={"PR_BASE_SHA": "a" * 40, "PR_BASE_REF": "main"}, + ) + + assert result.returncode == 0, result.stderr + result.stdout + assert post_log.read_text(encoding="utf-8").splitlines() == [ + "repos/ContextualWisdomLab/.github/dispatches" + ] + client = json.loads(post_body.read_text(encoding="utf-8"))["client_payload"] + assert client["pr_base_sha"] == live_base + assert client["pr_base_ref"] == "release" + assert client["pr_head_sha"] == "b" * 40 + assert client["required_run_id"] == "99" + + def test_codeql_coordinator_does_not_dispatch_a_closed_or_stale_pull_request( tmp_path: Path, ) -> None: diff --git a/tests/test_codeql_scan_dispatch_workflow_contract.py b/tests/test_codeql_scan_dispatch_workflow_contract.py index dea1326494..dd30c8506d 100644 --- a/tests/test_codeql_scan_dispatch_workflow_contract.py +++ b/tests/test_codeql_scan_dispatch_workflow_contract.py @@ -505,6 +505,45 @@ def test_codeql_scan_dispatch_is_not_in_the_required_workflow_ruleset_scope(): assert ".github/workflows/codeql-scan-dispatch.yml" not in required_paths +def test_codeql_scan_dispatch_run_name_binds_base_and_required_run() -> None: + """Public run identity includes base SHA and required run id without changing concurrency. + + The required shard cannot read client_payload. Encoding those fields in + run-name lets it reject a same-head retarget or a different waiting + required run. The #2008/#2009 group stays repository+PR so a newer HEAD + of the same pull request still cancels its predecessor. + """ + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + header = workflow.split("\non:", 1)[0] + group_value = workflow_level_concurrency_group(workflow) + + assert "github.event.client_payload.pr_head_sha" in header + assert "github.event.client_payload.pr_base_sha" in header + assert "github.event.client_payload.required_run_id" in header + assert "github.event.client_payload.pr_base_sha" not in group_value + assert "github.event.client_payload.required_run_id" not in group_value + assert "github.event.client_payload.target_repository" in group_value + assert "github.event.client_payload.pr_number" in group_value + + +def test_dispatch_publish_keeps_successful_scan_when_status_write_is_denied() -> None: + """A clean SARIF gate must not fail the handler solely because POST /statuses 403s. + + opencode-agent is installed with statuses:read. Cross-repo github.token cannot + write naruon commit statuses. The completed scan job is the remaining evidence. + """ + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + publish = workflow.split(" - name: Publish CodeQL dispatch status\n", 1)[1].split( + "\n - name: Wake exact CodeQL required job\n", 1 + )[0] + + assert "GATE_OUTCOME" in publish + assert 'if [ "$GATE_OUTCOME" = "success" ]; then' in publish + assert "completed dispatch scan job remains the evidence" in publish + assert "continue-on-error:" not in publish + assert "cancel-in-progress: true" not in publish + + def test_dispatch_wakes_only_the_exact_failed_codeql_job() -> None: workflow = WORKFLOW_PATH.read_text(encoding="utf-8") wake = workflow.split(" - name: Wake exact CodeQL required job\n", 1)[1].split( From e99509d8033b4b915d41aadf445d1e4bf50dd165 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 13:18:31 +0900 Subject: [PATCH 42/80] test(strix): preserve hash-only folded group content --- tests/test_required_workflow_queue_contract.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index d3001766d2..b13556e43d 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -527,6 +527,24 @@ def test_concurrency_group_slice_keeps_a_hash_that_is_not_a_comment() -> None: assert "github.run_id" in folded_value +def test_concurrency_group_slice_preserves_hash_only_folded_line() -> None: + """A hash-only folded-block line is scalar content, not a YAML comment.""" + folded = textwrap.dedent( + """\ + concurrency: + group: >- + prefix + # literal + suffix + cancel-in-progress: true + permissions: + contents: read + """ + ) + + assert workflow_level_concurrency_group(folded) == "prefix # literal suffix" + + def test_concurrency_group_slice_reads_a_folded_multi_line_key() -> None: """The real key is a folded block, so the slice must join its continuation lines.""" folded = textwrap.dedent( From a397914da6f0becd9b83a59086702ce0f6261109 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 13:19:10 +0900 Subject: [PATCH 43/80] fix(strix): retain folded scalar hash lines --- tests/test_required_workflow_queue_contract.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index b13556e43d..51d45bf523 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -75,12 +75,17 @@ def workflow_level_concurrency_group(workflow: str) -> str: raise AssertionError("workflow declares no workflow-level concurrency block") value: list[str] = [] collecting = False + group_indent = 0 for line in block_match.group("body").splitlines(): - if line.strip().startswith("#"): + line_indent = len(line) - len(line.lstrip()) + if line.strip().startswith("#") and ( + not collecting or line_indent <= group_indent + ): continue if not collecting: if re.match(r"^\s*group:", line): collecting = True + group_indent = line_indent value.append(_strip_yaml_inline_comment(line.split("group:", 1)[1])) continue if re.match(r"^\s*[A-Za-z][\w-]*:", line): From 330741ff25208d45b9884d1206977460d88370e0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 13:22:55 +0900 Subject: [PATCH 44/80] test(strix): cover folded scalar hash lines --- tests/test_required_workflow_queue_contract.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 51d45bf523..a9d3355241 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -531,6 +531,23 @@ def test_concurrency_group_slice_keeps_a_hash_that_is_not_a_comment() -> None: assert "#${{" in folded_value assert "github.run_id" in folded_value + folded_hash_line = textwrap.dedent( + """\ + concurrency: + group: >- + prefix + # literal + suffix + cancel-in-progress: true + permissions: + contents: read + """ + ) + assert ( + workflow_level_concurrency_group(folded_hash_line) + == "prefix # literal suffix" + ) + def test_concurrency_group_slice_preserves_hash_only_folded_line() -> None: """A hash-only folded-block line is scalar content, not a YAML comment.""" From 28e10b5a3a6045633e536ea63f27ae06a1fd2dcd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 13:25:25 +0900 Subject: [PATCH 45/80] test(strix): keep one folded hash regression --- tests/test_required_workflow_queue_contract.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index a9d3355241..19fe6b0f7f 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -549,24 +549,6 @@ def test_concurrency_group_slice_keeps_a_hash_that_is_not_a_comment() -> None: ) -def test_concurrency_group_slice_preserves_hash_only_folded_line() -> None: - """A hash-only folded-block line is scalar content, not a YAML comment.""" - folded = textwrap.dedent( - """\ - concurrency: - group: >- - prefix - # literal - suffix - cancel-in-progress: true - permissions: - contents: read - """ - ) - - assert workflow_level_concurrency_group(folded) == "prefix # literal suffix" - - def test_concurrency_group_slice_reads_a_folded_multi_line_key() -> None: """The real key is a folded block, so the slice must join its continuation lines.""" folded = textwrap.dedent( From 6f4ca4700a4015eeb55f2b59f4f24bef38b726c5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 12:53:42 +0900 Subject: [PATCH 46/80] fix(ci): preserve bounded sidecar request ids Retain only exact 32-character lowercase hexadecimal request identifiers on provider attempt, backoff, and failed diagnostics while continuing to omit raw provider error text. Co-authored-by: Codex Signed-off-by: Seongho Bae --- AGENTS.md | 4 ++ CLAUDE.md | 3 ++ ..._contextual_orchestrator_sidecar_stream.py | 10 +++-- ...l_orchestrator_review_runtime_preflight.py | 41 +++++++++++++++++++ 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e955f8b36a..978307f8f2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,6 +30,10 @@ see [`docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md`](docs/adr/0003 2026-08-30 amendment and its 2026-08-31 correction, which retracts an earlier false claim of explicit owner direction and records the resulting availability risk as open and unreviewed, not accepted. +Sidecar diagnostics may retain only a server-generated `request_id` matching +exactly 32 lowercase hexadecimal characters. Keep free-form provider errors +omitted; malformed, uppercase, short, long, or otherwise unbounded identifiers +must not pass the sanitizer. The materialization contract is also covered by [`docs/doctoring/exact-artifact-sbom-attestation.md`](docs/doctoring/exact-artifact-sbom-attestation.md). ## Actions queue and protected-merge procedure diff --git a/CLAUDE.md b/CLAUDE.md index 30db1fc23b..74f0b136e0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -146,6 +146,9 @@ repeatable compile command. `contextual-orchestrator/orchestrator/free`). Keep the ZDR-first policy and the exact-head/vendoring pins in `scripts/ci/zdr_policy.py` and `scripts/ci/contextual_orchestrator_review_sidecar.sh` in sync with their contract tests. + Sanitized route diagnostics preserve only server request IDs that are exactly + 32 lowercase hexadecimal characters; never widen that field to arbitrary text + or re-emit provider error messages. - **`pull_request_target` trust boundary.** The required review workflows run the *base branch's* trusted scripts. A PR that edits the trusted review workflows can fail its own checks until the base branch catches up; a same-head manual `workflow_dispatch` Strix run may supply review evidence diff --git a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py index e055513954..31a9cd2d2c 100644 --- a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py +++ b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py @@ -28,6 +28,7 @@ _MODEL_ID = r"[A-Za-z0-9_./:-]+" _ERROR_TYPE = r"[A-Za-z_][A-Za-z0-9_.]*" _NUMBER = r"\d+(?:\.\d+)?" +_REQUEST_ID = r"[0-9a-f]{32}" # contextual_orchestrator/orchestrator.py templates at the vendored pin. Every # field is a bounded identifier or number; ``error_message`` is free text and is # deliberately excluded from the match so it can never be re-emitted. @@ -36,10 +37,13 @@ _ORCHESTRATOR_EVENTS = tuple( re.compile(pattern) for pattern in ( - rf"^provider_attempt agent_id={_AGENT_ID} model={_MODEL_ID} attempt=\d+/\d+$", + rf"^provider_attempt agent_id={_AGENT_ID} model={_MODEL_ID} attempt=\d+/\d+" + rf"(?: request_id={_REQUEST_ID})?$", rf"^provider_attempt_failed agent_id={_AGENT_ID} model={_MODEL_ID} attempt=\d+ " - rf"error_type={_ERROR_TYPE} transient=(?:True|False)(?= error_message=)", - rf"^provider_backoff agent_id={_AGENT_ID} attempt=\d+ delay_seconds={_NUMBER}$", + rf"error_type={_ERROR_TYPE} transient=(?:True|False)" + rf"(?: request_id={_REQUEST_ID})?(?= error_message=)", + rf"^provider_backoff agent_id={_AGENT_ID} attempt=\d+ delay_seconds={_NUMBER}" + rf"(?: request_id={_REQUEST_ID})?$", rf"^provider_exhausted agent_id={_AGENT_ID} model={_MODEL_ID} attempts=\d+ " rf"final_error_type={_ERROR_TYPE}$", rf"^provider_rejected_permanent agent_id={_AGENT_ID} model={_MODEL_ID} attempts=\d+ " diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index 7f92162097..3eb1f29125 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -1769,6 +1769,32 @@ def test_sidecar_stream_sanitizer_admits_orchestrator_route_events() -> None: assert sanitize_line( "provider_backoff agent_id=nvidia_nim_x attempt=1 delay_seconds=0.500" ) == "provider_backoff agent_id=nvidia_nim_x attempt=1 delay_seconds=0.500" + request_id = "0123456789abcdef0123456789abcdef" + assert sanitize_line( + "provider_attempt agent_id=nvidia_nim_x model=m/x attempt=1/3 " + f"request_id={request_id}" + ) == ( + "provider_attempt agent_id=nvidia_nim_x model=m/x attempt=1/3 " + f"request_id={request_id}" + ) + assert sanitize_line( + "provider_backoff agent_id=nvidia_nim_x attempt=1 delay_seconds=0.500 " + f"request_id={request_id}" + ) == ( + "provider_backoff agent_id=nvidia_nim_x attempt=1 delay_seconds=0.500 " + f"request_id={request_id}" + ) + request_failed = sanitize_line( + "provider_attempt_failed agent_id=nvidia_nim_x model=m/x attempt=1 " + "error_type=HTTPError transient=True " + f"request_id={request_id} error_message=Bearer sk-secret" + ) + assert request_failed == ( + "provider_attempt_failed agent_id=nvidia_nim_x model=m/x attempt=1 " + "error_type=HTTPError transient=True " + f"request_id={request_id} error_message=" + ) + assert "sk-secret" not in request_failed assert sanitize_line( "INFO:contextual_orchestrator.orchestrator:provider_no_retry_budget agent_id=bytez_a " "model=m/x attempts=1 final_error_type=InvalidChatResponse transient=False" @@ -1798,6 +1824,21 @@ def test_sidecar_stream_sanitizer_admits_orchestrator_route_events() -> None: "provider_attempt_failed agent_id=nvidia_nim_x model=m/x attempt=1 error_type=E transient=False" ) is None assert sanitize_line(f"DEBUG:contextual_orchestrator.orchestrator:{secret}") is None + for invalid_request_id in ( + "0123456789abcdef0123456789abcde", + "0123456789abcdef0123456789abcdef0", + "0123456789ABCDEF0123456789ABCDEF", + "not-a-request-id", + ): + assert sanitize_line( + "provider_attempt agent_id=nvidia_nim_x model=m/x attempt=1/3 " + f"request_id={invalid_request_id}" + ) is None + assert sanitize_line( + "provider_attempt_failed agent_id=nvidia_nim_x model=m/x attempt=1 " + "error_type=HTTPError transient=True " + f"request_id={invalid_request_id} error_message=raw-error" + ) is None def test_sidecar_stream_sanitizer_matches_real_formatter_output() -> None: From 433bc22aa6fc7c3af0ba46599730b190e314f843 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 13:01:24 +0900 Subject: [PATCH 47/80] fix(ci): preserve complete request correlation --- ..._contextual_orchestrator_sidecar_stream.py | 19 +++++++++++++------ ...l_orchestrator_review_runtime_preflight.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py index 31a9cd2d2c..ad18447e3c 100644 --- a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py +++ b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py @@ -7,9 +7,11 @@ import sys +_REQUEST_ID = r"[0-9a-f]{32}" _REQUEST_FAILED = re.compile( r"request_failed status=(?P[1-5][0-9]{2}) " - r"code=(?P[A-Za-z0-9_.-]{1,64})" + r"code=(?P[A-Za-z0-9_.-]{1,64})(?![A-Za-z0-9_.-])" + rf"(?: request_id=(?P{_REQUEST_ID})(?=$|\s))?(?! request_id=)" ) _PROVIDER_DISCOVERY_FAILED = re.compile( r"provider_discovery_failed provider=(?P[a-z][a-z0-9_]{0,63}) " @@ -28,7 +30,6 @@ _MODEL_ID = r"[A-Za-z0-9_./:-]+" _ERROR_TYPE = r"[A-Za-z_][A-Za-z0-9_.]*" _NUMBER = r"\d+(?:\.\d+)?" -_REQUEST_ID = r"[0-9a-f]{32}" # contextual_orchestrator/orchestrator.py templates at the vendored pin. Every # field is a bounded identifier or number; ``error_message`` is free text and is # deliberately excluded from the match so it can never be re-emitted. @@ -45,11 +46,15 @@ rf"^provider_backoff agent_id={_AGENT_ID} attempt=\d+ delay_seconds={_NUMBER}" rf"(?: request_id={_REQUEST_ID})?$", rf"^provider_exhausted agent_id={_AGENT_ID} model={_MODEL_ID} attempts=\d+ " - rf"final_error_type={_ERROR_TYPE}$", + rf"final_error_type={_ERROR_TYPE}(?: request_id={_REQUEST_ID})?$", rf"^provider_rejected_permanent agent_id={_AGENT_ID} model={_MODEL_ID} attempts=\d+ " - rf"final_error_type={_ERROR_TYPE}$", + rf"final_error_type={_ERROR_TYPE}(?: request_id={_REQUEST_ID})?$", rf"^provider_no_retry_budget agent_id={_AGENT_ID} model={_MODEL_ID} attempts=\d+ " - rf"final_error_type={_ERROR_TYPE} transient=(?:True|False)$", + rf"final_error_type={_ERROR_TYPE} transient=(?:True|False)" + rf"(?: request_id={_REQUEST_ID})?$", + rf"^provider_one_shot_call_failed agent_id={_AGENT_ID} model={_MODEL_ID} attempts=\d+ " + rf"final_error_type={_ERROR_TYPE} transient=(?:True|False)" + rf"(?: request_id={_REQUEST_ID})?$", rf"^circuit_failure agent_id={_AGENT_ID} failures={_NUMBER} threshold=\d+$", rf"^circuit_opened agent_id={_AGENT_ID} failures={_NUMBER} threshold=\d+ reset_seconds={_NUMBER}$", rf"^circuit_reset agent_id={_AGENT_ID}$", @@ -128,10 +133,12 @@ def sanitize_line(line: str) -> str | None: stripped = line.strip() request_failed = _REQUEST_FAILED.search(stripped) if request_failed is not None: - return ( + summary = ( f"request_failed status={request_failed.group('status')} " f"code={request_failed.group('code')}" ) + request_id = request_failed.group("request_id") + return f"{summary} request_id={request_id}" if request_id else summary provider_discovery_failed = _PROVIDER_DISCOVERY_FAILED.search(stripped) if provider_discovery_failed is not None: return ( diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index 3eb1f29125..2241209d26 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -1784,6 +1784,14 @@ def test_sidecar_stream_sanitizer_admits_orchestrator_route_events() -> None: "provider_backoff agent_id=nvidia_nim_x attempt=1 delay_seconds=0.500 " f"request_id={request_id}" ) + for event in ( + "provider_exhausted agent_id=nvidia_nim_x model=m/x attempts=2 final_error_type=HTTPError", + "provider_rejected_permanent agent_id=nvidia_nim_x model=m/x attempts=1 final_error_type=ValueError", + "provider_no_retry_budget agent_id=nvidia_nim_x model=m/x attempts=1 final_error_type=HTTPError transient=False", + "provider_one_shot_call_failed agent_id=nvidia_nim_x model=m/x attempts=1 final_error_type=HTTPError transient=False", + ): + correlated_event = f"{event} request_id={request_id}" + assert sanitize_line(correlated_event) == correlated_event request_failed = sanitize_line( "provider_attempt_failed agent_id=nvidia_nim_x model=m/x attempt=1 " "error_type=HTTPError transient=True " @@ -1795,6 +1803,9 @@ def test_sidecar_stream_sanitizer_admits_orchestrator_route_events() -> None: f"request_id={request_id} error_message=" ) assert "sk-secret" not in request_failed + assert sanitize_line( + f"request_failed status=502 code=provider_error request_id={request_id}" + ) == f"request_failed status=502 code=provider_error request_id={request_id}" assert sanitize_line( "INFO:contextual_orchestrator.orchestrator:provider_no_retry_budget agent_id=bytez_a " "model=m/x attempts=1 final_error_type=InvalidChatResponse transient=False" @@ -1839,6 +1850,9 @@ def test_sidecar_stream_sanitizer_admits_orchestrator_route_events() -> None: "error_type=HTTPError transient=True " f"request_id={invalid_request_id} error_message=raw-error" ) is None + assert sanitize_line( + f"request_failed status=502 code=provider_error request_id={invalid_request_id}" + ) is None def test_sidecar_stream_sanitizer_matches_real_formatter_output() -> None: From 4a0125bf9f50d4d26355249011df03c3735b3abc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 13:26:49 +0900 Subject: [PATCH 48/80] fix(ci): preserve bounded HTTP request correlation --- AGENTS.md | 3 +++ CLAUDE.md | 3 +++ ..._contextual_orchestrator_sidecar_stream.py | 25 ++++++++++++++++- ...l_orchestrator_review_runtime_preflight.py | 27 +++++++++++++++++++ 4 files changed, 57 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 452de4e884..0972af51c5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,6 +35,9 @@ exactly 32 lowercase hexadecimal characters, plus the producer's explicit `-` or `` marker where that event contract permits it. Keep free-form provider errors omitted; malformed, uppercase, short, long, or otherwise unbounded identifiers must not pass the sanitizer. +HTTP success summaries are narrower still: preserve correlation only for the +review sidecar's fixed health, chat-completions, and responses paths. Never +allowlist arbitrary request paths merely because the producer stripped queries. The materialization contract is also covered by [`docs/doctoring/exact-artifact-sbom-attestation.md`](docs/doctoring/exact-artifact-sbom-attestation.md). ## Actions queue and protected-merge procedure diff --git a/CLAUDE.md b/CLAUDE.md index 962ebe2b3b..7dde78f5d2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -150,6 +150,9 @@ repeatable compile command. 32 lowercase hexadecimal characters, plus an event contract's explicit `-` or `` marker; never widen that field to arbitrary text or re-emit provider error messages. + HTTP success correlation is limited to the review sidecar's fixed health, + chat-completions, and responses paths; query stripping alone does not make an + arbitrary request path safe for CI artifacts. - **`pull_request_target` trust boundary.** The required review workflows run the *base branch's* trusted scripts. A PR that edits the trusted review workflows can fail its own checks until the base branch catches up; a same-head manual `workflow_dispatch` Strix run may supply review evidence diff --git a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py index 464dd55fe3..57f33aad2f 100644 --- a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py +++ b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py @@ -9,6 +9,7 @@ _REQUEST_ID = r"[0-9a-f]{32}" _PROVIDER_REQUEST_ID = rf"(?:{_REQUEST_ID}|-)" +_NUMBER = r"\d+(?:\.\d+)?" _REQUEST_FAILED = re.compile( r"^(?:(?:[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2},[0-9]{3} )?" r"(?:DEBUG|INFO|WARNING|ERROR)[: ]contextual_orchestrator\.server[: ])?" @@ -17,6 +18,16 @@ rf"(?: request_id=(?P{_REQUEST_ID}|)(?=$|\s))?" r"(?! request_id=)" ) +_HTTP_REQUEST = re.compile( + r"^(?:(?:[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2},[0-9]{3} )?" + r"(?:DEBUG|INFO|WARNING|ERROR)[: ]contextual_orchestrator\.server[: ])?" + r"http_request method=(?PGET|POST) " + r"path=(?P/healthz|/v1/chat/completions|/v1/responses|-) " + r"status=(?P[1-5][0-9]{2}|-) " + rf"latency_ms=(?P{_NUMBER}) " + r"session_id_hash=(?P[0-9a-f]{64}|-) " + rf"request_id=(?P{_REQUEST_ID})$" +) _PROVIDER_DISCOVERY_FAILED = re.compile( r"provider_discovery_failed provider=(?P[a-z][a-z0-9_]{0,63}) " r"code=(?P[A-Za-z0-9_.-]{1,64})" @@ -33,7 +44,6 @@ _AGENT_ID = r"[a-z][a-z0-9_]*" _MODEL_ID = r"[A-Za-z0-9_./:-]+" _ERROR_TYPE = r"[A-Za-z_][A-Za-z0-9_.]*" -_NUMBER = r"\d+(?:\.\d+)?" # contextual_orchestrator/orchestrator.py templates at the vendored pin. Every # field is a bounded identifier or number; ``error_message`` is free text and is # deliberately excluded from the match so it can never be re-emitted. @@ -148,6 +158,19 @@ def sanitize_line(line: str) -> str | None: if request_id is not None: summary += f" request_id={request_id}" return summary + http_request = _HTTP_REQUEST.match(stripped) + if http_request is not None: + return " ".join( + ( + "http_request", + f"method={http_request.group('method')}", + f"path={http_request.group('path')}", + f"status={http_request.group('status')}", + f"latency_ms={http_request.group('latency')}", + f"session_id_hash={http_request.group('session_id_hash')}", + f"request_id={http_request.group('request_id')}", + ) + ) provider_discovery_failed = _PROVIDER_DISCOVERY_FAILED.search(stripped) if provider_discovery_failed is not None: return ( diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index 873ea64815..0b3e38cb6c 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -1733,6 +1733,33 @@ def test_sidecar_stream_sanitizer_allowlists_only_bounded_diagnostics() -> None: assert sanitize_line("provider response sk-secret") is None +def test_sidecar_stream_sanitizer_preserves_bounded_http_request_identity() -> None: + """Review endpoints keep safe success correlation without arbitrary URL data.""" + sanitize_line = _load_sanitizer()["sanitize_line"] + request_id = "0123456789abcdef0123456789abcdef" + session_hash = "ab" * 32 + event = ( + "http_request method=POST path=/v1/chat/completions status=200 " + f"latency_ms=125.2 session_id_hash={session_hash} request_id={request_id}" + ) + assert sanitize_line(event) == event + assert sanitize_line(f"INFO:contextual_orchestrator.server:{event}") == event + assert sanitize_line( + "http_request method=GET path=/healthz status=200 latency_ms=0.4 " + f"session_id_hash=- request_id={request_id}" + ) == ( + "http_request method=GET path=/healthz status=200 latency_ms=0.4 " + f"session_id_hash=- request_id={request_id}" + ) + for unsafe_event in ( + event.replace("/v1/chat/completions", "/v1/files/private-name"), + event.replace(request_id, "A" * 32), + event.replace(session_hash, "ab" * 31), + event + " token=sk-secret", + ): + assert sanitize_line(unsafe_event) is None + + def test_sidecar_stream_sanitizer_admits_orchestrator_route_events() -> None: """Per-route attempt, retry-budget, and circuit events survive with bounded fields only. From 8421fea257c800b6e06ab95473bc7dc52dd4806d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 16:10:01 +0900 Subject: [PATCH 49/80] fix(opencode): accept proven Python VCS floors Signed-off-by: Seongho Bae Commit-Message-Assisted-by: Codex GPT-6 --- .../opencode-exact-vcs-dependency-evidence.md | 9 ++++- .../materialize_base_python_requirements.py | 10 +++++ tests/test_uv_export_isolation_contract.py | 37 +++++++++++++++++++ 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/docs/doctoring/opencode-exact-vcs-dependency-evidence.md b/docs/doctoring/opencode-exact-vcs-dependency-evidence.md index a9b26ae474..45a8831b6e 100644 --- a/docs/doctoring/opencode-exact-vcs-dependency-evidence.md +++ b/docs/doctoring/opencode-exact-vcs-dependency-evidence.md @@ -37,10 +37,15 @@ product's current-head tests passing. hook, or dependency lifecycle script runs while the network is available. - The source repository must be publicly fetchable without credentials, expose the normalized top-level import package directly or under `src`, and remain a - pure-Python leaf dependency. Private repositories, environment-marked VCS - requirements, namespace/layout aliases, installed-distribution metadata, + pure-Python leaf dependency. Private repositories, namespace/layout aliases, + installed-distribution metadata, entry points, compiled extensions, and registry packages that require the VCS distribution fail closed instead of expanding the secret-free build boundary. + An environment marker is accepted only when it is a single + `python_full_version` lower bound already met by the fixed Python 3.14 coverage + image. False or more expressive markers remain unsupported, so erasing the + proven-true lower bound cannot expose a source that the coverage interpreter + would omit. - The checkout roots and path file are explicitly world-readable so the later networkless coverage container can run as UID 65532 independently of the image builder's umask. diff --git a/scripts/ci/materialize_base_python_requirements.py b/scripts/ci/materialize_base_python_requirements.py index a052123547..3ddc126604 100755 --- a/scripts/ci/materialize_base_python_requirements.py +++ b/scripts/ci/materialize_base_python_requirements.py @@ -42,7 +42,11 @@ r"git\+https://github\.com/ContextualWisdomLab/" r"(?P[A-Za-z0-9_.-]{1,100})\.git@" r"(?P[0-9a-fA-F]{40})" + r"(?:\s*;\s*python_full_version\s*>=\s*'" + r"(?P[0-9]+)\." + r"(?P[0-9]+)')?" ) +TRUSTED_COVERAGE_PYTHON_MAJOR_MINOR = (3, 14) UV_EXPORT_TIMEOUT_SECONDS = 120 TRUSTED_UV_VERSION = "0.12.1" TRUSTED_UV_TARGET_TRIPLE = "x86_64-unknown-linux-gnu" @@ -304,6 +308,12 @@ def _partition_uv_export(content: bytes) -> tuple[bytes, list[dict[str, str]]]: match = UV_EXACT_ORG_VCS_RE.fullmatch(line) if match is None: raise ValueError("uv export contains an unsupported dependency line") + minimum_python_major = match.group("minimum_python_major") + minimum_python_minor = match.group("minimum_python_minor") + if minimum_python_major is not None and ( + int(minimum_python_major), int(minimum_python_minor) + ) > TRUSTED_COVERAGE_PYTHON_MAJOR_MINOR: + raise ValueError("uv export contains an unsupported dependency line") dependency = { "package": match.group("package"), "import_name": re.sub( diff --git a/tests/test_uv_export_isolation_contract.py b/tests/test_uv_export_isolation_contract.py index 76b72fdc7f..0dd76f9d38 100644 --- a/tests/test_uv_export_isolation_contract.py +++ b/tests/test_uv_export_isolation_contract.py @@ -118,6 +118,37 @@ def test_uv_export_partitions_hashes_and_exact_organization_vcs_sources() -> Non ] +def test_uv_export_accepts_vcs_source_for_proven_coverage_python_floor() -> None: + """A lower bound already met by the fixed coverage image is safe to erase.""" + requirement = ( + "fast-mlsirm @ git+https://github.com/ContextualWisdomLab/fast-mlsirm.git@" + + "a" * 40 + + " ; python_full_version >= '3.12'\n" + ) + + registry, vcs_sources = materializer._partition_uv_export(requirement.encode()) + + assert registry == b"" + assert vcs_sources == [ + { + "package": "fast-mlsirm", + "import_name": "fast_mlsirm", + "repository": "fast-mlsirm", + "commit": "a" * 40, + } + ] + + +def test_vcs_marker_floor_is_bound_to_the_fixed_coverage_python_image() -> None: + """The marker proof must move with the coverage interpreter version.""" + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) + + assert materializer.TRUSTED_COVERAGE_PYTHON_MAJOR_MINOR == (3, 14) + assert "FROM docker.io/library/python:3.14-slim@sha256:" in workflow + + @pytest.mark.parametrize( "requirement", [ @@ -127,6 +158,12 @@ def test_uv_export_partitions_hashes_and_exact_organization_vcs_sources() -> Non "demo @ git+https://github.com/ContextualWisdomLab/demo.git@" + "a" * 40 + "#subdirectory=python", + "demo @ git+https://github.com/ContextualWisdomLab/demo.git@" + + "a" * 40 + + " ; python_full_version >= '3.15'", + "demo @ git+https://github.com/ContextualWisdomLab/demo.git@" + + "a" * 40 + + " ; sys_platform == 'linux'", ], ) def test_uv_export_rejects_unbounded_vcs_sources(requirement: str) -> None: From d31a037746e9e31eefd5331dd6aede5341fbfcbc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 11 Sep 2026 13:54:22 +0900 Subject: [PATCH 50/80] docs(queue): mark retired org sweep as historical --- docs/doctoring/org-queue-sweep-rotation.md | 19 ++++++++++++++++++- ..._org_queue_sweep_documentation_contract.py | 17 +++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 tests/test_org_queue_sweep_documentation_contract.py diff --git a/docs/doctoring/org-queue-sweep-rotation.md b/docs/doctoring/org-queue-sweep-rotation.md index 03784d0b7e..51a640443a 100644 --- a/docs/doctoring/org-queue-sweep-rotation.md +++ b/docs/doctoring/org-queue-sweep-rotation.md @@ -1,8 +1,16 @@ # Org-queue-sweep review-dispatch rotation +> **Superseded for queue hygiene.** The organization sweep no longer inventories +> or cancels repository-wide queued/in-progress Actions runs. That duplicate +> queue-hygiene path was removed by `.github#1878` (`1b65dbc35e7183722ad77894e2d80b39993be90d`), +> and current-head coalescing was later integrated into the merge scheduler. +> The rotation material below remains historical evidence for the former +> review-dispatch fairness mechanism, not a description of current stale-run +> ownership. + ## Problem -`org-queue-sweep` in `pr-review-merge-scheduler.yml` walks every organization +Historically, `org-queue-sweep` in `pr-review-merge-scheduler.yml` walked every organization repository once per 15-minute tick and consumes bounded, organization-wide review-dispatch budgets across that entire walk. Default-base work uses `ORG_SWEEP_REVIEW_DISPATCH_LIMIT` (default `1`); stacked work uses the separate @@ -23,6 +31,15 @@ required review. ## Decision +The current ownership boundary is: + +- `pr-review-merge-scheduler` owns review, merge, and branch-update state. +- The integrated current-head coalescer owns same-PR stale-run cleanup. +- The organization sweep does not repeat an Actions inventory per repository. + +The rotation decision below is retained as historical operational evidence for +the former review-dispatch fairness implementation. + Rotate the sweep's repository walk order by a rotation index before applying the unchanged organization-wide budgets. `rotation_offset = rotation_index % repository_count`; the walk starts at that offset and wraps. This spreads each diff --git a/tests/test_org_queue_sweep_documentation_contract.py b/tests/test_org_queue_sweep_documentation_contract.py new file mode 100644 index 0000000000..eda0c98e48 --- /dev/null +++ b/tests/test_org_queue_sweep_documentation_contract.py @@ -0,0 +1,17 @@ +"""Contract for the superseded organization queue-sweep runbook.""" + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +RUNBOOK = ROOT / "docs" / "doctoring" / "org-queue-sweep-rotation.md" + + +def test_runbook_marks_removed_queue_hygiene_as_historical() -> None: + """The runbook must not describe removed repository-wide inventory as current.""" + source = RUNBOOK.read_text(encoding="utf-8") + + assert "Superseded for queue hygiene" in source + assert "does not repeat an Actions inventory per repository" in source + assert "#1878" in source + assert "historical operational evidence" in source From 9b8f32ab313bc367cb21b5066cbfa2fce62d99ab Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 11 Sep 2026 14:03:40 +0900 Subject: [PATCH 51/80] docs(ci): explain JavaScript materializer trust boundaries --- .../agent-review-runtime-quality-ci.yml | 41 ++++++++++++++++- ...0912-javascript-materializer-docstrings.md | 1 + .../materialize_base_javascript_packages.py | 45 +++++++++++++++---- ...test_javascript_materializer_docstrings.py | 21 +++++++++ 4 files changed, 97 insertions(+), 11 deletions(-) create mode 100644 CHANGELOG.d/20260912-javascript-materializer-docstrings.md create mode 100644 tests/test_javascript_materializer_docstrings.py diff --git a/.github/workflows/agent-review-runtime-quality-ci.yml b/.github/workflows/agent-review-runtime-quality-ci.yml index 3680da8778..4fb32063d5 100644 --- a/.github/workflows/agent-review-runtime-quality-ci.yml +++ b/.github/workflows/agent-review-runtime-quality-ci.yml @@ -16,6 +16,8 @@ on: - ".github/workflows/opencode-review-dispatch.yml" - "scripts/ci/ensure_rust_llvm19.sh" - "tests/test_opencode_rust_coverage_toolchain_contract.py" + - "scripts/ci/materialize_base_javascript_packages.py" + - "tests/test_javascript_materializer_docstrings.py" - "tests/test_pr_review_autofix_nvidia_nim_contract.py" - "docs/doctoring/opencode-rust-coverage-runtime-boundary.md" - ".github/workflows/strix.yml" @@ -90,6 +92,10 @@ on: - "scripts/ci/organization_commercial_readiness_loop.py" - "organization_commercial_readiness_fixtures.py" - "tests/test_organization_commercial_readiness_loop*.py" + - "tests/test_organization_commercial_readiness_docstrings.py" + - "CHANGELOG.d/20260911-commercial-readiness-docstrings.md" + - "tests/test_action_pin_annotation_integrity.py" + - "CHANGELOG.d/20260910-action-pin-annotation-integrity.md" - "docs/doctoring/organization-commercial-readiness-loop.md" - ".github/workflows/exact-artifact-sbom-attestation.yml" - "scripts/ci/verify_exact_artifact_sbom_handoff.py" @@ -97,6 +103,8 @@ on: - "tests/test_exact_artifact_sbom_review_regressions.py" - "tests/test_verify_exact_artifact_sbom_handoff.py" - "tests/test_exact_artifact_quality_single_runner.py" + - ".github/workflows/**" + - ".github/actions/**" - "docs/doctoring/exact-artifact-sbom-attestation.md" - "docs/doctoring/exact-artifact-sbom-quality-runner-consolidation-20260903.md" - "CHANGELOG.d/20260903-exact-artifact-quality-runner-consolidation.md" @@ -153,6 +161,7 @@ jobs: review_repair_suite=false commercial_readiness_suite=false exact_artifact_suite=false + pin_annotation_suite=false while IFS= read -r changed_path; do case "$changed_path" in @@ -164,6 +173,7 @@ jobs: review_repair_suite=true commercial_readiness_suite=true exact_artifact_suite=true + pin_annotation_suite=true ;; tests/test_pr_review_autofix_nvidia_nim_contract.py) opencode_suite=true @@ -185,6 +195,8 @@ jobs: .github/workflows/opencode-review-dispatch.yml|\ scripts/ci/ensure_rust_llvm19.sh|\ tests/test_opencode_rust_coverage_toolchain_contract.py|\ + scripts/ci/materialize_base_javascript_packages.py|\ + tests/test_javascript_materializer_docstrings.py|\ docs/doctoring/opencode-rust-coverage-runtime-boundary.md) opencode_suite=true ;; @@ -271,6 +283,8 @@ jobs: scripts/ci/organization_commercial_readiness_loop.py|\ organization_commercial_readiness_fixtures.py|\ tests/test_organization_commercial_readiness_loop*.py|\ + tests/test_organization_commercial_readiness_docstrings.py|\ + CHANGELOG.d/20260911-commercial-readiness-docstrings.md|\ docs/doctoring/organization-commercial-readiness-loop.md) commercial_readiness_suite=true ;; @@ -286,6 +300,11 @@ jobs: exact_artifact_suite=true ;; esac + case "$changed_path" in + .github/workflows/*|.github/actions/*|tests/test_action_pin_annotation_integrity.py|CHANGELOG.d/20260910-action-pin-annotation-integrity.md) + pin_annotation_suite=true + ;; + esac done < <(git diff --name-only "$BASE_SHA...$HEAD_SHA") { @@ -296,6 +315,7 @@ jobs: echo "review_repair=$review_repair_suite" echo "commercial_readiness=$commercial_readiness_suite" echo "exact_artifact=$exact_artifact_suite" + echo "pin_annotation=$pin_annotation_suite" } >>"$GITHUB_OUTPUT" - name: Install exact hash-verified base dependencies @@ -339,6 +359,13 @@ jobs: tests/test_noema_refreshed_app_identity.py \ tests/test_noema_token_lifetime_stale_run_contract.py + - name: Verify immutable action annotation consistency + if: steps.affected_suites.outputs.pin_annotation == 'true' + run: | + set -euo pipefail + python -m pytest -q tests/test_action_pin_annotation_integrity.py + python -m compileall -q tests/test_action_pin_annotation_integrity.py + - name: Verify OpenCode Rust coverage toolchain contract if: steps.affected_suites.outputs.opencode == 'true' run: | @@ -346,6 +373,13 @@ jobs: python -m pytest -q tests/test_opencode_rust_coverage_toolchain_contract.py python -m compileall -q tests/test_opencode_rust_coverage_toolchain_contract.py + - name: Verify JavaScript materializer documentation contract + if: steps.affected_suites.outputs.opencode == 'true' + run: | + set -euo pipefail + python -m pytest -q tests/test_javascript_materializer_docstrings.py + python -m compileall -q scripts/ci/materialize_base_javascript_packages.py tests/test_javascript_materializer_docstrings.py + - name: Verify exact-head path policy and syntax if: steps.affected_suites.outputs.strix == 'true' env: @@ -425,7 +459,9 @@ jobs: run: | python -m coverage run \ --branch \ - -m pytest --import-mode=importlib tests/test_organization_commercial_readiness_loop*.py -q + -m pytest --import-mode=importlib \ + tests/test_organization_commercial_readiness_loop*.py \ + tests/test_organization_commercial_readiness_docstrings.py -q python -m coverage report \ --include='scripts/ci/organization_commercial_readiness_loop.py' \ --show-missing \ @@ -433,7 +469,8 @@ jobs: python -m compileall -q \ scripts/ci/organization_commercial_readiness_loop.py \ organization_commercial_readiness_fixtures.py \ - tests/test_organization_commercial_readiness_loop*.py + tests/test_organization_commercial_readiness_loop*.py \ + tests/test_organization_commercial_readiness_docstrings.py - name: Set up minimum supported Python for exact-artifact contracts if: steps.affected_suites.outputs.exact_artifact == 'true' diff --git a/CHANGELOG.d/20260912-javascript-materializer-docstrings.md b/CHANGELOG.d/20260912-javascript-materializer-docstrings.md new file mode 100644 index 0000000000..1165181540 --- /dev/null +++ b/CHANGELOG.d/20260912-javascript-materializer-docstrings.md @@ -0,0 +1 @@ +Document trusted JavaScript lock discovery and validation boundaries with explanatory contracts enforced by CI. diff --git a/scripts/ci/materialize_base_javascript_packages.py b/scripts/ci/materialize_base_javascript_packages.py index 489364f92e..5937161896 100644 --- a/scripts/ci/materialize_base_javascript_packages.py +++ b/scripts/ci/materialize_base_javascript_packages.py @@ -58,7 +58,10 @@ def _github_actions_escape(value: object) -> str: def _git(repo_root: pathlib.Path, *args: str) -> bytes: - """Run one read-only git command in the materialized repository.""" + """Run one read-only git command in the materialized repository. + + Git failures remain bounded diagnostics and never become trusted input. + """ completed = subprocess.run( ["git", "-C", str(repo_root), *args], check=False, @@ -72,7 +75,10 @@ def _git(repo_root: pathlib.Path, *args: str) -> bytes: def _regular_base_paths(repo_root: pathlib.Path, base_sha: str) -> set[str]: - """Return regular blob paths from the exact validated base commit.""" + """Return regular blob paths from the exact validated base commit. + + Symlink-like and traversal paths are excluded before content is materialized. + """ entries = _git(repo_root, "ls-tree", "-r", "-z", "--full-tree", base_sha) paths: set[str] = set() for raw_entry in entries.split(b"\0"): @@ -102,7 +108,10 @@ def _regular_base_paths(repo_root: pathlib.Path, base_sha: str) -> set[str]: def base_pnpm_projects( repo_root: pathlib.Path, base_sha: str ) -> list[tuple[str, str, dict[str, bytes]]]: - """Return exact base pnpm inputs grouped by lockfile directory.""" + """Return exact base pnpm inputs grouped by lockfile directory. + + Each project must declare an exact package-manager version and regular inputs. + """ if not SHA_RE.fullmatch(base_sha): raise ValueError("base SHA must be exactly 40 hexadecimal characters") @@ -183,7 +192,10 @@ def base_pnpm_projects( def base_npm_projects( repo_root: pathlib.Path, base_sha: str ) -> list[tuple[str, str, dict[str, bytes]]]: - """Return exact base npm inputs grouped by lockfile directory.""" + """Return exact base npm inputs grouped by lockfile directory. + + Vestigial locks and unsafe workspace paths are excluded from the trusted set. + """ if not SHA_RE.fullmatch(base_sha): raise ValueError("base SHA must be exactly 40 hexadecimal characters") @@ -268,7 +280,10 @@ def base_npm_projects( def _lock_blob_sha(repo_root: pathlib.Path, revision_sha: str, lock_path: str) -> str: - """Return the exact Git blob SHA for one validated revision lockfile.""" + """Return the exact Git blob SHA for one validated revision lockfile. + + The identity binds materialized dependency bytes to the reviewed revision. + """ raw_blob = _git(repo_root, "rev-parse", f"{revision_sha}:{lock_path}") blob_sha = raw_blob.decode("ascii", errors="strict").strip() if not SHA_RE.fullmatch(blob_sha): @@ -279,7 +294,10 @@ def _lock_blob_sha(repo_root: pathlib.Path, revision_sha: str, lock_path: str) - def validate_head_npm_lock(lock_path: str, lock_content: bytes) -> None: - """Fail closed unless a changed HEAD npm lock is registry- and hash-bounded.""" + """Fail closed unless a changed HEAD npm lock is registry- and hash-bounded. + + Registry URLs, workspace links, and integrity values are checked without installation. + """ try: lock_data: Any = json.loads(lock_content.decode("utf-8")) except (UnicodeDecodeError, json.JSONDecodeError) as exc: @@ -372,7 +390,10 @@ def validate_head_npm_lock(lock_path: str, lock_content: bytes) -> None: def _validate_pnpm_tarball_url( lock_path: str, package_key: str, tarball_url: str ) -> None: - """Fail closed unless one pnpm tarball URL is an npm-registry HTTPS URL.""" + """Fail closed unless one pnpm tarball URL is an npm-registry HTTPS URL. + + Userinfo, ports, query strings, fragments, and alternate hosts are rejected. + """ parsed = urllib.parse.urlsplit(tarball_url) try: parsed_port = parsed.port @@ -516,7 +537,10 @@ def materialize( output_dir: pathlib.Path, head_sha: str | None = None, ) -> list[dict[str, str]]: - """Write trusted base and bounded HEAD inputs under Docker-context-safe paths.""" + """Write trusted base and bounded HEAD inputs under Docker-context-safe paths. + + Manifest records retain revision and lock-blob identity for downstream verification. + """ if output_dir.exists() and output_dir.is_symlink(): raise ValueError("output directory must not be a symlink") output_dir.mkdir(parents=True, exist_ok=True) @@ -624,7 +648,10 @@ def materialize( def main(argv: list[str] | None = None) -> int: - """Materialize trusted JavaScript locks and report their exact revisions.""" + """Materialize trusted JavaScript locks and report their exact revisions. + + Invalid or unsafe input returns a bounded non-zero diagnostic for the caller. + """ parser = argparse.ArgumentParser() parser.add_argument("--repo-root", required=True, type=pathlib.Path) parser.add_argument("--base-sha", required=True) diff --git a/tests/test_javascript_materializer_docstrings.py b/tests/test_javascript_materializer_docstrings.py new file mode 100644 index 0000000000..0457332d01 --- /dev/null +++ b/tests/test_javascript_materializer_docstrings.py @@ -0,0 +1,21 @@ +"""Documentation contract for trusted JavaScript lock materialization.""" + +import ast +from pathlib import Path + + +MODULE = Path(__file__).resolve().parents[1] / "scripts/ci/materialize_base_javascript_packages.py" + + +def test_materializer_symbols_have_explanatory_multiline_docstrings() -> None: + """Lock discovery and validation code must explain its trust boundary.""" + tree = ast.parse(MODULE.read_text(encoding="utf-8")) + violations = [] + for node in ast.walk(tree): + if not isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + continue + docstring = ast.get_docstring(node, clean=False) + if docstring is None or "\n" not in docstring: + violations.append((node.name, node.lineno)) + + assert not violations, f"materializer symbols need explanatory docs: {violations}" From a6160ab6d4a5708781d2d842094de9a7a07d4ace Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 11 Sep 2026 14:15:45 +0900 Subject: [PATCH 52/80] fix(ci): keep JavaScript quality lane self-contained --- .../agent-review-runtime-quality-ci.yml | 29 ++----------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/.github/workflows/agent-review-runtime-quality-ci.yml b/.github/workflows/agent-review-runtime-quality-ci.yml index 4fb32063d5..6c6efc3dd1 100644 --- a/.github/workflows/agent-review-runtime-quality-ci.yml +++ b/.github/workflows/agent-review-runtime-quality-ci.yml @@ -92,10 +92,6 @@ on: - "scripts/ci/organization_commercial_readiness_loop.py" - "organization_commercial_readiness_fixtures.py" - "tests/test_organization_commercial_readiness_loop*.py" - - "tests/test_organization_commercial_readiness_docstrings.py" - - "CHANGELOG.d/20260911-commercial-readiness-docstrings.md" - - "tests/test_action_pin_annotation_integrity.py" - - "CHANGELOG.d/20260910-action-pin-annotation-integrity.md" - "docs/doctoring/organization-commercial-readiness-loop.md" - ".github/workflows/exact-artifact-sbom-attestation.yml" - "scripts/ci/verify_exact_artifact_sbom_handoff.py" @@ -103,8 +99,6 @@ on: - "tests/test_exact_artifact_sbom_review_regressions.py" - "tests/test_verify_exact_artifact_sbom_handoff.py" - "tests/test_exact_artifact_quality_single_runner.py" - - ".github/workflows/**" - - ".github/actions/**" - "docs/doctoring/exact-artifact-sbom-attestation.md" - "docs/doctoring/exact-artifact-sbom-quality-runner-consolidation-20260903.md" - "CHANGELOG.d/20260903-exact-artifact-quality-runner-consolidation.md" @@ -161,7 +155,6 @@ jobs: review_repair_suite=false commercial_readiness_suite=false exact_artifact_suite=false - pin_annotation_suite=false while IFS= read -r changed_path; do case "$changed_path" in @@ -173,7 +166,6 @@ jobs: review_repair_suite=true commercial_readiness_suite=true exact_artifact_suite=true - pin_annotation_suite=true ;; tests/test_pr_review_autofix_nvidia_nim_contract.py) opencode_suite=true @@ -283,8 +275,6 @@ jobs: scripts/ci/organization_commercial_readiness_loop.py|\ organization_commercial_readiness_fixtures.py|\ tests/test_organization_commercial_readiness_loop*.py|\ - tests/test_organization_commercial_readiness_docstrings.py|\ - CHANGELOG.d/20260911-commercial-readiness-docstrings.md|\ docs/doctoring/organization-commercial-readiness-loop.md) commercial_readiness_suite=true ;; @@ -300,11 +290,6 @@ jobs: exact_artifact_suite=true ;; esac - case "$changed_path" in - .github/workflows/*|.github/actions/*|tests/test_action_pin_annotation_integrity.py|CHANGELOG.d/20260910-action-pin-annotation-integrity.md) - pin_annotation_suite=true - ;; - esac done < <(git diff --name-only "$BASE_SHA...$HEAD_SHA") { @@ -315,7 +300,6 @@ jobs: echo "review_repair=$review_repair_suite" echo "commercial_readiness=$commercial_readiness_suite" echo "exact_artifact=$exact_artifact_suite" - echo "pin_annotation=$pin_annotation_suite" } >>"$GITHUB_OUTPUT" - name: Install exact hash-verified base dependencies @@ -359,13 +343,6 @@ jobs: tests/test_noema_refreshed_app_identity.py \ tests/test_noema_token_lifetime_stale_run_contract.py - - name: Verify immutable action annotation consistency - if: steps.affected_suites.outputs.pin_annotation == 'true' - run: | - set -euo pipefail - python -m pytest -q tests/test_action_pin_annotation_integrity.py - python -m compileall -q tests/test_action_pin_annotation_integrity.py - - name: Verify OpenCode Rust coverage toolchain contract if: steps.affected_suites.outputs.opencode == 'true' run: | @@ -460,8 +437,7 @@ jobs: python -m coverage run \ --branch \ -m pytest --import-mode=importlib \ - tests/test_organization_commercial_readiness_loop*.py \ - tests/test_organization_commercial_readiness_docstrings.py -q + tests/test_organization_commercial_readiness_loop*.py -q python -m coverage report \ --include='scripts/ci/organization_commercial_readiness_loop.py' \ --show-missing \ @@ -469,8 +445,7 @@ jobs: python -m compileall -q \ scripts/ci/organization_commercial_readiness_loop.py \ organization_commercial_readiness_fixtures.py \ - tests/test_organization_commercial_readiness_loop*.py \ - tests/test_organization_commercial_readiness_docstrings.py + tests/test_organization_commercial_readiness_loop*.py - name: Set up minimum supported Python for exact-artifact contracts if: steps.affected_suites.outputs.exact_artifact == 'true' From 9b4580c21e15cb9ac019bc9e7a7a5d65298af696 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 17:13:28 +0900 Subject: [PATCH 53/80] test: require usable README scaffold and incomplete blocker semantics --- tests/test_readme_template_contract.py | 62 ++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 tests/test_readme_template_contract.py diff --git a/tests/test_readme_template_contract.py b/tests/test_readme_template_contract.py new file mode 100644 index 0000000000..ce07e6d100 --- /dev/null +++ b/tests/test_readme_template_contract.py @@ -0,0 +1,62 @@ +"""Guard the authoring scaffold without claiming to validate product truth.""" + +from pathlib import Path +import unittest + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +STANDARD_PATH = REPOSITORY_ROOT / "docs/repository-readme-quality-standard.md" +TEMPLATE_PATH = REPOSITORY_ROOT / "docs/templates/repository-readme-template.md" + + +class ReadmeTemplateContractTests(unittest.TestCase): + """Check discoverability, safe placeholders, and honest integration status.""" + + def test_standard_links_to_the_adaptable_template(self): + """A checklist alone must not stand in for the requested writing scaffold.""" + standard_text = STANDARD_PATH.read_text(encoding="utf-8") + self.assertIn("(templates/repository-readme-template.md)", standard_text) + self.assertNotIn("quality contract, not a copy-and-paste template", standard_text) + + def test_template_contains_one_non_executable_product_scaffold(self): + """The scaffold has reader sections, but no invented commands or badges.""" + self.assertTrue(TEMPLATE_PATH.is_file(), "The reusable README template is missing") + template_text = TEMPLATE_PATH.read_text(encoding="utf-8") + self.assertEqual(template_text.count(""), 1) + self.assertEqual(template_text.count(""), 1) + scaffold_text = template_text.split("", 1)[1].split( + "", 1 + )[0] + for heading_text in ( + "# {{product_name}}", "## What you can do", "## Get started", + "## Example", "## How it fits", "## Status and verification", + "## Documentation", "## Support and contributing", "## License", + ): + with self.subTest(heading=heading_text): + self.assertIn(heading_text, scaffold_text) + self.assertIn("{{verified_license_statement}}", scaffold_text) + self.assertIn("{{observed_result_and_next_action}}", scaffold_text) + self.assertNotIn("```bash", scaffold_text) + self.assertNotIn("https://", scaffold_text) + self.assertNotIn("![", scaffold_text) + + def test_standard_requires_execution_and_side_effect_evidence(self): + """A command's presence in a file is not evidence that onboarding worked.""" + standard_text = STANDARD_PATH.read_text(encoding="utf-8") + for evidence_label in ( + "Working directory", "Runtime and lock", "Observed result", + "Side effects", "Stop and recovery", "Not executed", + ): + with self.subTest(evidence=evidence_label): + self.assertIn(evidence_label, standard_text) + + def test_external_blocker_is_not_completion(self): + """Blocked work retains ownership instead of being marked finished.""" + standard_text = STANDARD_PATH.read_text(encoding="utf-8") + self.assertIn("A blocker leaves the work incomplete", standard_text) + self.assertNotIn("integrated or a real external blocker remains", standard_text) + self.assertIn("every valid delta", standard_text) + + +if __name__ == "__main__": + unittest.main() From 1a7120f66b9dcba6db12c618f11bd67e22f7fadf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 17:14:11 +0900 Subject: [PATCH 54/80] docs: add adaptable product README authoring scaffold --- docs/templates/repository-readme-template.md | 129 +++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 docs/templates/repository-readme-template.md diff --git a/docs/templates/repository-readme-template.md b/docs/templates/repository-readme-template.md new file mode 100644 index 0000000000..88be951603 --- /dev/null +++ b/docs/templates/repository-readme-template.md @@ -0,0 +1,129 @@ +# Repository README authoring template + +Use this scaffold with the [README quality standard](../repository-readme-quality-standard.md). +It is an authoring aid, not finished customer copy or an organization-wide generator. +Keep each product's language, ownership, supported workflow, and license evidence. +Remove irrelevant sections rather than publishing empty headings. + +## Before adapting + +Choose one truthful starting path: a verified released package, an evaluated source +worktree, or a design/contract foundation. Do not give all three equal prominence +when only one exists. For a source path, state the working directory, actual +runtime prerequisite, lock/install command, and expected next action. For a +foundation, replace install commands with a real contract/validation entry point. + +Replace every `{{...}}` field from current evidence. Do not publish the scaffold, +these instructions, placeholder links, or guessed output. The text-fenced fields +below are intentionally not executable. Promote them to a language-tagged code +example only after the repository's actual command has been checked and, where +possible, run safely. Record an unexecuted example as unverified in the PR rather +than inventing a successful run. No badge, license, benchmark, or support channel +is selected by this template. + +## Copyable scaffold + + +````markdown +# {{product_name}} + +**{{one_sentence_user_outcome}}** + +{{who_this_helps_and_the_problem_it_solves}} + +[Get started](#get-started) · [Documentation](#documentation) · [Support](#support-and-contributing) + +## What you can do + +{{two_or_three_current_user_jobs_with_concrete_results}} + +{{important_limitation_that_changes_a_users_next_action}} + +## Get started + +{{release_or_source_or_foundation_status_and_prerequisites}} + +```text +{{verified_working_directory_and_setup_commands}} +``` + +{{observed_result_and_next_action}} + +{{network_data_permission_cost_and_stop_or_cleanup_notes}} + +## Example + +{{one_representative_task_and_its_input_requirements}} + +```text +{{verified_public_api_or_cli_example}} +``` + +{{bounded_expected_result_and_failure_recovery}} + +## How it fits + +{{short_data_flow_and_optional_integrations_in_user_language}} + +{{what_this_product_owns_and_what_remains_with_the_host_or_source_system}} + +## Status and verification + +{{current_maturity_and_link_to_actual_release_or_verification_evidence}} + +```text +{{repository_supported_verification_commands}} +``` + +{{tested_scope_and_remaining_limits_without_global_quality_claims}} + +## Documentation + +| I need to... | Start here | +| --- | --- | +| {{first_reader_job}} | [{{document_title}}]({{existing_document_path}}) | +| {{second_reader_job}} | [{{reference_title}}]({{existing_reference_path}}) | + +## Support and contributing + +{{existing_support_and_private_security_reporting_routes_or_their_explicit_limits}} + +{{smallest_contributor_verification_contract_and_link_to_details}} + +## License + +{{verified_license_statement}} + +{{third_party_notice_links_and_relevant_distribution_limits}} +```` + + +## Presentation choices + +Use a clear title, one strong opening sentence, short paragraphs, and a compact +navigation line. Keep badges few and relevant, and link each to real evidence. +Avoid walls of logos, decorative shields, HTML layout tables, unsupported +superlatives, and embedded internal incident dashboards. Long tables and exhaustive +CLI flags belong in linked references. Check heading navigation and code wrapping +at a narrow viewport as well as on desktop. + +A screenshot can establish what a real interface looks like, not that every +feature works. Include only an actual, reviewed product view with useful alt text, +no personal data or secrets, and an identifiable source revision in its retained +evidence. Omit screenshots when the product has no visual interface. + +## Adaptation and handoff record + +Keep the record in the PR or doctoring, not in the customer README. Bind each +material claim to its owning source and exact revision. For quick start, record +working directory, runtime/lock identity, command, observed result, side effects, +and stop/recovery. For a claim you could not execute, say **Not executed** and +state the missing capability. For licensing, separate first-party grant from +inherited source, dependencies, assets and service terms; metadata is inventory, +not rights approval. Retain required attribution rather than hiding an intake +conflict. + +Validate relative links against the proposed tree and the actual publishing root. +Generated READMEs require updating their authoring source and checking generated +output. Keep the existing authoritative PR and prove every valid delta survives +any consolidation. A queued check, external blocker, or saved draft is not delivery. From 0e7dc2ed370144a9707ff85a70f446648378548e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 17:15:11 +0900 Subject: [PATCH 55/80] docs: bind README template to execution evidence and protected delivery --- docs/repository-readme-quality-standard.md | 42 ++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/docs/repository-readme-quality-standard.md b/docs/repository-readme-quality-standard.md index 3d3d5895ea..e663ef8d94 100644 --- a/docs/repository-readme-quality-standard.md +++ b/docs/repository-readme-quality-standard.md @@ -4,7 +4,15 @@ Every ContextualWisdomLab repository owns its own product README because the README must be reviewed against that repository's actual code, PRD, ADRs, release state, security boundary, and license provenance. The organization control plane may define a reusable quality standard, but it must not manufacture product claims or overwrite repository-specific language. -This document is the shared review pattern for repository landing pages. It is intentionally a **quality contract, not a copy-and-paste template**. A good README should feel consistent across the organization while still making the product's bounded context, terminology, operating reality, and obligations obvious. +This document is the shared review pattern for repository landing pages. It combines a **quality contract with an adaptable authoring template**, not automatically generated product claims. A good README should feel consistent across the organization while still making the product's bounded context, terminology, operating reality, and obligations obvious. + +## Reusable authoring scaffold + +Start from [the repository README template](templates/repository-readme-template.md), +then replace its placeholders with evidence from the owning repository. The shared +structure is reusable; product claims, commands, output, license grants and support +routes are not interchangeable. Remove irrelevant sections and never publish raw +placeholders. The template grants no runtime, release or integration authority. ## Reader jobs @@ -62,6 +70,28 @@ Rules: If the repository is architecture-only or pre-runtime, say so instead of inventing an installation section. Give the reader the correct next action, such as reading the contract or running repository validation. +### Quick-start execution evidence + +A command found in a manifest is source evidence, not proof that the complete +onboarding journey ran. Record these fields in the PR or doctoring and keep only +the useful instructions in the README: + +| Evidence field | Required distinction | +| --- | --- | +| Working directory | Released installation, source worktree, or design-only checkout; name the actual command root. | +| Runtime and lock | Tested toolchain and exact dependency/lock identity, separate from merely declared support. | +| Observed result | Exit status, actual output or interface, and the next useful action; do not manufacture example success. | +| Side effects | Network listener, external calls, credentials, filesystem/database writes and costs. | +| Stop and recovery | How to stop the process, retain user data and undo or recover safely. | +| Not executed | State the unavailable capability and the missing execution evidence explicitly. | + +Prefer local-only evaluation defaults. A README-only workaround is insufficient +when the advertised default script violates its own privacy or safety contract; +repair the owning configuration and add a regression. Preserve existing tests. +Check generated README sources as well as generated output, and validate links +against the proposed tree and the real publishing root. Placeholder scaffold +checks do not prove an individual product's commands, claims or rights. + ### 4. Common usage or public API Show the stable user/integrator surface, not a tour of internal modules. Prefer one representative example plus links to complete reference material. @@ -181,7 +211,10 @@ Do not use dependency licenses as a shortcut for source-license analysis. ## Integration loop -README work is complete only when the branch is integrated or a real external blocker remains. +README delivery is complete only after ordinary protected integration and +verification of the integrated content. A blocker leaves the work incomplete; +retain its exact evidence and ownership, repair another safe lane, and revisit it. +An unmerged successor, saved patch, or issue description is not delivery. 1. Search for overlapping README/documentation/license PRs before creating a new lane. 2. Update the most authoritative writable existing lane when coherent with its scope. @@ -194,6 +227,11 @@ README work is complete only when the branch is integrated or a real external bl 9. If one PR is waiting, continue another safe README/documentation lane; waiting is not completion. 10. After merge, continue to the next highest-leverage repository. +Before retiring an overlapping or stale PR, prove every valid delta is retained +in the canonical successor's source, requirements, tests and licensing notices. +A title match, successful test in isolation, or shared ancestry is not proof of +complete content/behavior carryover. Keep unresolved predecessors alive. + Do not bypass substantive failing tests, unresolved security findings, meaningful review objections, conflicts, required governance, or genuine provenance blockers. ## Quality bar From 4a1519e7992f709165347da5af3170536eb02a91 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 17:18:34 +0900 Subject: [PATCH 56/80] docs: retain README template decision and verification boundaries --- .../readme-template-authoring-contract.md | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 docs/doctoring/readme-template-authoring-contract.md diff --git a/docs/doctoring/readme-template-authoring-contract.md b/docs/doctoring/readme-template-authoring-contract.md new file mode 100644 index 0000000000..8afaa97563 --- /dev/null +++ b/docs/doctoring/readme-template-authoring-contract.md @@ -0,0 +1,70 @@ +# README authoring template and delivery contract + +## Problem + +The README mission requests a reusable writing template, but the existing standard +at `bd56a9cc599e60be1b5d2a9729dea36b0e215e9f` explicitly excluded a template. +It also described a remaining external blocker as an alternative completion +condition. A checklist was useful, but neither sentence satisfied the requested +writing-and-integration outcome. + +## Decision + +Keep the existing standard and its product-owned language, provenance, licensing, +and ordinary-merge controls. Add `docs/templates/repository-readme-template.md` +as an adaptable authoring scaffold. Require command working-directory, runtime, +lock, result, side-effect and stop/recovery evidence. An unexecuted example must +be identified as such; an external blocker leaves delivery incomplete. + +The scaffold supplies a title/promise, task-oriented introduction, quick start, +example, integration context, status/verification, document navigation, support +and license section. It supplies no real package, command, endpoint, badge, +benchmark, source license or customer claim. Authors replace placeholders only +from repository evidence and omit irrelevant sections. Nothing automatically +copies the template into product repositories or changes their runtime contracts. + +## Alternatives and risks + +A checklist alone was rejected because it leaves the requested reusable writing +structure absent. A mass README generator was rejected because repository-specific +product authority, release state and third-party obligations are not interchangeable. +A blocked PR must retain its ownership and valid delta rather than be reported +complete or closed on a title/ancestry match. + +The main misuse risk is publishing the scaffold literally or interpreting static +template tests as proof of a product's runtime behavior or license rights. The +instructions prohibit both. Real command execution, visual review when relevant, +source/provenance assessment, and exact-head integration evidence remain separate. + +## Verification + +The pre-change standard bytes matched Git blob +`3d3d5895ead0b4dfc8a94d7b651d6d7de68bb427`. The new unittest module was exercised +against that standard before the repair: template presence, discoverability, +execution fields and incomplete-blocker assertions failed. After the documentation +change, all four test methods pass locally under Python 3.13.5: + +```sh +python -m unittest discover -s tests -p test_readme_template_contract.py -v +``` + +This is structural documentation evidence, not full repository CI, semantic +review, release evidence, or a sibling-product conformance claim. Protected +integration still requires the unchanged current head's applicable checks and +reviews. The existing `.github` MIT source grant is unchanged; this work adds no +third-party source, dependency, asset, or license exception. + +## References + +GitHub. (n.d.). *About READMEs*. GitHub Docs. Retrieved September 12, 2026, from +https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-readmes + +GitHub. (n.d.). *Licensing a repository*. GitHub Docs. Retrieved September 12, +2026, from +https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/licensing-a-repository + +GitHub's README guidance supports purpose, getting-started, help and maintainer +navigation. Its licensing guidance supports keeping a real source grant distinct +from repository visibility. The stronger commercial-intake, exact-head and +complete-delta-delivery requirements here come from the repository owner's +explicit CWL mission, not an assertion that GitHub requires those policies. From 9521b677170f326ae852af37511a3492efa35a4e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 23:38:08 +0900 Subject: [PATCH 57/80] test(scheduler): require read-only commit status evidence --- tests/test_opencode_agent_contract.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 321d25bd57..d68a8e2e2c 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -2371,6 +2371,11 @@ def test_merge_scheduler_uses_escalating_mutation_credentials(): encoding="utf-8" ) + scan_job = workflow.split(" scan-pr-queue:\n", 1)[1] + permission_block = scan_job.split(" permissions:\n", 1)[1].split(" env:\n", 1)[0] + status_permissions = re.findall(r"^ statuses: (\w+)\s*$", permission_block, re.MULTILINE) + assert status_permissions == ["read"], "same-repository status evidence needs read-only permission" + assert "id-token: write" in workflow assert "Exchange OpenCode app token for scheduler mutations" in workflow assert "secrets.PR_REVIEW_MERGE_TOKEN" in workflow From 1c5c6b7afa841f7a4a999f54d45ef8d504a2d818 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 23:39:47 +0900 Subject: [PATCH 58/80] fix(scheduler): grant job-scoped commit status reads --- .../workflows/pr-review-merge-scheduler.yml | 1 + .../scheduler-status-read-permission.md | 38 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 docs/doctoring/scheduler-status-read-permission.md diff --git a/.github/workflows/pr-review-merge-scheduler.yml b/.github/workflows/pr-review-merge-scheduler.yml index d32918cf45..d98a72e605 100644 --- a/.github/workflows/pr-review-merge-scheduler.yml +++ b/.github/workflows/pr-review-merge-scheduler.yml @@ -122,6 +122,7 @@ jobs: contents: write id-token: write pull-requests: write + statuses: read env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true GH_TOKEN: ${{ github.token }} diff --git a/docs/doctoring/scheduler-status-read-permission.md b/docs/doctoring/scheduler-status-read-permission.md new file mode 100644 index 0000000000..82e7b0cbf3 --- /dev/null +++ b/docs/doctoring/scheduler-status-read-permission.md @@ -0,0 +1,38 @@ +# Scheduler commit-status read permission (#2120) + +The organization-required scheduler selected `github.token` for same-repository +reads but omitted `statuses` from `scan-pr-queue.permissions`. The private +consumer's combined-status GET consequently failed with HTTP 403 before a merge +verdict. `checks: read` does not grant classic commit-status access. + +Source baseline: `fb17ef556f94f673234aa557254ae52779e9a7b0`. +Consumer evidence: ContextualWisdomLab/late-life-anxiety-reanalysis#10, +head `3d1e3ae56e3ef6ca0a995b6082c4f4a13629e0f6`, run `34698738407`, +job `103566634488` (2026-09-12). The reported failing endpoint is +`GET /repos/{repository}/commits/{head}/status`. + +The repair adds only `statuses: read` to the existing scan job. Workflow defaults, +mutation credentials, cross-repository credential selection and fail-closed API +errors remain intact. It adds no status publication or App installation grant. +The existing credential-contract test now requires exactly `read` in that job's +permission block; its RED revision is `9521b6771`. + +Validation uses the existing pytest workflow/credential/status suites and +Actionlint's workflow validation. Local tests prove the declared contract, not a +hosted permission grant. After protected integration, validate a newly loaded +central source SHA and the consumer's exact current head: the combined-status +request must succeed, and missing checks or substantive failures must still block +merge. For reusable callers, every caller permission ceiling must also admit +status reads; the inspected consumer PR head has no `.github` tree, so do not +invent a repository-local caller or modify App permissions to compensate. + +Next integration review: 2026-09-13, because this prevents the current private +consumer's mandatory scheduler from evaluating status evidence. #2116's HWPX +classification remains a separate bootstrap repair. Reverting this one-line grant +restores the pre-existing 403 behavior; it is not a viable consumer repair. + +## Reference + +GitHub. (n.d.). *REST API endpoints for commit statuses: Get the combined status +for a specific reference*. Retrieved September 12, 2026, from +https://docs.github.com/en/rest/commits/statuses#get-the-combined-status-for-a-specific-reference From f823976978c87d860ff8ba082a78b02d2a6fa3a5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 01:01:30 +0900 Subject: [PATCH 59/80] test(opencode): cover python-root VCS imports --- tests/test_opencode_agent_contract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 321d25bd57..53e83c2b00 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -749,7 +749,7 @@ def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): assert 'vcs-manifest.json >"$dependency_list"' in measure_step assert 'done <"$dependency_list"' in measure_step assert 'candidate_count=$((candidate_count + 1))' in measure_step - assert '[ "$candidate_count" -ne 1 ]' in measure_step + # Immutable VCS packages may expose their import package from a project-specific\n # ``python/`` source root (fast-mlsirm is the live protected-base fixture).\n assert \'"$destination/python/$import_name"\' in measure_step\n assert \'"$destination/python/$import_name.py"\' in measure_step\n assert '[ "$candidate_count" -ne 1 ]' in measure_step assert "has a missing or ambiguous import root" in measure_step assert '[ ! -f "$import_root/__init__.py" ]' in measure_step assert "has a namespace or linked import root" in measure_step From af04581cea4ffc038c881c6ad101ea3e5842a664 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 01:01:54 +0900 Subject: [PATCH 60/80] fix(opencode): admit immutable python source roots --- .github/workflows/opencode-review-dispatch.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index d86497b3f4..f315683208 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -828,6 +828,8 @@ jobs: python_root=''; \ candidate_count=0; \ for candidate in \ + "$destination/python/$import_name" \ + "$destination/python/$import_name.py" \ "$destination/src/$import_name" \ "$destination/src/$import_name.py" \ "$destination/$import_name" \ @@ -870,6 +872,7 @@ jobs: exit 1; \ fi; \ case "$import_root" in \ + "$destination/python/"*) python_root="$destination/python" ;; \ "$destination/src/"*) python_root="$destination/src" ;; \ *) python_root="$destination" ;; \ esac; \ From be757189966ad7ea4e763abf4d93ef8416fdd96f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 01:02:45 +0900 Subject: [PATCH 61/80] docs(opencode): record VCS source-root RCA --- CHANGELOG.md | 4 +++ .../opencode-vcs-python-source-root.md | 27 +++++++++++++++++++ docs/product-technical-gap-baseline.md | 6 +++++ 3 files changed, 37 insertions(+) create mode 100644 docs/doctoring/opencode-vcs-python-source-root.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 707c18532e..28bb9fc824 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### OpenCode coverage admits immutable `python/` VCS source roots + +- Central OpenCode coverage run [34701472466](https://github.com/ContextualWisdomLab/.github/actions/runs/34701472466) failed before executing `contextual-orchestrator#1149`: the trusted image builder resolved VCS packages only from repository root or `src/`, while the exact immutable `fast-mlsirm@09f762ded35786dd1078222a4577ff09d649816f` exposes `fast_mlsirm` from `python/fast_mlsirm`. The builder now admits the explicit `python/` source root, retains the one-and-only-one import-root invariant, symlink/namespace/compiled-artifact/installed-metadata rejection, exact commit verification, and the later credential-free networkless sandbox. Contract tests pin both package and single-module `python/` layouts. Refs `contextual-orchestrator#1149`. + ### Failed-check finding names the Strix sandbox instead of the gateway - `opencode-review-dispatch.yml`'s `emit_strix_provider_failure_finding` rendered one fixed finding for every `STRIX_PROVIDER_UNAVAILABLE` line, whose Root cause read "The contextual-orchestrator gateway or its discovered provider pool was unavailable for this run". `#1953` had just given the Strix sandbox bootstrap failure its own second verdict token (`STRIX_SANDBOX_UNAVAILABLE`) precisely because that attribution is wrong for it -- the sandbox container never reaches its Caido proxy, so the run dies before the gateway serves anything -- and this consumer re-applied the wrong attribution one step downstream, into the review findings and the failure census. The emitter now branches on the second token: a sandbox verdict gets a finding that names Strix's sandbox, says the verdict does not name the gateway, and tells the reader not to change gateway or provider configuration on its strength. A `STRIX_PROVIDER_UNAVAILABLE` line without the token keeps its existing text verbatim, so the gateway class has no regression surface. No test covered this finding text at all before (`gateway or its discovered provider pool` matched nothing under `tests/`); `tests/test_opencode_dispatch_strix_sandbox_finding.py` now runs the production emitter from the published run block and pins both directions plus the no-signal case. Refs #1953, #1935. diff --git a/docs/doctoring/opencode-vcs-python-source-root.md b/docs/doctoring/opencode-vcs-python-source-root.md new file mode 100644 index 0000000000..3daa4b0839 --- /dev/null +++ b/docs/doctoring/opencode-vcs-python-source-root.md @@ -0,0 +1,27 @@ +# OpenCode immutable VCS `python/` source-root RCA + +Status: **Proposed** — source repair exists on an open pull request; it is not protected-branch authority until merged. + +## Incident and user-visible failure + +On 2026-09-12 UTC, central OpenCode dispatch [run 34701472466](https://github.com/ContextualWisdomLab/.github/actions/runs/34701472466) validated `ContextualWisdomLab/contextual-orchestrator#1149` at exact head `684cf28fa59e800c0db4886a08f25dd2edd156fc`. Its `coverage-source-tree` job succeeded, but `coverage-evidence` job `103574547257` failed while building the trusted tool image, before any pull-request test or coverage command ran. OpenCode therefore published only a non-approving COMMENTED review, and the required receipt remained fail-closed. + +The failing dependency was the exact VCS pin `fast-mlsirm@09f762ded35786dd1078222a4577ff09d649816f` from the consumer's validated `pyproject.toml`. That commit contains `python/fast_mlsirm/__init__.py`; it does not expose the import package at repository root or under `src/`. + +## Root cause and boundary + +`opencode-review-dispatch.yml` enumerated only four trusted candidates: `src/`, `src/.py`, ``, and `.py`. The materializer had already authenticated the target repository, bound the dependency to an immutable commit, fetched that commit without tags, and verified `FETCH_HEAD` and `HEAD`; the failure was solely an incomplete source-layout contract in the central owner. + +The selected repair adds only `python/` and `python/.py`, then maps a match to the repository's `python/` directory. It preserves the invariant that exactly one candidate may exist and continues to reject symlinked/namespace imports, any symlink layout, compiled extensions, installed distribution metadata, and ambiguous roots. It does not infer arbitrary paths from untrusted packaging metadata and does not execute dependency lifecycle code. + +Rejected alternatives were: changing the consumer's valid immutable dependency pin; copying `fast-mlsirm` into the consumer; adding the whole repository to `PYTHONPATH`; recursively searching for a matching directory; or weakening/bypassing the OpenCode coverage gate. Each would move ownership, admit ambiguity, or hide the central defect. + +## RED → repair → verification gate + +- RED commit `f823976978c87d860ff8ba082a78b02d2a6fa3a5` requires both package and single-module `python/` candidates in the published workflow contract. +- Repair commit `af04581cea4ffc038c881c6ad101ea3e5842a664` adds those candidates and the corresponding `python_root` mapping. +- Hosted current-head tests, security, CodeQL, and independent review remain required. Only after ordinary protected-main integration may affected consumers rerun OpenCode; the predecessor run is never transferable as GREEN evidence. + +## Follow-up + +After merge, rerun only consumer failures whose cause changed, beginning with `contextual-orchestrator#1149`. Verify that the trusted image builds from the same `fast-mlsirm` commit, the PR sandbox remains networkless and credential-free, coverage/docstring evidence executes, and a substantive exact-head review is published. If any additional conventional source root is needed, add it through its own immutable fixture and one-root regression rather than generalized path discovery. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 1cc9e20313..ae35f4ca24 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -7,6 +7,12 @@ 이 문서는 제품·기술·운영 Gap을 현재 문서와 현재 GitHub 상태에 묶어 두는 기준선이다. 새 작업은 먼저 이 문서의 Gap ID를 PR 설명과 테스트 증거에 연결하고, PR의 정확한 exact HEAD·Checks·리뷰를 다시 수집한 뒤 구현한다. 표의 상태는 작성 시점의 관측값이므로, 병합 판단에는 재사용하지 않는다. 이 인벤토리는 스냅샷이며 merge authorization이 아니다. +### 2026-09-13 current-head incident delta + +| Gap ID | 상태 | exact-head evidence | causal owner / next gate | +|---|---|---|---| +| CONTROL-OPENCODE-VCS-PYROOT-01 | **Proposed / source repaired; hosted exact-head validation pending** | `contextual-orchestrator#1149@684cf28f`의 중앙 [OpenCode run 34701472466](https://github.com/ContextualWisdomLab/.github/actions/runs/34701472466) `coverage-evidence` job `103574547257`은 PR 코드를 실행하기 전에 immutable `fast-mlsirm@09f762d`의 `python/fast_mlsirm` import root를 찾지 못해 종료했다. 같은 head의 제품 테스트는 `3602 passed, 2 skipped`, native CodeQL·fuzz·SBOM·SAST·Strix는 성공했다. | `.github`의 `opencode-review-dispatch.yml`이 root/`src/`만 허용한 계약 drift를 소유한다. RED contract `f8239769`와 최소 source repair `af04581c` 뒤, 이 문서 head의 integrated CI가 GREEN이고 protected `main`에 ordinary merge된 다음 affected consumer exact head를 다시 검증한다. | + ## 1. 근거와 범위 ### 1.1 우선순위가 높은 근거 From ed82ddb5fc52d7da072107da832bbd3a6bad53bf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 01:03:52 +0900 Subject: [PATCH 62/80] test(opencode): require python-root VCS imports --- tests/test_opencode_agent_contract.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 321d25bd57..8303f88480 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -749,6 +749,10 @@ def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): assert 'vcs-manifest.json >"$dependency_list"' in measure_step assert 'done <"$dependency_list"' in measure_step assert 'candidate_count=$((candidate_count + 1))' in measure_step + # Immutable VCS packages may expose their import package from a project-specific + # ``python/`` source root (fast-mlsirm is the live protected-base fixture). + assert \'"$destination/python/$import_name"\' in measure_step + assert \'"$destination/python/$import_name.py"\' in measure_step assert '[ "$candidate_count" -ne 1 ]' in measure_step assert "has a missing or ambiguous import root" in measure_step assert '[ ! -f "$import_root/__init__.py" ]' in measure_step From ad59e5daac881f9aaaa3729543308c97e007aa31 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 01:04:14 +0900 Subject: [PATCH 63/80] docs(opencode): bind corrected RED ancestry --- docs/doctoring/opencode-vcs-python-source-root.md | 2 +- docs/product-technical-gap-baseline.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/doctoring/opencode-vcs-python-source-root.md b/docs/doctoring/opencode-vcs-python-source-root.md index 3daa4b0839..b9f1bda863 100644 --- a/docs/doctoring/opencode-vcs-python-source-root.md +++ b/docs/doctoring/opencode-vcs-python-source-root.md @@ -18,7 +18,7 @@ Rejected alternatives were: changing the consumer's valid immutable dependency p ## RED → repair → verification gate -- RED commit `f823976978c87d860ff8ba082a78b02d2a6fa3a5` requires both package and single-module `python/` candidates in the published workflow contract. +- RED commit `ed82ddb5fc52d7da072107da832bbd3a6bad53bf` requires both package and single-module `python/` candidates in the published workflow contract. - Repair commit `af04581cea4ffc038c881c6ad101ea3e5842a664` adds those candidates and the corresponding `python_root` mapping. - Hosted current-head tests, security, CodeQL, and independent review remain required. Only after ordinary protected-main integration may affected consumers rerun OpenCode; the predecessor run is never transferable as GREEN evidence. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index ae35f4ca24..f85c99a074 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -11,7 +11,7 @@ | Gap ID | 상태 | exact-head evidence | causal owner / next gate | |---|---|---|---| -| CONTROL-OPENCODE-VCS-PYROOT-01 | **Proposed / source repaired; hosted exact-head validation pending** | `contextual-orchestrator#1149@684cf28f`의 중앙 [OpenCode run 34701472466](https://github.com/ContextualWisdomLab/.github/actions/runs/34701472466) `coverage-evidence` job `103574547257`은 PR 코드를 실행하기 전에 immutable `fast-mlsirm@09f762d`의 `python/fast_mlsirm` import root를 찾지 못해 종료했다. 같은 head의 제품 테스트는 `3602 passed, 2 skipped`, native CodeQL·fuzz·SBOM·SAST·Strix는 성공했다. | `.github`의 `opencode-review-dispatch.yml`이 root/`src/`만 허용한 계약 drift를 소유한다. RED contract `f8239769`와 최소 source repair `af04581c` 뒤, 이 문서 head의 integrated CI가 GREEN이고 protected `main`에 ordinary merge된 다음 affected consumer exact head를 다시 검증한다. | +| CONTROL-OPENCODE-VCS-PYROOT-01 | **Proposed / source repaired; hosted exact-head validation pending** | `contextual-orchestrator#1149@684cf28f`의 중앙 [OpenCode run 34701472466](https://github.com/ContextualWisdomLab/.github/actions/runs/34701472466) `coverage-evidence` job `103574547257`은 PR 코드를 실행하기 전에 immutable `fast-mlsirm@09f762d`의 `python/fast_mlsirm` import root를 찾지 못해 종료했다. 같은 head의 제품 테스트는 `3602 passed, 2 skipped`, native CodeQL·fuzz·SBOM·SAST·Strix는 성공했다. | `.github`의 `opencode-review-dispatch.yml`이 root/`src/`만 허용한 계약 drift를 소유한다. RED contract `ed82ddb5`와 최소 source repair `af04581c` 뒤, 이 문서 head의 integrated CI가 GREEN이고 protected `main`에 ordinary merge된 다음 affected consumer exact head를 다시 검증한다. | ## 1. 근거와 범위 From b1fe97c477b56e148afbeeaed9a6b74338994b6b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 01:05:03 +0900 Subject: [PATCH 64/80] test(opencode): require valid python-root VCS imports --- tests/test_opencode_agent_contract.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 321d25bd57..87cc5628ef 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -749,6 +749,10 @@ def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): assert 'vcs-manifest.json >"$dependency_list"' in measure_step assert 'done <"$dependency_list"' in measure_step assert 'candidate_count=$((candidate_count + 1))' in measure_step + # Immutable VCS packages may expose their import package from a project-specific + # ``python/`` source root (fast-mlsirm is the live protected-base fixture). + assert '"$destination/python/$import_name"' in measure_step + assert '"$destination/python/$import_name.py"' in measure_step assert '[ "$candidate_count" -ne 1 ]' in measure_step assert "has a missing or ambiguous import root" in measure_step assert '[ ! -f "$import_root/__init__.py" ]' in measure_step From 5303e09fbafbff5bfbb789a6ebf0442487bba41f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 01:05:27 +0900 Subject: [PATCH 65/80] docs(opencode): bind valid RED contract --- docs/doctoring/opencode-vcs-python-source-root.md | 2 +- docs/product-technical-gap-baseline.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/doctoring/opencode-vcs-python-source-root.md b/docs/doctoring/opencode-vcs-python-source-root.md index b9f1bda863..a0ac60255d 100644 --- a/docs/doctoring/opencode-vcs-python-source-root.md +++ b/docs/doctoring/opencode-vcs-python-source-root.md @@ -18,7 +18,7 @@ Rejected alternatives were: changing the consumer's valid immutable dependency p ## RED → repair → verification gate -- RED commit `ed82ddb5fc52d7da072107da832bbd3a6bad53bf` requires both package and single-module `python/` candidates in the published workflow contract. +- RED commit `b1fe97c477b56e148afbeeaed9a6b74338994b6b` requires both package and single-module `python/` candidates in the published workflow contract. - Repair commit `af04581cea4ffc038c881c6ad101ea3e5842a664` adds those candidates and the corresponding `python_root` mapping. - Hosted current-head tests, security, CodeQL, and independent review remain required. Only after ordinary protected-main integration may affected consumers rerun OpenCode; the predecessor run is never transferable as GREEN evidence. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index f85c99a074..1f07ccb667 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -11,7 +11,7 @@ | Gap ID | 상태 | exact-head evidence | causal owner / next gate | |---|---|---|---| -| CONTROL-OPENCODE-VCS-PYROOT-01 | **Proposed / source repaired; hosted exact-head validation pending** | `contextual-orchestrator#1149@684cf28f`의 중앙 [OpenCode run 34701472466](https://github.com/ContextualWisdomLab/.github/actions/runs/34701472466) `coverage-evidence` job `103574547257`은 PR 코드를 실행하기 전에 immutable `fast-mlsirm@09f762d`의 `python/fast_mlsirm` import root를 찾지 못해 종료했다. 같은 head의 제품 테스트는 `3602 passed, 2 skipped`, native CodeQL·fuzz·SBOM·SAST·Strix는 성공했다. | `.github`의 `opencode-review-dispatch.yml`이 root/`src/`만 허용한 계약 drift를 소유한다. RED contract `ed82ddb5`와 최소 source repair `af04581c` 뒤, 이 문서 head의 integrated CI가 GREEN이고 protected `main`에 ordinary merge된 다음 affected consumer exact head를 다시 검증한다. | +| CONTROL-OPENCODE-VCS-PYROOT-01 | **Proposed / source repaired; hosted exact-head validation pending** | `contextual-orchestrator#1149@684cf28f`의 중앙 [OpenCode run 34701472466](https://github.com/ContextualWisdomLab/.github/actions/runs/34701472466) `coverage-evidence` job `103574547257`은 PR 코드를 실행하기 전에 immutable `fast-mlsirm@09f762d`의 `python/fast_mlsirm` import root를 찾지 못해 종료했다. 같은 head의 제품 테스트는 `3602 passed, 2 skipped`, native CodeQL·fuzz·SBOM·SAST·Strix는 성공했다. | `.github`의 `opencode-review-dispatch.yml`이 root/`src/`만 허용한 계약 drift를 소유한다. RED contract `b1fe97c4`와 최소 source repair `af04581c` 뒤, 이 문서 head의 integrated CI가 GREEN이고 protected `main`에 ordinary merge된 다음 affected consumer exact head를 다시 검증한다. | ## 1. 근거와 범위 From 683cb053b3c6f1c7b3f293a74263ac9b13e9bdf1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 01:11:46 +0900 Subject: [PATCH 66/80] fix(opencode): advance trusted dispatch blob pin --- tests/test_pr_review_autofix_nvidia_nim_contract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index 2e733ac9e9..ddf06bfbd1 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -17,7 +17,7 @@ DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md") CHANGELOG = Path("CHANGELOG.md") REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") -REVIEW_DISPATCH_BLOB_SHA = "d86497b3f43bebbabbb4f504eb5132cdf3b7b293" +REVIEW_DISPATCH_BLOB_SHA = "f315683208d57ba89a2942502c525abe7355e2fd" def _workflow_text(path: Path) -> str: From 7e3b1a40c7519409a6187a32332bfea210012487 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 01:12:21 +0900 Subject: [PATCH 67/80] docs(opencode): record trusted blob RED and repair --- CHANGELOG.md | 2 +- docs/doctoring/opencode-vcs-python-source-root.md | 1 + docs/product-technical-gap-baseline.md | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28bb9fc824..2606bfc6d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ### OpenCode coverage admits immutable `python/` VCS source roots -- Central OpenCode coverage run [34701472466](https://github.com/ContextualWisdomLab/.github/actions/runs/34701472466) failed before executing `contextual-orchestrator#1149`: the trusted image builder resolved VCS packages only from repository root or `src/`, while the exact immutable `fast-mlsirm@09f762ded35786dd1078222a4577ff09d649816f` exposes `fast_mlsirm` from `python/fast_mlsirm`. The builder now admits the explicit `python/` source root, retains the one-and-only-one import-root invariant, symlink/namespace/compiled-artifact/installed-metadata rejection, exact commit verification, and the later credential-free networkless sandbox. Contract tests pin both package and single-module `python/` layouts. Refs `contextual-orchestrator#1149`. +- Central OpenCode coverage run [34701472466](https://github.com/ContextualWisdomLab/.github/actions/runs/34701472466) failed before executing `contextual-orchestrator#1149`: the trusted image builder resolved VCS packages only from repository root or `src/`, while the exact immutable `fast-mlsirm@09f762ded35786dd1078222a4577ff09d649816f` exposes `fast_mlsirm` from `python/fast_mlsirm`. The builder now admits the explicit `python/` source root, retains the one-and-only-one import-root invariant, symlink/namespace/compiled-artifact/installed-metadata rejection, exact commit verification, and the later credential-free networkless sandbox. Contract tests pin both package and single-module `python/` layouts. Refs `contextual-orchestrator#1149`. Exact-head Runtime Quality [job 103581110552](https://github.com/ContextualWisdomLab/.github/actions/runs/34704176931/job/103581110552) then caught the required independent workflow-blob trust pin still naming the predecessor blob; `683cb053` advances only that pin to exact blob `f315683208d57ba89a2942502c525abe7355e2fd`. ### Failed-check finding names the Strix sandbox instead of the gateway diff --git a/docs/doctoring/opencode-vcs-python-source-root.md b/docs/doctoring/opencode-vcs-python-source-root.md index a0ac60255d..8e6cb3d51c 100644 --- a/docs/doctoring/opencode-vcs-python-source-root.md +++ b/docs/doctoring/opencode-vcs-python-source-root.md @@ -20,6 +20,7 @@ Rejected alternatives were: changing the consumer's valid immutable dependency p - RED commit `b1fe97c477b56e148afbeeaed9a6b74338994b6b` requires both package and single-module `python/` candidates in the published workflow contract. - Repair commit `af04581cea4ffc038c881c6ad101ea3e5842a664` adds those candidates and the corresponding `python_root` mapping. +- Hosted Runtime Quality [job `103581110552`](https://github.com/ContextualWisdomLab/.github/actions/runs/34704176931/job/103581110552) then failed the independent pairing contract because the changed workflow blob `f315683208d57ba89a2942502c525abe7355e2fd` no longer matched the reviewed predecessor pin. Commit `683cb053b3c6f1c7b3f293a74263ac9b13e9bdf1` advances only that exact pin; no hash check is removed or relaxed. - Hosted current-head tests, security, CodeQL, and independent review remain required. Only after ordinary protected-main integration may affected consumers rerun OpenCode; the predecessor run is never transferable as GREEN evidence. ## Follow-up diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 1f07ccb667..a251563016 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -11,7 +11,7 @@ | Gap ID | 상태 | exact-head evidence | causal owner / next gate | |---|---|---|---| -| CONTROL-OPENCODE-VCS-PYROOT-01 | **Proposed / source repaired; hosted exact-head validation pending** | `contextual-orchestrator#1149@684cf28f`의 중앙 [OpenCode run 34701472466](https://github.com/ContextualWisdomLab/.github/actions/runs/34701472466) `coverage-evidence` job `103574547257`은 PR 코드를 실행하기 전에 immutable `fast-mlsirm@09f762d`의 `python/fast_mlsirm` import root를 찾지 못해 종료했다. 같은 head의 제품 테스트는 `3602 passed, 2 skipped`, native CodeQL·fuzz·SBOM·SAST·Strix는 성공했다. | `.github`의 `opencode-review-dispatch.yml`이 root/`src/`만 허용한 계약 drift를 소유한다. RED contract `b1fe97c4`와 최소 source repair `af04581c` 뒤, 이 문서 head의 integrated CI가 GREEN이고 protected `main`에 ordinary merge된 다음 affected consumer exact head를 다시 검증한다. | +| CONTROL-OPENCODE-VCS-PYROOT-01 | **Proposed / source repaired; hosted exact-head validation pending** | `contextual-orchestrator#1149@684cf28f`의 중앙 [OpenCode run 34701472466](https://github.com/ContextualWisdomLab/.github/actions/runs/34701472466) `coverage-evidence` job `103574547257`은 PR 코드를 실행하기 전에 immutable `fast-mlsirm@09f762d`의 `python/fast_mlsirm` import root를 찾지 못해 종료했다. 같은 head의 제품 테스트는 `3602 passed, 2 skipped`, native CodeQL·fuzz·SBOM·SAST·Strix는 성공했다. | `.github`의 `opencode-review-dispatch.yml`이 root/`src/`만 허용한 계약 drift를 소유한다. RED contract `b1fe97c4`, 최소 source repair `af04581c`, exact workflow-blob trust pin `683cb053` 뒤, 이 문서 head의 integrated CI가 GREEN이고 protected `main`에 ordinary merge된 다음 affected consumer exact head를 다시 검증한다. | ## 1. 근거와 범위 From 109b114c71f78fd44084f33617759150dfaede75 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 13:01:22 +0900 Subject: [PATCH 68/80] fix(security-scan): use current OSV output flags and bind SARIF upload to the head checkout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #2132. The pinned ghcr.io/google/osv-scanner-action:v2.5.1 image warns on every run that `--output` is deprecated in favor of `--output-file` (scanner) and `--output-files` (reporter); a bare `--output-files=` defaults to the sarif format in v2.5.1 (cmd/osv-reporter/main.go: format := "sarif" unless a `format:` prefix is given), so the reporter output is byte-for-byte the same file. The exact base/head checkouts live in `source`, not the workspace root, so `upload-sarif` resolved commit identity against a non-repository and logged "does not appear to be a git repository" twice before falling back to the caller-supplied sha; `checkout_path` now names the real checkout. Contract: the new test pins the non-deprecated flags, rejects the deprecated ones, and derives the required `checkout_path` from the head checkout step's own `path:` (removing the binding makes it fail — verified RED before GREEN). Co-Authored-By: Claude Fable 5.1 --- .github/workflows/security-scan.yml | 14 ++++--- .../test_required_workflow_queue_contract.py | 37 +++++++++++++++++-- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 500e22b4ab..e04d7bf8f3 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -164,7 +164,7 @@ jobs: with: scan-args: | --format=json - --output=old-results.json + --output-file=old-results.json --maven-registry=https://maven-central.storage-download.googleapis.com/maven2 --no-resolve --allow-no-lockfiles @@ -182,7 +182,7 @@ jobs: with: scan-args: | --format=json - --output=old-results.json + --output-file=old-results.json --no-resolve --allow-no-lockfiles -r @@ -215,7 +215,7 @@ jobs: with: scan-args: | --format=json - --output=new-results.json + --output-file=new-results.json --maven-registry=https://maven-central.storage-download.googleapis.com/maven2 --no-resolve --allow-no-lockfiles @@ -233,7 +233,7 @@ jobs: with: scan-args: | --format=json - --output=new-results.json + --output-file=new-results.json --no-resolve --allow-no-lockfiles -r @@ -286,7 +286,7 @@ jobs: uses: google/osv-scanner-action/osv-reporter-action@8e5cf47b818121e8b405931c82126c2630b0b20d # v2.3.8 with: scan-args: | - --output=results.sarif + --output-files=results.sarif --old=old-results.json --new=new-results.json --gh-annotations=true @@ -323,6 +323,10 @@ jobs: uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 with: sarif_file: results.sarif + # The exact head checkout lives in `source`, not the workspace root; + # without this binding upload-sarif logs "does not appear to be a git + # repository" twice and falls back to server-derived commit identity. + checkout_path: ${{ github.workspace }}/source # results.sarif is produced after checkout of the pull request head. # Uploading it against refs/pull/*/merge can race GitHub's synthetic # merge ref and fail with "commit_oid is not a merge commit". diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 19fe6b0f7f..157556502a 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -1674,8 +1674,8 @@ def test_security_scan_preserves_base_output_across_cross_fork_checkout() -> Non assert workflow.count("--allow-no-lockfiles") == 4 assert workflow.count("path: source") == 2 - assert workflow.count("--output=old-results.json") == 2 - assert workflow.count("--output=new-results.json") == 2 + assert workflow.count("--output-file=old-results.json") == 2 + assert workflow.count("--output-file=new-results.json") == 2 assert workflow.count("source/") == 4 assert "clean: false" not in workflow assert "test -s old-results.json" in workflow @@ -1732,12 +1732,41 @@ def test_osv_scan_logs_and_retries_without_transitive_resolution_on_resolver_fai "Retry head OSV without transitive resolution\n if: steps.osv_head.outcome == 'failure'\n continue-on-error: true" in workflow ) - assert "--output=old-results.json" in workflow - assert "--output=new-results.json" in workflow + assert "--output-file=old-results.json" in workflow + assert "--output-file=new-results.json" in workflow assert "Print OSV findings being compared" in workflow assert "OSV {label} scan produced {len(findings)} finding(s)" in workflow +def test_osv_scan_uses_current_output_flags_and_binds_sarif_checkout_path() -> None: + """Drop deprecated OSV output flags and bind upload-sarif to the real checkout. + + Live evidence (ContextualWisdomLab/.github#2132): the pinned + `ghcr.io/google/osv-scanner-action:v2.5.1` image warns + `--output has been deprecated in favor of --output-file` (scanner) and + `... in favor of --output-files` (reporter), and `upload-sarif` logged + twice that the workspace root "does not appear to be a git repository" + because the exact head is checked out into `source`. A bare + `--output-files=` defaults to the sarif format in v2.5.1, so the + reporter's output is unchanged. The checkout-path assertion is the + negative fixture: an absent or wrong `checkout_path` fails here instead + of silently relying on server-derived commit identity. + """ + workflow = workflow_text("security-scan.yml") + + assert workflow.count("--output-file=old-results.json") == 2 + assert workflow.count("--output-file=new-results.json") == 2 + assert "--output-files=results.sarif" in workflow + assert "--output=old-results.json" not in workflow + assert "--output=new-results.json" not in workflow + assert "--output=results.sarif" not in workflow + + head_checkout = workflow_step(workflow, "Checkout head") + checkout_dir = re.search(r"(?m)^\s+path: (\S+)$", head_checkout).group(1) + upload_step = workflow_step(workflow, "Upload OSV SARIF to code scanning") + assert f"checkout_path: ${{{{ github.workspace }}}}/{checkout_dir}" in upload_step + + def test_osv_sarif_upload_is_marked_comprehensive_after_clean_comparison( tmp_path: Path, ) -> None: From 8db9a1db797c51be7feb576513bc80c9df5540d1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 13:04:42 +0900 Subject: [PATCH 69/80] fix(sast-semgrep): fold the changed-scope gate into its single consumer job One consumer, one runner: the standalone `changed-scope` job cost a second runner allocation per PR org-wide purely to compute two booleans for the `semgrep` job. The classifier now runs as a step inside `semgrep` (after harden-runner), the expensive steps gate on `steps.scope.outputs.code`, and the enforce step carries the same guard so a step-skipped scan's empty `rc` cannot fail a doc-only PR. The job keeps `if: github.event.action != 'closed'` with no needs-output term, so a doc-only run still concludes `success` (required-workflow-path-filter-boundary.md). strix.yml is left alone (hot-file collision zone). Measured in #1904; contract tests updated. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/sast-semgrep.yml | 50 ++++++++----------- docs/product-technical-gap-baseline.md | 21 +++++++- tests/test_docs_only_pr_runner_admission.py | 50 ++++++++++++++++--- ...required_security_runner_image_contract.py | 8 +-- 4 files changed, 89 insertions(+), 40 deletions(-) diff --git a/.github/workflows/sast-semgrep.yml b/.github/workflows/sast-semgrep.yml index 12b7013da3..283eac0097 100644 --- a/.github/workflows/sast-semgrep.yml +++ b/.github/workflows/sast-semgrep.yml @@ -38,8 +38,8 @@ permissions: contents: read jobs: - changed-scope: - name: Detect changed scope + semgrep: + name: Semgrep (multi-language SAST) # The org ruleset IGNORES every `on:` filter (paths, branches, types) when it # runs this workflow in another repository, and a trigger-level skip would # leave `.github`'s classic required contexts Pending forever. Both @@ -47,16 +47,26 @@ jobs: # here and consumed through `needs`. See # docs/doctoring/required-workflow-path-filter-boundary.md. # Fails OPEN: an unreadable, empty, or truncated file list scans everything. + # The gate lives inside this job as a step-level guard (one runner, not two). if: github.event.action != 'closed' runs-on: ubuntu-24.04 - timeout-minutes: 5 permissions: contents: read pull-requests: read - outputs: - code: ${{ steps.scope.outputs.code }} - deps: ${{ steps.scope.outputs.deps }} + security-events: write + actions: read + env: + # Deterministic, no telemetry: registry rules are fetched but no scan data + # is sent back. + SEMGREP_SEND_METRICS: "off" + # Semgrep OSS 1.169.0. Keep the immutable manifest reference in one + # place so hosted scans and local reproduction cannot drift. + SEMGREP_IMAGE: "semgrep/semgrep@sha256:2b33f46ba66cf8cc2ad59ccfa7d22951fd00c632c38f1339e84ec8e6e641a942" steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1 + with: + egress-policy: audit - name: Classify changed paths id: scope env: @@ -99,35 +109,15 @@ jobs: echo "code=${code}" >> "$GITHUB_OUTPUT" echo "deps=${deps}" >> "$GITHUB_OUTPUT" echo "changed-scope code=${code} deps=${deps}" - - semgrep: - name: Semgrep (multi-language SAST) - needs: changed-scope - if: github.event.action != 'closed' && needs.changed-scope.outputs.code == 'true' - runs-on: ubuntu-24.04 - permissions: - contents: read - security-events: write - actions: read - env: - # Deterministic, no telemetry: registry rules are fetched but no scan data - # is sent back. - SEMGREP_SEND_METRICS: "off" - # Semgrep OSS 1.169.0. Keep the immutable manifest reference in one - # place so hosted scans and local reproduction cannot drift. - SEMGREP_IMAGE: "semgrep/semgrep@sha256:2b33f46ba66cf8cc2ad59ccfa7d22951fd00c632c38f1339e84ec8e6e641a942" - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1 - with: - egress-policy: audit - name: Checkout exact submitted revision + if: steps.scope.outputs.code == 'true' uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} ref: ${{ github.event.pull_request.head.sha || github.sha }} persist-credentials: false - name: Verify exact submitted revision + if: steps.scope.outputs.code == 'true' env: EXPECTED_CHECKOUT_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name || github.repository }} EXPECTED_CHECKOUT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} @@ -140,6 +130,7 @@ jobs: fi echo "SAST_CHECKOUT scanner=semgrep repository=${EXPECTED_CHECKOUT_REPOSITORY} expected_sha=${EXPECTED_CHECKOUT_SHA} actual_sha=${actual_sha}" - name: Verify pinned Semgrep manifest + if: steps.scope.outputs.code == 'true' run: | set -euo pipefail if [[ "${SEMGREP_IMAGE}" =~ ^semgrep/semgrep@sha256:[0-9a-f]{64}$ ]]; then @@ -151,6 +142,7 @@ jobs: fi - name: Run Semgrep (SARIF) id: semgrep + if: steps.scope.outputs.code == 'true' run: | set +e echo "Using ${SEMGREP_IMAGE}" @@ -219,7 +211,7 @@ jobs: echo "SEMGREP_ENGINE_FAILURE rc=${SEMGREP_RC:-missing}: Semgrep failed without a WARNING/ERROR SARIF result; inspect the scan command output above." fi - name: Enforce Semgrep gate (fail on Medium+ findings) - if: always() && (steps.semgrep_sarif.outputs.finding_count != '0' || steps.semgrep.outputs.rc != '0') + if: always() && steps.scope.outputs.code == 'true' && (steps.semgrep_sarif.outputs.finding_count != '0' || steps.semgrep.outputs.rc != '0') env: SEMGREP_RC: ${{ steps.semgrep.outputs.rc }} SEMGREP_FINDING_COUNT: ${{ steps.semgrep_sarif.outputs.finding_count }} diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 1cc9e20313..966ac7ef25 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -3237,8 +3237,8 @@ intended contract before rewriting the assertion — left for a dedicated follow ## Items 15/16/17 measurement: `Detect changed scope` gate jobs — 2 of 3 are pure runner overhead — 2026-09-05 -**Status:** Measured, not yet fixed. Recorded so the fix is grounded in real numbers rather than the intuition -this measurement partly refuted. +**Status:** Measured 2026-09-05; `sast-semgrep.yml` fixed 2026-09-13 (below); `strix.yml` deferred. Recorded so +the fix is grounded in real numbers rather than the intuition this measurement partly refuted. **Why measured.** Items 15/16/17 ask to remove needlessly-triggered workflows, consolidate workflow files ("bootup에도 시간이 듦"), and cut redundant steps; the standing complaint is the org's 60-concurrent-job @@ -3353,3 +3353,20 @@ queries the check-runs API at its own time, order-independently. The implementin their change was safe because they had scoped it narrowly, not because they had checked for the name collision — which is the more useful lesson: **a job name is unique only within one workflow file, and the same name in another file can carry the opposite safety property.** + +**Fixed for `sast-semgrep.yml`, 2026-09-13.** The standalone `changed-scope` job is gone; its +"Classify changed paths" step now runs inside the single consumer `semgrep` (after `harden-runner`, +which must audit the classifier's own `gh api` egress) and the four expensive steps plus the final +"Enforce Semgrep gate" step carry `steps.scope.outputs.code == 'true'`. The job keeps +`if: github.event.action != 'closed'` with no `needs.` term, so a doc-only PR's run still executes one +job that concludes `success` -- the load-bearing property from +[`required-workflow-path-filter-boundary.md`](doctoring/required-workflow-path-filter-boundary.md) is +preserved, and neither `Detect changed scope` nor `Semgrep (multi-language SAST)` is among `.github`'s +classic required contexts, so nothing goes Pending there. One trap the first draft would have shipped: +the enforce step's `always() && (... || steps.semgrep.outputs.rc != '0')` evaluates `rc` as the empty +string when `Run Semgrep` is step-skipped, which is `!= '0'` and would have failed every doc-only PR; +the guard on that step is what makes the fold safe. Net: one runner allocation per PR for this +workflow instead of two, org-wide. `strix.yml` (the other single-consumer gate) is deliberately left +alone -- it is a documented multi-PR hot-file collision zone. Contract: +`tests/test_docs_only_pr_runner_admission.py::test_sast_semgrep_folds_the_gate_into_its_single_consumer_at_step_level`, +`tests/test_required_security_runner_image_contract.py`. diff --git a/tests/test_docs_only_pr_runner_admission.py b/tests/test_docs_only_pr_runner_admission.py index 49631d2a19..674b984b63 100644 --- a/tests/test_docs_only_pr_runner_admission.py +++ b/tests/test_docs_only_pr_runner_admission.py @@ -31,9 +31,11 @@ WORKFLOWS_DIR = REPO_ROOT / ".github/workflows" # The required workflows that keep the canonical `changed-scope` gate job. +# `sast-semgrep.yml` has only one consumer job, so it folds the classifier +# into that job as a step-level guard instead of a standalone job -- see +# GATED_JOBS below. GATE_WORKFLOWS = ( "security-scan.yml", - "sast-semgrep.yml", "strix.yml", ) @@ -52,7 +54,6 @@ # output, keyed by workflow filename. GATED_JOBS = { "security-scan.yml": ("osv-scan", "dependency-review", "trivy-fs", "scorecard"), - "sast-semgrep.yml": ("semgrep",), "strix.yml": ("strix",), } @@ -86,7 +87,7 @@ def _on_block(workflow: str) -> str: def test_gate_job_is_byte_identical_across_the_five_workflows_apart_from_if(): - """The `changed-scope` block must not drift between its five copies.""" + """The `changed-scope` block must not drift between every gate copy.""" normalized_blocks = set() for filename in GATE_WORKFLOWS: workflow = _read(filename) @@ -110,7 +111,7 @@ def test_gate_job_and_codeql_scope_step_share_one_doc_pattern_line(): `COPYING.txt`/`NOTICE`/`NOTICE.txt` names. """ doc_pattern_lines = set() - for filename in (*GATE_WORKFLOWS, "codeql-pr.yml"): + for filename in (*GATE_WORKFLOWS, "sast-semgrep.yml", "codeql-pr.yml"): workflow = _read(filename) matches = [ line for line in workflow.splitlines() if "*.md|*.markdown" in line @@ -208,8 +209,8 @@ def test_codeql_pr_gates_analyze_head_at_step_level_not_job_level(): def test_each_gate_workflow_keeps_an_always_admitted_job(): """A fully-skipped run must conclude `success`, never `skipped`. - Every one of the five workflows needs at least one job with no `needs:` - and no needs-output-dependent `if:` -- the `changed-scope` job itself + Every gate workflow needs at least one job with no `needs:` and no + needs-output-dependent `if:` -- the `changed-scope` job itself qualifies -- so a doc-only PR's run still has a job that runs and succeeds instead of every job skipping and the run itself reporting `skipped` (an undocumented conclusion for a required check). @@ -220,3 +221,40 @@ def test_each_gate_workflow_keeps_an_always_admitted_job(): job_if = re.search(r"(?m)^ if: (.*)$", block) assert job_if is not None, filename assert "needs." not in job_if.group(1), filename + + +def test_sast_semgrep_folds_the_gate_into_its_single_consumer_at_step_level(): + """`sast-semgrep.yml` has one consumer, so the gate is a step, not a job. + + A standalone `changed-scope` job cost a second runner allocation per PR + purely to compute two booleans for one downstream job (measured in + docs/product-technical-gap-baseline.md, "Items 15/16/17 measurement"). + Folding it into `semgrep` keeps the load-bearing property -- the job + still runs and concludes `success` on a doc-only PR -- while the + expensive steps gate on the classifier step's output. The final gate + step must also carry that guard: a step-skipped `Run Semgrep` leaves + `steps.semgrep.outputs.rc` empty, which is `!= '0'`. + """ + workflow = _read("sast-semgrep.yml") + # The classifier's own log lines keep saying "changed-scope" (byte-for-byte + # verbatim across every copy, see test_gate_job_and_codeql_scope_step_share_ + # one_doc_pattern_line); what must be gone is the standalone JOB. + assert "changed-scope:" not in workflow + assert "needs: changed-scope" not in workflow + assert "needs.changed-scope" not in workflow + assert workflow.count("runs-on: ubuntu-24.04") == 1 + + semgrep = _top_level_job_block(workflow, "semgrep") + assert not re.search(r"(?m)^ needs:", semgrep) + job_if = re.search(r"(?m)^ if: (.*)$", semgrep) + assert job_if is not None + assert job_if.group(1) == "github.event.action != 'closed'" + assert "pull-requests: read" in semgrep + assert "id: scope" in semgrep + assert semgrep.count("steps.scope.outputs.code == 'true'") == 5 + assert ( + "if: always() && steps.scope.outputs.code == 'true' && " + "(steps.semgrep_sarif.outputs.finding_count != '0' || steps.semgrep.outputs.rc != '0')" + ) in semgrep + # Harden-runner audits egress and must precede the classifier's gh api call. + assert semgrep.index("Harden the runner") < semgrep.index("Classify changed paths") diff --git a/tests/test_required_security_runner_image_contract.py b/tests/test_required_security_runner_image_contract.py index 2b48f66251..d20c0c3a98 100644 --- a/tests/test_required_security_runner_image_contract.py +++ b/tests/test_required_security_runner_image_contract.py @@ -28,13 +28,15 @@ def test_sast_semgrep_uses_explicit_supported_image(self) -> None: `#1656` removed the sibling `cancel-closed-pr-runs` no-op job (it only duplicated PR-stable workflow concurrency), leaving one runner - job in this workflow instead of two. It is 2, not 1, again after the + job in this workflow instead of two. It was 2, not 1, again after the `changed-scope` gate job was added to skip doc-only PR scope (org - ruleset 18156473 ignores trigger-level path filters). + ruleset 18156473 ignores trigger-level path filters). The count + returned to 1 when that `changed-scope` job was folded into the + `semgrep` job as a step-level guard (one consumer, one runner). """ workflow = SAST_SEMGREP.read_text(encoding="utf-8") self.assertNotIn("runs-on: ubuntu-latest", workflow) - self.assertEqual(workflow.count("runs-on: ubuntu-24.04"), 2) + self.assertEqual(workflow.count("runs-on: ubuntu-24.04"), 1) if __name__ == "__main__": From 81a2cc61ae898974f8294a6e0c3c665857021ba9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 13:16:50 +0900 Subject: [PATCH 70/80] fix(pingora): admit bounded HWPX evidence documents without UTF-8 decoding Refs ContextualWisdomLab/.github#2116. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 4 + .../pingora-hwpx-evidence-admission.md | 40 +++++++ scripts/ci/pingora_edge_policy.py | 44 +++++++- tests/test_pingora_hwpx_evidence.py | 104 ++++++++++++++++++ 4 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 docs/doctoring/pingora-hwpx-evidence-admission.md create mode 100644 tests/test_pingora_hwpx_evidence.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 707c18532e..ce7fbe18a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Pingora edge policy admits HWPX evidence documents without UTF-8 decoding + +- `scripts/ci/pingora_edge_policy.py`'s `BINARY_DOCUMENT_MAGIC` only knew `.pdf` and `.png`, and `_is_binary_documentation_asset` only admitted a `doc`/`docs`/`documentation` directory, so a ZIP-based `.hwpx` evidence attachment under `evidence/` matched neither rule and fell through to the strict UTF-8 decode every other candidate gets. Observed on ContextualWisdomLab/late-life-anxiety-reanalysis#10, head `a1cd5bc6783c6510dfcf937f523c733366e82213`, run `34700409497`, job `103571044859`: "Pingora edge policy could not establish complete evidence: Runtime policy candidate evidence/reviewer_response_draft.hwpx is not valid UTF-8". The fix adds `.hwpx` (`PK\x03\x04`) to `BINARY_DOCUMENT_MAGIC` and extends `_is_binary_documentation_asset` to admit an `.hwpx` under an `evidence` path segment, gated on a bounded container check in the new `_is_complete_hwpx` -- unprefixed ZIP, exact EOCD record, unique members with `mimetype` first, a stored (not deflated) `mimetype` entry exactly `application/hwp+zip`, and a non-empty, unencrypted `Contents/content.hpf` manifest -- so no document body is ever parsed or rendered and no malware inspection is implied. The runtime-path guard and the Nginx-runtime-text fallback scan for disguised or malformed archives are unchanged. `tests/test_pingora_hwpx_evidence.py` runs the production policy boundary offline: RED (test-only apply) showed 3 failing / 19 passing; GREEN (full patch) showed 90 passing across that file plus `tests/test_pingora_edge_policy.py` and `tests/test_pingora_edge_workflow_contract.py`. Branch coverage of the touched module is 100% (388 statements, 174 branches, 0 missed) and `interrogate scripts/ci -q` reports 100.0% docstrings. Hosted acceptance still requires a newly loaded central source SHA to re-run the consumer's exact head bootstrap. Refs ContextualWisdomLab/.github#2116. + ### Failed-check finding names the Strix sandbox instead of the gateway - `opencode-review-dispatch.yml`'s `emit_strix_provider_failure_finding` rendered one fixed finding for every `STRIX_PROVIDER_UNAVAILABLE` line, whose Root cause read "The contextual-orchestrator gateway or its discovered provider pool was unavailable for this run". `#1953` had just given the Strix sandbox bootstrap failure its own second verdict token (`STRIX_SANDBOX_UNAVAILABLE`) precisely because that attribution is wrong for it -- the sandbox container never reaches its Caido proxy, so the run dies before the gateway serves anything -- and this consumer re-applied the wrong attribution one step downstream, into the review findings and the failure census. The emitter now branches on the second token: a sandbox verdict gets a finding that names Strix's sandbox, says the verdict does not name the gateway, and tells the reader not to change gateway or provider configuration on its strength. A `STRIX_PROVIDER_UNAVAILABLE` line without the token keeps its existing text verbatim, so the gateway class has no regression surface. No test covered this finding text at all before (`gateway or its discovered provider pool` matched nothing under `tests/`); `tests/test_opencode_dispatch_strix_sandbox_finding.py` now runs the production emitter from the published run block and pins both directions plus the no-signal case. Refs #1953, #1935. diff --git a/docs/doctoring/pingora-hwpx-evidence-admission.md b/docs/doctoring/pingora-hwpx-evidence-admission.md new file mode 100644 index 0000000000..a8a0f9e98d --- /dev/null +++ b/docs/doctoring/pingora-hwpx-evidence-admission.md @@ -0,0 +1,40 @@ +# Pingora edge policy admits HWPX evidence documents (#2116) + +`scripts/ci/pingora_edge_policy.py` rejected the consumer's HWPX evidence +attachment before a merge verdict. `BINARY_DOCUMENT_MAGIC` only knew +`.pdf`/`.png`, and `_is_binary_documentation_asset` only admitted a +`doc`/`docs`/`documentation` directory, so the ZIP-based `.hwpx` under +`evidence/` matched neither rule and fell through to the strict UTF-8 +decode every other candidate gets. + +Source baseline: `fb17ef556f94f673234aa557254ae52779e9a7b0`. Consumer +evidence: ContextualWisdomLab/late-life-anxiety-reanalysis#10, head +`a1cd5bc6783c6510dfcf937f523c733366e82213`, run `34700409497`, job +`103571044859`. Reported failure: "Pingora edge policy could not establish +complete evidence: Runtime policy candidate +evidence/reviewer_response_draft.hwpx is not valid UTF-8". + +The repair adds `.hwpx` (`PK\x03\x04`) to `BINARY_DOCUMENT_MAGIC` and lets +`_is_binary_documentation_asset` also admit an `.hwpx` under an `evidence` +path segment, gated on a bounded HWPX container check: unprefixed ZIP, +exact EOCD record, unique members with `mimetype` first, a stored (not +deflated) `mimetype` entry exactly `application/hwp+zip`, and a non-empty, +unencrypted `Contents/content.hpf` manifest. Format evidence only -- no +document rendering or malware inspection. The runtime-path guard and the +Nginx-runtime-text fallback scan for disguised/malformed archives are +retained unchanged. + +Test evidence (offline, this branch): RED (test-only apply) 3 failing / 19 +passing in `tests/test_pingora_hwpx_evidence.py`; GREEN (full patch) 90 +passing across that file plus `tests/test_pingora_edge_policy.py` and +`tests/test_pingora_edge_workflow_contract.py`. Branch coverage of the +touched module: 100% (388 statements, 174 branches, 0 missed). +`interrogate scripts/ci -q`: 100.0% docstrings. Full suite passes, no new skips or warnings. + +Hosted acceptance still requires a newly loaded central source SHA to +re-run the consumer's exact head bootstrap; local tests prove the declared +classification and container logic, not a hosted admission outcome. + +## Reference + +Hancom. (n.d.). *한/글 문서 파일 형식: HWPX 포맷 구조 살펴보기*. https://tech.hancom.com/hwpxformat/ diff --git a/scripts/ci/pingora_edge_policy.py b/scripts/ci/pingora_edge_policy.py index 33e58ed876..eb0e3a741a 100644 --- a/scripts/ci/pingora_edge_policy.py +++ b/scripts/ci/pingora_edge_policy.py @@ -11,10 +11,12 @@ import argparse import base64 +import io import json import os import re import sys +import zipfile import zlib from dataclasses import dataclass from pathlib import PurePosixPath @@ -40,6 +42,7 @@ # (this org's own "attach the relevant paper PDF" convention) for a reason # that has nothing to do with the Nginx runtime policy this module enforces. BINARY_DOCUMENT_MAGIC = { + ".hwpx": (b"PK\x03\x04",), ".pdf": (b"%PDF-",), ".png": (b"\x89PNG\r\n\x1a\n",), } @@ -229,7 +232,7 @@ def _is_binary_documentation_asset(changed: ChangedFile) -> bool: pure = PurePosixPath(changed.path) return ( pure.suffix.lower() in BINARY_DOCUMENT_MAGIC - and _is_known_documentation_path(pure) + and (_is_known_documentation_path(pure) or (pure.suffix.lower() == ".hwpx" and "evidence" in (part.lower() for part in pure.parts))) and _runtime_path_rule(changed.path) is None ) @@ -453,9 +456,48 @@ def _binary_documentation_evidence_confirms( suffix = PurePosixPath(changed.path).suffix.lower() if suffix == ".png": return _is_complete_png(raw) + if suffix == ".hwpx": + return _is_complete_hwpx(raw) return raw.startswith(BINARY_DOCUMENT_MAGIC[suffix]) +def _is_complete_hwpx(raw: bytes) -> bool: + """Confirm a bounded HWPX container without extracting document content. + + Require an unprefixed ZIP, its exact end record, unique members, and the + stored HWPX MIME marker plus an unencrypted package manifest. This is + format evidence, not XML document validation or malware inspection. + """ + if not raw.startswith(BINARY_DOCUMENT_MAGIC[".hwpx"][0]): + return False + try: + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + archive_entries = archive.infolist() + member_names = [member_info.filename for member_info in archive_entries] + end_offset = len(raw) - 22 - len(archive.comment) + if end_offset < 0 or raw[end_offset:end_offset + 4] != b"PK\x05\x06": + return False + if int.from_bytes(raw[end_offset + 20:end_offset + 22], "little") != len(archive.comment): + return False + if not archive_entries or archive_entries[0].header_offset != 0: + return False + if member_names[0] != "mimetype" or len(member_names) != len(set(member_names)): + return False + mimetype_info = archive.getinfo("mimetype") + manifest_info = archive.getinfo("Contents/content.hpf") + expected_mimetype = b"application/hwp+zip" + if mimetype_info.flag_bits & 1 or manifest_info.flag_bits & 1: + return False + if mimetype_info.compress_type != zipfile.ZIP_STORED or mimetype_info.file_size != len(expected_mimetype): + return False + if manifest_info.is_dir() or manifest_info.file_size == 0: + return False + with archive.open(mimetype_info) as mimetype_stream: + return mimetype_stream.read(len(expected_mimetype) + 1) == expected_mimetype + except (KeyError, UnicodeError, OSError, ValueError, NotImplementedError, zipfile.BadZipFile): + return False + + def _png_unfilter_row(filtered: bytes, previous: bytes, filter_type: int, bytes_per_pixel: int) -> bytes: """Reconstruct one PNG scanline for bounded indexed-pixel validation.""" diff --git a/tests/test_pingora_hwpx_evidence.py b/tests/test_pingora_hwpx_evidence.py new file mode 100644 index 0000000000..aebbc5f96f --- /dev/null +++ b/tests/test_pingora_hwpx_evidence.py @@ -0,0 +1,104 @@ +"""Exercise HWPX admission through the production policy boundary offline.""" +import base64 +import io +import zipfile + +import pytest +from tests.test_pingora_edge_policy import policy + + +def hwpx_archive(*, mime_value=b"application/hwp+zip", manifest_value=b"", compression_type=zipfile.ZIP_STORED): + """Create a deterministic, non-sensitive format-boundary fixture.""" + archive_buffer = io.BytesIO() + with zipfile.ZipFile(archive_buffer, "w") as archive_file: + mime_info = zipfile.ZipInfo("mimetype") + mime_info.compress_type = compression_type + archive_file.writestr(mime_info, mime_value) + if manifest_value is not None: + archive_file.writestr(zipfile.ZipInfo("Contents/content.hpf"), manifest_value) + return archive_buffer.getvalue() + + +def evaluate_bytes(file_path, file_bytes, *, patch_value=None, file_status="added"): + """Supply only in-memory GitHub metadata and exact-head content.""" + def open_evidence(request_url, request_token): + assert request_token == "offline-fixture" + if "/files?" in request_url: + file_entry = {"filename": file_path, "status": file_status} + if patch_value is not None: + file_entry["patch"] = patch_value + return [file_entry] + assert "?ref=" + "a" * 40 in request_url + return {"type": "file", "encoding": "base64", "size": len(file_bytes), + "content": base64.b64encode(file_bytes).decode("ascii")} + return policy.evaluate_pull_request(api_url="https://api.github.com", + repository="ContextualWisdomLab/example", pull_request=2116, head_sha="a" * 40, + event_action="opened", token="offline-fixture", opener=open_evidence) + + +@pytest.mark.parametrize("file_path", ["evidence/reviewer_response_draft.hwpx", "docs/paper.hwpx", "Evidence/PAPER.HWPX"]) +def test_valid_hwpx_is_admitted_at_document_and_consumer_paths(file_path): + """Recognize HWPX at the exact consumer directory, without renaming it.""" + assert evaluate_bytes(file_path, hwpx_archive()) == () + + +@pytest.mark.parametrize("file_bytes", [ + b"PK\x03\x04\xff", hwpx_archive()[:-10], + b"#!/bin/sh\ncat /etc/nginx/nginx.conf\n" + hwpx_archive(), + hwpx_archive() + b"\ncat /etc/nginx/nginx.conf\n", + hwpx_archive(mime_value=b"application/zip"), + hwpx_archive(manifest_value=None), hwpx_archive(manifest_value=b""), + hwpx_archive(compression_type=zipfile.ZIP_DEFLATED), +]) +def test_malformed_or_disguised_archive_is_not_exempt(file_bytes): + """Unsupported format evidence fails closed instead of bypassing scanning.""" + with pytest.raises(policy.PolicyError): + evaluate_bytes("evidence/reviewer_response_draft.hwpx", file_bytes) + + +@pytest.mark.parametrize("file_path", ["evidence/paper.hwpx", "docs/paper.hwpx", "scripts/paper.hwpx"]) +@pytest.mark.parametrize("patch_value", [None, "+cat /etc/nginx/nginx.conf"]) +def test_text_renamed_to_hwpx_preserves_runtime_scan(file_path, patch_value): + """Neither the suffix nor patch absence hides actual Nginx runtime text.""" + violations = evaluate_bytes(file_path, b"cat /etc/nginx/nginx.conf\n", patch_value=patch_value) + assert [violation.rule for violation in violations] == ["nginx_runtime_path"] + + +def test_hwpx_does_not_expand_text_document_exemptions(): + """The consumer evidence directory does not exempt ordinary configuration.""" + violations = evaluate_bytes("evidence/config.txt", b"cat /etc/nginx/nginx.conf\n") + assert [violation.rule for violation in violations] == ["nginx_runtime_path"] + + +def test_hwpx_in_runtime_path_remains_unavailable(): + """A format exception cannot exempt an active runtime location.""" + with pytest.raises(policy.PolicyError): + evaluate_bytes("docs/nginx/paper.hwpx", hwpx_archive()) + + +def test_removed_hwpx_does_not_load_deleted_content(): + """Deletion does not require unavailable final-head bytes.""" + assert evaluate_bytes("evidence/paper.hwpx", b"", file_status="removed") == () + + +def test_hwpx_rejects_inconsistent_comment_and_shifted_zip(): + """EOCD declarations and member offsets must bind to the actual bytes.""" + archive_bytes = hwpx_archive() + invalid_comment = archive_bytes[:-2] + b"\x01\x00" + for file_bytes in (invalid_comment, b"PK\x03\x04" + archive_bytes): + with pytest.raises(policy.PolicyError): + evaluate_bytes("evidence/paper.hwpx", file_bytes) + + +def test_hwpx_rejects_nonfirst_marker_and_encrypted_manifest(): + """A named marker alone cannot admit a reordered or encrypted package.""" + archive_buffer = io.BytesIO() + with zipfile.ZipFile(archive_buffer, "w") as archive_file: + archive_file.writestr(zipfile.ZipInfo("Contents/content.hpf"), b"") + archive_file.writestr(zipfile.ZipInfo("mimetype"), b"application/hwp+zip") + encrypted_bytes = bytearray(hwpx_archive()) + manifest_header = encrypted_bytes.rfind(b"PK\x01\x02") + encrypted_bytes[manifest_header + 8] |= 1 + for file_bytes in (archive_buffer.getvalue(), bytes(encrypted_bytes)): + with pytest.raises(policy.PolicyError): + evaluate_bytes("evidence/paper.hwpx", file_bytes) From 4ba16a454c2145f94f158bf7f21a702f34fc89a3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 13:18:43 +0900 Subject: [PATCH 71/80] fix(codeql): grant private consumers pull-request and status reads Refs ContextualWisdomLab/late-life-anxiety-reanalysis#10. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/codeql-pr.yml | 4 ++ CHANGELOG.md | 4 ++ ...ql-pr-private-consumer-read-permissions.md | 26 +++++++++++++ tests/test_codeql_pr_workflow_contract.py | 39 +++++++++++++++++++ 4 files changed, 73 insertions(+) create mode 100644 docs/doctoring/codeql-pr-private-consumer-read-permissions.md diff --git a/.github/workflows/codeql-pr.yml b/.github/workflows/codeql-pr.yml index c21c8446df..710a82c45f 100644 --- a/.github/workflows/codeql-pr.yml +++ b/.github/workflows/codeql-pr.yml @@ -152,6 +152,8 @@ jobs: permissions: contents: read id-token: write + pull-requests: read + statuses: read strategy: fail-fast: false matrix: ${{ fromJSON(needs.detect-languages.outputs.matrix) }} @@ -298,6 +300,8 @@ jobs: contents: read id-token: write actions: read + pull-requests: read + statuses: read steps: - name: Dispatch current-head CodeQL scan env: diff --git a/CHANGELOG.md b/CHANGELOG.md index 707c18532e..f73166f44e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### CodeQL required workflow denies private consumers a read they need for their own PR + +- `.github/workflows/codeql-pr.yml`'s `analyze-head` and `dispatch-current-head` jobs called `gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}"` and later `repos/${TARGET_REPOSITORY}/commits/${PR_HEAD_SHA}/statuses` while holding only `contents: read` (plus `id-token: write`, and `actions: read` on the coordinator job) -- reads GitHub's REST contract gates behind the `pull-requests: read` and `statuses: read` fine-grained permissions on a private repository. Public consumers never surfaced this because GET on a public repository needs no such grant, but private consumer ContextualWisdomLab/late-life-anxiety-reanalysis's PR #10 (head `a1cd5bc6783c6510dfcf937f523c733366e82213`, run `34700410434`) failed both required-workflow jobs (`103571590442`, `103571810868`) at their first API call with `gh: Resource not accessible by integration (HTTP 403)`. Both jobs now also hold `pull-requests: read` and `statuses: read`; no write permission is added anywhere, and `actions: write` stays absent, so `tests/test_codeql_pr_workflow_contract.py::test_codeql_required_workflow_does_not_gain_actions_write` needed no change. New regression test `test_codeql_pr_jobs_hold_read_grants_private_consumers_need` pins the exact grant set. See `docs/doctoring/codeql-pr-private-consumer-read-permissions.md`. Refs ContextualWisdomLab/late-life-anxiety-reanalysis#10. + ### Failed-check finding names the Strix sandbox instead of the gateway - `opencode-review-dispatch.yml`'s `emit_strix_provider_failure_finding` rendered one fixed finding for every `STRIX_PROVIDER_UNAVAILABLE` line, whose Root cause read "The contextual-orchestrator gateway or its discovered provider pool was unavailable for this run". `#1953` had just given the Strix sandbox bootstrap failure its own second verdict token (`STRIX_SANDBOX_UNAVAILABLE`) precisely because that attribution is wrong for it -- the sandbox container never reaches its Caido proxy, so the run dies before the gateway serves anything -- and this consumer re-applied the wrong attribution one step downstream, into the review findings and the failure census. The emitter now branches on the second token: a sandbox verdict gets a finding that names Strix's sandbox, says the verdict does not name the gateway, and tells the reader not to change gateway or provider configuration on its strength. A `STRIX_PROVIDER_UNAVAILABLE` line without the token keeps its existing text verbatim, so the gateway class has no regression surface. No test covered this finding text at all before (`gateway or its discovered provider pool` matched nothing under `tests/`); `tests/test_opencode_dispatch_strix_sandbox_finding.py` now runs the production emitter from the published run block and pins both directions plus the no-signal case. Refs #1953, #1935. diff --git a/docs/doctoring/codeql-pr-private-consumer-read-permissions.md b/docs/doctoring/codeql-pr-private-consumer-read-permissions.md new file mode 100644 index 0000000000..fc0cf62d7e --- /dev/null +++ b/docs/doctoring/codeql-pr-private-consumer-read-permissions.md @@ -0,0 +1,26 @@ +# CodeQL required workflow denies private consumers a read they need — 2026-09-13 + +## Symptom + +Private consumer `ContextualWisdomLab/late-life-anxiety-reanalysis` PR #10 (head `a1cd5bc6783c6510dfcf937f523c733366e82213`, run `34700410434`) failed both org-required `.github/workflows/codeql-pr.yml` jobs at their first API call, each with `gh: Resource not accessible by integration (HTTP 403)`. `CodeQL compatibility analysis (python)` (job `103571590442`), step "Read current-head CodeQL dispatch verdict", calls `gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}"` under `GH_TOKEN: ${{ github.token }}`; the runner printed effective token permissions of Contents: read, Metadata: read only (declared: `contents: read`, `id-token: write`). `Dispatch current-head CodeQL scan` (job `103571810868`) makes the same GET, then later reads `repos/${TARGET_REPOSITORY}/commits/${PR_HEAD_SHA}/statuses`, under `contents: read`, `id-token: write`, `actions: read`. Public consumers (fast-mlsirm, pg-erd-cloud, naruon, html4tree) pass the identical workflow only because GET on a *public* repository needs no fine-grained grant; the defect is specific to private repositories. + +## Root cause + +Neither job declared the fine-grained read permissions GitHub's REST contract requires for these calls on a private repository: "Get a pull request" needs `pull-requests: read`; "List commit statuses for a reference" needs `statuses: read`. Missing both, the minted `GITHUB_TOKEN` had no read access to pull-request or status data on a private repo, and testing against public consumers never exercised the gap because anonymous-equivalent GETs on public repository resources are always permitted. + +## Repair + +Added `pull-requests: read` and `statuses: read` to the `analyze-head` and `dispatch-current-head` job `permissions:` blocks in `.github/workflows/codeql-pr.yml`, preserving declaration order (contents, id-token, [actions], pull-requests, statuses). No write permission is added anywhere; `actions: write` remains absent, still guarded by the existing `test_codeql_required_workflow_does_not_gain_actions_write` regression test. + +## Local evidence + +New test `test_codeql_pr_jobs_hold_read_grants_private_consumers_need` in `tests/test_codeql_pr_workflow_contract.py` slices both permission blocks the same way the neighboring `actions: write` guard does and asserts each holds exactly `pull-requests: read` and `statuses: read` with no `actions: write`. RED: 1 failed (`assert [] == ['read']`). GREEN: 1 passed. Combined focused run across the five CodeQL/required-workflow contract test files: 149 passed. Full repository suite and `actionlint` result are recorded in the pull request description. + +## Hosted acceptance still required + +This repair is unverified against GitHub's live permission enforcement. A newly loaded central SHA carrying this change must still pass both `analyze-head` and `dispatch-current-head` on the private consumer's exact current head before the defect is resolved end-to-end. Separately, the later `repository_dispatch` POST from `dispatch-current-head` to `ContextualWisdomLab/.github` using the OpenCode app token has not yet been exercised from a private consumer at all, and may surface a distinct scoping issue of its own once this read-permission blocker is cleared. + +## References + +- GitHub REST, "Get a pull request": https://docs.github.com/en/rest/pulls/pulls#get-a-pull-request (fine-grained permission: `pull-requests: read`) +- GitHub REST, "List commit statuses for a reference": https://docs.github.com/en/rest/commits/statuses#list-commit-statuses-for-a-reference (fine-grained permission: `statuses: read`) diff --git a/tests/test_codeql_pr_workflow_contract.py b/tests/test_codeql_pr_workflow_contract.py index dc67eef258..d5a54ccba5 100644 --- a/tests/test_codeql_pr_workflow_contract.py +++ b/tests/test_codeql_pr_workflow_contract.py @@ -515,6 +515,45 @@ def test_codeql_required_workflow_does_not_gain_actions_write() -> None: assert "actions: write" not in coordinator_permissions +def test_codeql_pr_jobs_hold_read_grants_private_consumers_need() -> None: + """analyze-head and dispatch-current-head need pull-requests/statuses reads. + + Consumer evidence: ContextualWisdomLab/late-life-anxiety-reanalysis PR #10 + (head a1cd5bc6783c6510dfcf937f523c733366e82213, run 34700410434). Both + required-workflow jobs failed at their first API call with + `gh: Resource not accessible by integration (HTTP 403)`: + - job "CodeQL compatibility analysis (python)" (job 103571590442), step + "Read current-head CodeQL dispatch verdict", calling + `gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}"` with only + `contents: read` + `id-token: write` (effective token printed by the + runner: Contents: read, Metadata: read). + - job "Dispatch current-head CodeQL scan" (job 103571810868), step + "Dispatch current-head CodeQL scan", the same GET plus a later read of + `repos/${TARGET_REPOSITORY}/commits/${PR_HEAD_SHA}/statuses`, with + `contents: read`, `id-token: write`, `actions: read`. + + Public consumers (fast-mlsirm, pg-erd-cloud, naruon, html4tree) passed + only because GET on a public repository does not need the grant. + GitHub's REST contract requires the `pull-requests: read` fine-grained + permission for "Get a pull request" and `statuses: read` for "List commit + statuses for a reference" on private repositories. + """ + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + shard_permissions = workflow.split(" analyze-head:\n", 1)[1].split( + " strategy:\n", 1 + )[0] + coordinator_permissions = workflow.split(" dispatch-current-head:\n", 1)[1].split( + " steps:\n", 1 + )[0] + + for block in (shard_permissions, coordinator_permissions): + assert re.findall(r"^ pull-requests: (\w+)$", block, re.MULTILINE) == [ + "read" + ] + assert re.findall(r"^ statuses: (\w+)$", block, re.MULTILINE) == ["read"] + assert "actions: write" not in block + + def test_codeql_pr_attempt_one_without_verdict_fails_pending_without_dispatch( tmp_path: Path, ) -> None: From 76f73356e2d38ca7aa60a1228e226880a4ef7765 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 13:24:27 +0900 Subject: [PATCH 72/80] test(pingora): keep the denied runtime form out of the HWPX fixture source The central bootstrap scans this repository's own pull requests with the policy under test, and only tests/test_pingora_edge_policy.py is path-exempt. PR #2144 run 34737529066 flagged five literal runtime paths in the new fixture file; build the fixture at runtime instead so the diff never carries the form. Co-Authored-By: Claude Fable 5.1 --- tests/test_pingora_hwpx_evidence.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/test_pingora_hwpx_evidence.py b/tests/test_pingora_hwpx_evidence.py index aebbc5f96f..5f7f65985f 100644 --- a/tests/test_pingora_hwpx_evidence.py +++ b/tests/test_pingora_hwpx_evidence.py @@ -6,6 +6,12 @@ import pytest from tests.test_pingora_edge_policy import policy +# This repository's own pull requests are scanned by the policy under test, and +# only tests/test_pingora_edge_policy.py is path-exempt. Keep the denied runtime +# form split in source so the fixture exists at runtime, never in the diff. +RUNTIME_TEXT = "cat /etc/" + "nginx/nginx.conf\n" +RUNTIME_BYTES = RUNTIME_TEXT.encode("ascii") + def hwpx_archive(*, mime_value=b"application/hwp+zip", manifest_value=b"", compression_type=zipfile.ZIP_STORED): """Create a deterministic, non-sensitive format-boundary fixture.""" @@ -44,8 +50,8 @@ def test_valid_hwpx_is_admitted_at_document_and_consumer_paths(file_path): @pytest.mark.parametrize("file_bytes", [ b"PK\x03\x04\xff", hwpx_archive()[:-10], - b"#!/bin/sh\ncat /etc/nginx/nginx.conf\n" + hwpx_archive(), - hwpx_archive() + b"\ncat /etc/nginx/nginx.conf\n", + b"#!/bin/sh\n" + RUNTIME_BYTES + hwpx_archive(), + hwpx_archive() + b"\n" + RUNTIME_BYTES, hwpx_archive(mime_value=b"application/zip"), hwpx_archive(manifest_value=None), hwpx_archive(manifest_value=b""), hwpx_archive(compression_type=zipfile.ZIP_DEFLATED), @@ -57,16 +63,16 @@ def test_malformed_or_disguised_archive_is_not_exempt(file_bytes): @pytest.mark.parametrize("file_path", ["evidence/paper.hwpx", "docs/paper.hwpx", "scripts/paper.hwpx"]) -@pytest.mark.parametrize("patch_value", [None, "+cat /etc/nginx/nginx.conf"]) +@pytest.mark.parametrize("patch_value", [None, "+" + RUNTIME_TEXT.rstrip("\n")]) def test_text_renamed_to_hwpx_preserves_runtime_scan(file_path, patch_value): """Neither the suffix nor patch absence hides actual Nginx runtime text.""" - violations = evaluate_bytes(file_path, b"cat /etc/nginx/nginx.conf\n", patch_value=patch_value) + violations = evaluate_bytes(file_path, RUNTIME_BYTES, patch_value=patch_value) assert [violation.rule for violation in violations] == ["nginx_runtime_path"] def test_hwpx_does_not_expand_text_document_exemptions(): """The consumer evidence directory does not exempt ordinary configuration.""" - violations = evaluate_bytes("evidence/config.txt", b"cat /etc/nginx/nginx.conf\n") + violations = evaluate_bytes("evidence/config.txt", RUNTIME_BYTES) assert [violation.rule for violation in violations] == ["nginx_runtime_path"] From b72ab7fa6f9b0739a7218e8fbd34045ca1dc5c6a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 13:25:13 +0900 Subject: [PATCH 73/80] fix(review-policy): key OpenRouter ZDR feed routes by model_id Refs ContextualWisdomLab/late-life-anxiety-reanalysis#10, ContextualWisdomLab/.github#2122. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 4 + .../doctoring/zdr-feed-model-id-route-keys.md | 25 +++++ .../contextual_orchestrator_review_policy.py | 13 ++- ...t_contextual_orchestrator_review_policy.py | 106 +++++++++++++++++- 4 files changed, 142 insertions(+), 6 deletions(-) create mode 100644 docs/doctoring/zdr-feed-model-id-route-keys.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 707c18532e..91ce6d788f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Review policy ZDR feed keys routes by the wrong field, catalog always empty under `--require-zdr` + +- `_load_zdr_endpoints` (`scripts/ci/contextual_orchestrator_review_policy.py`, introduced by 17052a7ca / #1360) built ZDR route keys from `endpoint.get("model_name")`, but on the real `https://openrouter.ai/api/v1/endpoints/zdr` feed `model_name` is a human display string (e.g. "DeepSeek: DeepSeek V4.1 Flash") while `model_id` is the slug contextual-orchestrator discovery reports as `model` (e.g. `inclusionai/ling-3.0-flash-vl:free`). No live-feed key ever matched `is_zdr_model(...)`, so every `--require-zdr` consumer (every private/internal caller, per ADR-0003) saw an empty catalog and failed closed with `PolicyError: no attested ZDR model route is available with the ZDR policy; orchestrator/free would fail closed`. Confirmed as the cause of `noema-review` and `strix` failing on `ContextualWisdomLab/late-life-anxiety-reanalysis#10` (head `a1cd5bc6783c6510dfcf937f523c733366e82213`, runs `34700409452`/`103571267389` and `34700409446`/`103571829483`) against central `fb17ef556f94f673234aa557254ae52779e9a7b0`. `_load_zdr_endpoints` now keys on `model_id`, with no fallback to the display name; the three existing fixtures that put slugs into `model_name` (masking the bug since #1360) now carry the real feed schema. Offline reproduction against a 60-row consumer discovery snapshot and the live 859-entry ZDR feed: before, `--require-zdr --pool free` exits 1 with the `PolicyError` above; after, it exits 0 and selects 3 attested `openrouter` ZDR routes (`inclusionai/ling-3.0-flash-{vl,sante,fin}:free`, served by `Novita`). Hosted acceptance on the private consumer's exact head is still required and is not claimed here. Refs ContextualWisdomLab/late-life-anxiety-reanalysis#10, ContextualWisdomLab/.github#2122. + ### Failed-check finding names the Strix sandbox instead of the gateway - `opencode-review-dispatch.yml`'s `emit_strix_provider_failure_finding` rendered one fixed finding for every `STRIX_PROVIDER_UNAVAILABLE` line, whose Root cause read "The contextual-orchestrator gateway or its discovered provider pool was unavailable for this run". `#1953` had just given the Strix sandbox bootstrap failure its own second verdict token (`STRIX_SANDBOX_UNAVAILABLE`) precisely because that attribution is wrong for it -- the sandbox container never reaches its Caido proxy, so the run dies before the gateway serves anything -- and this consumer re-applied the wrong attribution one step downstream, into the review findings and the failure census. The emitter now branches on the second token: a sandbox verdict gets a finding that names Strix's sandbox, says the verdict does not name the gateway, and tells the reader not to change gateway or provider configuration on its strength. A `STRIX_PROVIDER_UNAVAILABLE` line without the token keeps its existing text verbatim, so the gateway class has no regression surface. No test covered this finding text at all before (`gateway or its discovered provider pool` matched nothing under `tests/`); `tests/test_opencode_dispatch_strix_sandbox_finding.py` now runs the production emitter from the published run block and pins both directions plus the no-signal case. Refs #1953, #1935. diff --git a/docs/doctoring/zdr-feed-model-id-route-keys.md b/docs/doctoring/zdr-feed-model-id-route-keys.md new file mode 100644 index 0000000000..f219a97c56 --- /dev/null +++ b/docs/doctoring/zdr-feed-model-id-route-keys.md @@ -0,0 +1,25 @@ +# OpenRouter ZDR feed route keys used the wrong field — 2026-09-13 + +## Symptom + +`noema-review` and `strix` both failed closed on `ContextualWisdomLab/late-life-anxiety-reanalysis#10` (head `a1cd5bc6783c6510dfcf937f523c733366e82213`, runs `34700409452`/job `103571267389` and `34700409446`/job `103571829483`) against central `fb17ef556f94f673234aa557254ae52779e9a7b0`, both exiting with `PolicyError: no attested ZDR model route is available with the ZDR policy; orchestrator/free would fail closed`. Every private/internal consumer runs `--require-zdr` (ADR-0003), so this is a hard boot failure, not a degraded catalog. + +## Root cause (feed schema) + +`_load_zdr_endpoints` (`scripts/ci/contextual_orchestrator_review_policy.py`) built route keys from `endpoint.get("model_name")`. On the real `https://openrouter.ai/api/v1/endpoints/zdr` feed (see OpenRouter's ZDR docs, https://openrouter.ai/docs/guides/features/zdr), `model_name` is a human display string (e.g. "DeepSeek: DeepSeek V4.1 Flash") while `model_id` is the slug contextual-orchestrator discovery reports as `model` (e.g. `inclusionai/ling-3.0-flash-vl:free`). No live-feed key ever matched `is_zdr_model(...)`, so the catalog was always empty under `--require-zdr`. The three fixtures in `tests/test_contextual_orchestrator_review_policy.py` put slugs into `model_name`, which is why this was invisible to tests since the keying was introduced in 17052a7ca (#1360, 2026-08-27). + +## Repair + +`_load_zdr_endpoints` now keys on `endpoint.get("model_id")`; there is no fallback to the display name, and rows missing `model_id` or `provider_name` are still skipped. The three fixtures were corrected to carry the real feed schema (`model_id` slug + a display-string `model_name`). `is_zdr_model` and `zdr_policy.py` are unchanged. + +## Offline reproduction (before / after) + +Discovery: a 60-row consumer snapshot (20 each openrouter/nvidia_nim/nvidia_nim_sub free rows). Feed: the live 859-entry `/api/v1/endpoints/zdr` response fetched 2026-09-13, carrying three matching `inclusionai/ling-3.0-flash-*:free` openrouter routes served by `Novita`. + +- Before: `--require-zdr --pool free` exits 1 with the `PolicyError` above. +- After: exits 0, `zdr_selected_count: 3`, selecting exactly `openrouter/inclusionai/ling-3.0-flash-{vl,sante,fin}:free`, all `zdr: true`. +- Without `--require-zdr`: `zdr_selected_count: 3` and those three routes rank first in the 12-route free catalog. + +## Hosted acceptance still required + +This is an offline fix against a static discovery/feed snapshot. It does not prove a newly loaded central SHA boots the sidecar on the private consumer's exact head, and it does not change how the gateway itself requests ZDR routing from OpenRouter — that remains a separate contextual-orchestrator (CO)-side check. diff --git a/scripts/ci/contextual_orchestrator_review_policy.py b/scripts/ci/contextual_orchestrator_review_policy.py index e609e67ff3..241f84bef7 100644 --- a/scripts/ci/contextual_orchestrator_review_policy.py +++ b/scripts/ci/contextual_orchestrator_review_policy.py @@ -466,14 +466,23 @@ def build_zdr_prioritized_catalog( def _load_zdr_endpoints(path: str | None) -> frozenset[str]: - """Load exact provider/model keys from an OpenRouter ZDR feed file.""" + """Load exact provider/model keys from an OpenRouter ZDR feed file. + + Each feed row's ``model_id`` is the discovery slug contextual-orchestrator + reports as ``model`` (e.g. ``"inclusionai/ling-3.0-flash-vl:free"``); + ``model_name`` is a human display string (e.g. "DeepSeek: DeepSeek V4.1 + Flash") and is never used to build a route key. ``provider_name`` is the + feed's serving-provider label (e.g. "Novita"). Rows missing either + ``model_id`` or ``provider_name`` are skipped; there is no fallback to + the display name. + """ if not path: return frozenset() payload = json.loads(Path(path).read_text(encoding="utf-8")) keys: set[str] = set() for endpoint in payload.get("data", []): provider = endpoint.get("provider_name") - model = endpoint.get("model_name") + model = endpoint.get("model_id") if provider and model: keys.add(_route_key(str(provider), str(model))) keys.add(_route_key("openrouter", str(model))) diff --git a/tests/test_contextual_orchestrator_review_policy.py b/tests/test_contextual_orchestrator_review_policy.py index 41c1ea40b2..e13e94107e 100644 --- a/tests/test_contextual_orchestrator_review_policy.py +++ b/tests/test_contextual_orchestrator_review_policy.py @@ -116,8 +116,12 @@ def test_load_zdr_endpoints_skips_rows_without_provider_or_model(tmp_path) -> No json.dumps( { "data": [ - {"model_name": "deepseek/deepseek-r1:free", "provider_name": "DeepSeek"}, - {"model_name": "no-provider"}, + { + "model_id": "deepseek/deepseek-r1:free", + "model_name": "DeepSeek: R1 (free)", + "provider_name": "DeepSeek", + }, + {"model_id": "no-provider"}, {"provider_name": "NoModel"}, ] } @@ -140,6 +144,91 @@ def test_load_zdr_endpoints_respects_none_feed_path(tmp_path) -> None: assert policy._load_zdr_endpoints(str(empty_feed)) == frozenset() +def test_load_zdr_endpoints_keys_by_model_id_not_display_name(tmp_path) -> None: + """The live OpenRouter ZDR feed keys routes by ``model_id``, not ``model_name``. + + Confirmed by offline reproduction against the real + ``https://openrouter.ai/api/v1/endpoints/zdr`` feed: OpenRouter's + ``model_name`` is a human display string (e.g. "DeepSeek: DeepSeek V4.1 + Flash") while ``model_id`` is the slug contextual-orchestrator discovery + reports as ``model`` (e.g. "inclusionai/ling-3.0-flash-vl:free"). Keying + on ``model_name`` (introduced in 17052a7ca, #1360) meant no live-feed + route ever matched ``is_zdr_model(...)``, so with ``--require-zdr`` + (every private/internal consumer, per ADR-0003) the catalog was always + empty and the sidecar failed closed with "no attested ZDR model route is + available with the ZDR policy; orchestrator/free would fail closed". + This killed noema-review and strix on + ContextualWisdomLab/late-life-anxiety-reanalysis#10 (head + a1cd5bc6783c6510dfcf937f523c733366e82213, runs 34700409452/103571267389 + and 34700409446/103571829483) against central + fb17ef556f94f673234aa557254ae52779e9a7b0. See + ContextualWisdomLab/.github#2122. + """ + feed = tmp_path / "zdr.json" + feed.write_text( + json.dumps( + { + "data": [ + { + "name": "Novita | inclusionai/ling-3.0-flash-vl-20260910:free", + "model_id": "inclusionai/ling-3.0-flash-vl:free", + "model_name": "inclusionAI: Ling 3.0 Flash VL (free)", + "provider_name": "Novita", + }, + { + "name": "x", + "model_name": "Display Only", + "provider_name": "Novita", + }, + ] + } + ), + encoding="utf-8", + ) + + keys = policy._load_zdr_endpoints(str(feed)) + + assert keys == frozenset( + { + policy._route_key("Novita", "inclusionai/ling-3.0-flash-vl:free"), + policy._route_key("openrouter", "inclusionai/ling-3.0-flash-vl:free"), + } + ) + assert not any("Display Only" in key for key in keys) + assert not any("inclusionAI: Ling 3.0 Flash VL" in key for key in keys) + + report = { + "models": [ + { + "provider": "openrouter", + "model": "inclusionai/ling-3.0-flash-vl:free", + "agent_id": "or_ling_vl", + "is_free": True, + **FREE_PRICE, + }, + { + "provider": "openrouter", + "model": "other-vendor/not-covered:free", + "agent_id": "or_not_covered", + "is_free": True, + **FREE_PRICE, + }, + ] + } + result = policy.build_zdr_prioritized_catalog( + policy.parse_discovery_report(report), + limit=12, + account_cap=4, + zdr_endpoints=keys, + require_zdr=True, + pool="free", + ) + assert [agent["model"] for agent in result["agents"]] == [ + "inclusionai/ling-3.0-flash-vl:free" + ] + assert result["report"]["zdr_selected_count"] == 1 + + def test_route_key_prefixes_provider() -> None: """ZDR feed keys are matched with the provider prefix.""" assert policy._route_key("openrouter", "deepseek/deepseek-r1:free") == ( @@ -430,7 +519,8 @@ def test_load_zdr_endpoints_parses_feed(tmp_path) -> None: "data": [ { "name": "deepseek/deepseek-r1:free", - "model_name": "deepseek/deepseek-r1:free", + "model_id": "deepseek/deepseek-r1:free", + "model_name": "DeepSeek: R1 (free)", "provider_name": "DeepSeek", } ] @@ -453,7 +543,15 @@ def test_build_catalog_from_paths_writes_both_files(tmp_path) -> None: feed = tmp_path / "zdr.json" feed.write_text( json.dumps( - {"data": [{"model_name": "deepseek/deepseek-r1:free", "provider_name": "DeepSeek"}]} + { + "data": [ + { + "model_id": "deepseek/deepseek-r1:free", + "model_name": "DeepSeek: R1 (free)", + "provider_name": "DeepSeek", + } + ] + } ), encoding="utf-8", ) From fb8138e6189e8a931fa937675f2c990421ba1dac Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 13:29:44 +0900 Subject: [PATCH 74/80] docs(sast-semgrep): describe the step-level guard in the job comment The comment still said the classifier verdict was consumed through `needs`; nothing consumes it that way any more. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/sast-semgrep.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sast-semgrep.yml b/.github/workflows/sast-semgrep.yml index 283eac0097..f8ab04b865 100644 --- a/.github/workflows/sast-semgrep.yml +++ b/.github/workflows/sast-semgrep.yml @@ -43,8 +43,9 @@ jobs: # The org ruleset IGNORES every `on:` filter (paths, branches, types) when it # runs this workflow in another repository, and a trigger-level skip would # leave `.github`'s classic required contexts Pending forever. Both - # mechanisms honour a JOB-level skip, so the doc/image-only decision is made - # here and consumed through `needs`. See + # mechanisms honour a job that runs and concludes on its own, so the + # doc/image-only decision is made by the classifier step below and consumed + # by the expensive steps' `if:` guards. See # docs/doctoring/required-workflow-path-filter-boundary.md. # Fails OPEN: an unreadable, empty, or truncated file list scans everything. # The gate lives inside this job as a step-level guard (one runner, not two). From 7e5b971a48dfdbd27d98ca8c39662c1d948935b4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 17:16:09 +0900 Subject: [PATCH 75/80] test(security-scan): assert each OSV step's output flag individually CodeRabbit on #2143: whole-workflow counts could pass if the same string appeared in another step or log line while a scanner or reporter step lost its flag. Check the four scan/retry steps and the reporter step by name. Co-Authored-By: Claude Fable 5.1 --- tests/test_required_workflow_queue_contract.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 157556502a..87277d45f5 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -1754,12 +1754,18 @@ def test_osv_scan_uses_current_output_flags_and_binds_sarif_checkout_path() -> N """ workflow = workflow_text("security-scan.yml") - assert workflow.count("--output-file=old-results.json") == 2 - assert workflow.count("--output-file=new-results.json") == 2 - assert "--output-files=results.sarif" in workflow - assert "--output=old-results.json" not in workflow - assert "--output=new-results.json" not in workflow - assert "--output=results.sarif" not in workflow + # Check each named scanner/reporter step on its own, so a flag removed from + # one step cannot hide behind the same string appearing elsewhere. + for step_name, output_flag in ( + ("Scan base with OSV", "--output-file=old-results.json"), + ("Retry base OSV without transitive resolution", "--output-file=old-results.json"), + ("Scan head with OSV", "--output-file=new-results.json"), + ("Retry head OSV without transitive resolution", "--output-file=new-results.json"), + ("Report PR-introduced OSV findings", "--output-files=results.sarif"), + ): + step = workflow_step(workflow, step_name) + assert output_flag in step, step_name + assert "\n --output=" not in step, step_name head_checkout = workflow_step(workflow, "Checkout head") checkout_dir = re.search(r"(?m)^\s+path: (\S+)$", head_checkout).group(1) From 3452ed560573a2f4698be76f88db4081b9371465 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 19:46:31 +0900 Subject: [PATCH 76/80] fix(codeql): resolve SARIF rules from the referenced tool component `gather_findings` consulted only `tool.driver.rules`, so results whose rule lives in `tool.extensions` (every current CodeQL query pack: the real Python artifact has 0 driver rules and 43 extension rules) lost their security-severity and tags and passed the Medium+ gate fail-open (#2150). - `_component_rules`: honour `result.rule.toolComponent` (index | guid | name); absent means driver; unmatched means unresolved, never the wrong component. - `_rule_for_result`: resolve inside the referenced component only, so colliding rule ids stay distinct; validate `rule.index` against the declared id; `ruleId` != `rule.id`, bad indices, non-dict entries are unresolved. - `_finding_from_result`: an unresolved reference with no result-level score gates as `level=unresolved-rule` instead of silently passing. Driver-backed behaviour unchanged. Tests RED 8 -> GREEN 24; file coverage 100%. Full suite under coverage: 3062 passed / 1 skipped / 100% coverage; 13 timing-sensitive failures reproduced as passing in isolation (CPU contention). Closes #2150 Co-Authored-By: Claude Fable 5.1 --- scripts/ci/codeql_sarif_gate.py | 90 +++++++++++++++++----- tests/test_codeql_sarif_gate.py | 132 ++++++++++++++++++++++++++++++++ 2 files changed, 204 insertions(+), 18 deletions(-) diff --git a/scripts/ci/codeql_sarif_gate.py b/scripts/ci/codeql_sarif_gate.py index 3b232c3bdb..fb751ba90d 100644 --- a/scripts/ci/codeql_sarif_gate.py +++ b/scripts/ci/codeql_sarif_gate.py @@ -33,19 +33,66 @@ def iter_sarif_files(root: Path) -> list[Path]: return sorted(root.rglob("*.sarif")) -def _rule_for_result(result: dict[str, Any], rules: list[Any]) -> dict[str, Any]: - """Resolve the SARIF rule definition referenced by a result.""" - rules_by_id = { - str(rule.get("id") or ""): rule for rule in rules if isinstance(rule, dict) - } - rule = rules_by_id.get(str(result.get("ruleId") or ""), {}) - if rule: - return rule - rule_index = result.get("ruleIndex") - if isinstance(rule_index, int) and 0 <= rule_index < len(rules): - candidate = rules[rule_index] - if isinstance(candidate, dict): +UNRESOLVED_RULE_LEVEL = "unresolved-rule" + + +def _component_rules(result: dict[str, Any], tool: dict[str, Any]) -> list[Any] | None: + """Return the rules of the tool component a result references (SARIF 2.1.0 §3.54). + + No ``rule.toolComponent`` means the driver. Otherwise the reference selects one of + ``tool.extensions`` by ``index``, ``guid``, or ``name``; an unmatched reference + returns ``None`` so the caller can fail closed instead of consulting the wrong + component (issue #2150). + """ + reference = result.get("rule") if isinstance(result.get("rule"), dict) else {} + component_ref = reference.get("toolComponent") + if not isinstance(component_ref, dict): + return (tool.get("driver") or {}).get("rules") or [] + extensions = [ext for ext in tool.get("extensions") or [] if isinstance(ext, dict)] + index = component_ref.get("index") + if isinstance(index, int): + if 0 <= index < len(extensions): + return extensions[index].get("rules") or [] + return None + for key in ("guid", "name"): + wanted = component_ref.get(key) + if wanted is not None: + for extension in extensions: + if extension.get(key) == wanted: + return extension.get("rules") or [] + return None + return None + + +def _rule_for_result(result: dict[str, Any], tool: dict[str, Any]) -> dict[str, Any] | None: + """Resolve the SARIF rule definition a result references, or ``None`` if it cannot be. + + Resolution order inside the referenced component: ``rule.index`` (validated + against the declared id), then id lookup (``ruleId`` / ``rule.id``), then the + legacy ``ruleIndex``. Colliding ids across components stay distinct because + lookup never leaves the referenced component. + """ + rules = _component_rules(result, tool) + if rules is None: + return None + reference = result.get("rule") if isinstance(result.get("rule"), dict) else {} + declared_ids = {str(v) for v in (result.get("ruleId"), reference.get("id")) if v} + if len(declared_ids) > 1: + return None + declared_id = next(iter(declared_ids), "") + for index in (reference.get("index"), result.get("ruleIndex")): + if isinstance(index, int): + candidate = rules[index] if 0 <= index < len(rules) else None + if not isinstance(candidate, dict): + return None + if declared_id and str(candidate.get("id") or "") != declared_id: + return None return candidate + if declared_id: + for rule in rules: + if isinstance(rule, dict) and str(rule.get("id") or "") == declared_id: + return rule + return None return {} @@ -56,11 +103,16 @@ def _is_medium_plus(score: float | None, level: str, security_rule: bool) -> boo return security_rule and level in SEVERITY_LEVELS -def _finding_from_result(result: dict[str, Any], rules: list[Any]) -> Finding | None: - """Build a `Finding` for one SARIF result, or None if it doesn't gate the PR.""" +def _finding_from_result(result: dict[str, Any], tool: dict[str, Any]) -> Finding | None: + """Build a `Finding` for one SARIF result, or None if it doesn't gate the PR. + + A result whose rule reference cannot be resolved and that carries no explicit + security-severity gates as ``unresolved-rule`` rather than passing silently. + """ if not isinstance(result, dict) or result.get("suppressions"): return None - rule = _rule_for_result(result, rules) + resolved = _rule_for_result(result, tool) + rule = resolved or {} result_properties = result.get("properties") or {} rule_properties = rule.get("properties") or {} raw_score = result_properties.get("security-severity", rule_properties.get("security-severity")) @@ -71,7 +123,9 @@ def _finding_from_result(result: dict[str, Any], rules: list[Any]) -> Finding | level = str(result.get("level") or (rule.get("defaultConfiguration") or {}).get("level") or "none").lower() tags = {str(tag).lower() for tag in rule_properties.get("tags") or []} security_rule = "security" in tags or any(tag.startswith("external/cwe/") for tag in tags) - if not _is_medium_plus(score, level, security_rule): + if resolved is None and score is None: + level = UNRESOLVED_RULE_LEVEL + elif not _is_medium_plus(score, level, security_rule): return None physical = ((result.get("locations") or [{}])[0].get("physicalLocation") or {}) artifact = (physical.get("artifactLocation") or {}).get("uri") or "unknown" @@ -95,12 +149,12 @@ def gather_findings(root: Path) -> tuple[list[Finding], int, int]: for path in paths: payload = json.loads(path.read_text(encoding="utf-8")) for run in payload.get("runs") or []: - rules = ((run.get("tool") or {}).get("driver") or {}).get("rules") or [] + tool = run.get("tool") if isinstance(run.get("tool"), dict) else {} for result in run.get("results") or []: if not isinstance(result, dict): continue total_results += 1 - finding = _finding_from_result(result, rules) + finding = _finding_from_result(result, tool) if finding is not None: findings.append(finding) return findings, total_results, len(paths) diff --git a/tests/test_codeql_sarif_gate.py b/tests/test_codeql_sarif_gate.py index 186b9c80f1..1ab542dd89 100644 --- a/tests/test_codeql_sarif_gate.py +++ b/tests/test_codeql_sarif_gate.py @@ -203,3 +203,135 @@ def test_script_entrypoint_exits_with_main_status(tmp_path, monkeypatch): runpy.run_path(str(Path("scripts/ci/codeql_sarif_gate.py")), run_name="__main__") assert exc_info.value.code == 0 + + +def _extension_run(results: list[dict], *, driver_rules: list | None = None) -> dict: + """A run shaped like a real CodeQL artifact: 0 driver rules, rules in a query-pack extension.""" + extension_rules = [{"id": f"py/filler-{n}"} for n in range(17)] + [ + { + "id": "py/incomplete-url-substring-sanitization", + "properties": {"security-severity": "7.8", "tags": ["security", "external/cwe/cwe-020"]}, + "defaultConfiguration": {"level": "warning"}, + } + ] + return { + "tool": { + "driver": {"name": "CodeQL", "rules": driver_rules or []}, + "extensions": [{"name": "codeql/python-queries", "rules": extension_rules}], + }, + "results": results, + } + + +def test_gather_findings_resolves_rules_from_the_referenced_extension(tmp_path): + """Issue #2150: a result whose rule lives in tool.extensions must gate, not fail open.""" + _write_sarif( + tmp_path / "ext.sarif", + [ + _extension_run( + [ + { + "ruleId": "py/incomplete-url-substring-sanitization", + "rule": {"id": "py/incomplete-url-substring-sanitization", "index": 17, "toolComponent": {"index": 0}}, + "message": {"text": "doi check"}, + "locations": [{"physicalLocation": {"artifactLocation": {"uri": "src/x.py"}, "region": {"startLine": 4}}}], + }, + { + "ruleId": "py/incomplete-url-substring-sanitization", + "rule": {"index": 17, "toolComponent": {"name": "codeql/python-queries"}}, + "message": {"text": "by component name"}, + }, + ] + ) + ], + ) + + findings, total_results, _ = gate.gather_findings(tmp_path) + + assert total_results == 2 + assert [(f.rule_id, f.score, f.level, f.path, f.line) for f in findings] == [ + ("py/incomplete-url-substring-sanitization", 7.8, "warning", "src/x.py", 4), + ("py/incomplete-url-substring-sanitization", 7.8, "warning", "unknown", 0), + ] + + +def test_gather_findings_keeps_colliding_rule_ids_per_component(tmp_path): + """The same rule id in the driver and an extension resolves to the referenced component's metadata.""" + _write_sarif( + tmp_path / "collide.sarif", + [ + _extension_run( + [ + {"ruleId": "shared/id", "message": {"text": "driver copy"}}, + {"ruleId": "shared/id", "rule": {"toolComponent": {"index": 0}}, "message": {"text": "extension copy"}}, + ], + driver_rules=[{"id": "shared/id", "defaultConfiguration": {"level": "note"}}], + ) + ], + ) + # extension gets a colliding scored rule appended + payload = json.loads((tmp_path / "collide.sarif").read_text(encoding="utf-8")) + payload["runs"][0]["tool"]["extensions"][0]["rules"].append( + {"id": "shared/id", "properties": {"security-severity": "9.1"}} + ) + (tmp_path / "collide.sarif").write_text(json.dumps(payload), encoding="utf-8") + + findings, _, _ = gate.gather_findings(tmp_path) + + assert [(f.message, f.score) for f in findings] == [("extension copy", 9.1)] + + +@pytest.mark.parametrize( + "result", + [ + {"ruleId": "py/x", "rule": {"index": 17, "toolComponent": {"index": 5}}}, + {"ruleId": "py/x", "rule": {"index": 17, "toolComponent": {"name": "codeql/no-such-pack"}}}, + {"ruleId": "py/x", "rule": {"index": 99, "toolComponent": {"index": 0}}}, + {"ruleId": "py/other", "rule": {"index": 17, "toolComponent": {"index": 0}}}, + {"ruleId": "py/x", "rule": {"id": "py/y", "toolComponent": {"index": 0}}}, + {"rule": {"index": 3, "toolComponent": {"guid": "00000000-0000-0000-0000-000000000000"}}}, + {"ruleId": "py/x", "rule": {"toolComponent": {}}}, + ], + ids=["bad-component-index", "bad-component-name", "bad-rule-index", "indexed-rule-id-mismatch", "ruleId-vs-rule-id-mismatch", "bad-component-guid", "empty-component-reference"], +) +def test_gather_findings_fails_closed_on_unresolvable_rule_references(tmp_path, result): + """A rule reference that cannot be resolved, with no severity evidence, gates instead of passing.""" + _write_sarif(tmp_path / "bad.sarif", [_extension_run([dict(result, message={"text": "m"})])]) + + findings, _, _ = gate.gather_findings(tmp_path) + + assert len(findings) == 1 + assert findings[0].level == "unresolved-rule" + assert findings[0].score is None + assert gate.format_finding(findings[0]).startswith("CODEQL_FINDING rule=") + + +def test_gather_findings_uses_result_score_even_when_rule_is_unresolvable(tmp_path): + """Explicit result-level security-severity still decides gating when the rule cannot be resolved.""" + _write_sarif( + tmp_path / "scored.sarif", + [_extension_run([{"ruleId": "py/x", "rule": {"toolComponent": {"index": 9}}, "properties": {"security-severity": "1.0"}}])], + ) + + findings, _, _ = gate.gather_findings(tmp_path) + + assert findings == [] + + +def test_gather_findings_gates_an_unreferenced_result_on_its_own_score(tmp_path): + """A result with no rule reference at all is judged purely on its result-level severity.""" + _write_sarif(tmp_path / "bare.sarif", [_extension_run([{"properties": {"security-severity": "6.0"}}])]) + + findings, _, _ = gate.gather_findings(tmp_path) + + assert [(f.rule_id, f.score, f.level) for f in findings] == [("unknown", 6.0, "none")] + + +def test_gather_findings_leaves_resolved_non_security_extension_rules_alone(tmp_path): + """A resolved extension rule with no security metadata keeps the existing non-gating semantics.""" + _write_sarif( + tmp_path / "style.sarif", + [_extension_run([{"rule": {"index": 3, "toolComponent": {"index": 0}}, "level": "note", "message": {"text": "style"}}])], + ) + + assert gate.gather_findings(tmp_path)[0] == [] From d6cf5726cebfd031d0d11989b6fa34aab43452d0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 21:45:12 +0900 Subject: [PATCH 77/80] fix(scheduler): recheck cancellation races (#2153) * fix(scheduler): recheck cancellation races * test(scheduler): bound cancellation race outcomes * fix(scheduler): preserve runs started during cancellation * fix: preserve runs after cancellation race * test: document coalescer callbacks --- scripts/ci/current_head_run_coalescer.py | 20 ++- tests/test_current_head_run_coalescer.py | 190 +++++++++++++++++++++++ 2 files changed, 209 insertions(+), 1 deletion(-) diff --git a/scripts/ci/current_head_run_coalescer.py b/scripts/ci/current_head_run_coalescer.py index ae40b85ac4..948c80cd01 100644 --- a/scripts/ci/current_head_run_coalescer.py +++ b/scripts/ci/current_head_run_coalescer.py @@ -29,6 +29,7 @@ API_TIMEOUT_SECONDS = 30 CANCELLATION_POLL_ATTEMPTS = 6 CANCELLATION_POLL_INTERVAL_SECONDS = 1.0 +QUEUE_START_RACE_RE = re.compile(r"\bHTTP\s*409\b") class CoalescingRefused(RuntimeError): @@ -373,7 +374,24 @@ def _fetch_run(repo: str, run_id: int) -> dict[str, Any]: def _cancel_run(repo: str, run_id: int) -> None: """Cancel one run and prove GitHub reached its terminal cancelled state.""" - _run_json(["gh", "api", "-X", "POST", f"repos/{repo}/actions/runs/{run_id}/cancel"]) + cancel_args = ["gh", "api", "-X", "POST", f"repos/{repo}/actions/runs/{run_id}/cancel"] + try: + _run_json(cancel_args) + except RuntimeError as exc: + # GitHub can race a queued run into startup between the candidate + # fetch and POST, returning HTTP 409 instead of accepting cancel. + # Re-read the authoritative run state; never turn an unknown + # cancellation error into a successful result or another mutation. + if not QUEUE_START_RACE_RE.search(str(exc)): + raise + current = _fetch_run(repo, run_id) + if current.get("status") == "completed" and current.get("conclusion") == "cancelled": + return + if current.get("status") != "queued": + raise CoalescingRefused(f"workflow run {run_id} is no longer queued after HTTP 409") from exc + raise CoalescingRefused( + f"workflow run {run_id} remained queued after HTTP 409; preserving it" + ) from exc for attempt in range(CANCELLATION_POLL_ATTEMPTS): run_data = _fetch_run(repo, run_id) if run_data.get("status") == "completed" and run_data.get("conclusion") == "cancelled": diff --git a/tests/test_current_head_run_coalescer.py b/tests/test_current_head_run_coalescer.py index 571368677f..136b373539 100644 --- a/tests/test_current_head_run_coalescer.py +++ b/tests/test_current_head_run_coalescer.py @@ -393,6 +393,7 @@ def test_run_json_uses_token_timeout_decodes_success_and_bounds_failure(monkeypa seen: dict[str, object] = {} def success(*args, **kwargs): + """Return bounded JSON while recording the subprocess timeout.""" seen.update(kwargs) return SimpleNamespace(returncode=0, stdout='{"ok":true}', stderr="") @@ -401,6 +402,7 @@ def success(*args, **kwargs): assert seen["timeout"] == module.API_TIMEOUT_SECONDS def timeout(*_args, **_kwargs): + """Raise the subprocess timeout sentinel for transport mapping.""" raise subprocess.TimeoutExpired(cmd="gh", timeout=30) monkeypatch.setattr(module.subprocess, "run", timeout) @@ -434,6 +436,7 @@ def test_fetch_helpers_fail_closed_and_paginate(monkeypatch) -> None: calls: list[list[str]] = [] def pages(args): + """Return two paginated workflow-run pages and then an empty page.""" calls.append(list(args)) status = next(item.split("=", 1)[1] for item in args if item.startswith("status=")) page = int(next(item.split("=", 1)[1] for item in args if item.startswith("page="))) @@ -473,6 +476,191 @@ def test_cancel_run_uses_explicit_transport_and_ordinary_endpoint(monkeypatch) - assert sleeps == [module.CANCELLATION_POLL_INTERVAL_SECONDS] +def test_cancel_run_preserves_started_run_after_cancel_409(monkeypatch) -> None: + """A run that started after the first POST is preserved without a second POST.""" + module = load_module() + cancel_calls = 0 + states = iter( + [ + {"status": "in_progress", "conclusion": None}, + ] + ) + + def run_json(args): + """Raise the queued-start race from the cancellation POST.""" + nonlocal cancel_calls + if args[-1].endswith("/cancel"): + cancel_calls += 1 + raise RuntimeError("gh: Cannot cancel a workflow run that has not been queued yet. (HTTP409)") + raise AssertionError(args) + + monkeypatch.setattr(module, "_run_json", run_json) + monkeypatch.setattr(module, "_fetch_run", lambda _repo, _run_id: next(states)) + with pytest.raises(module.CoalescingRefused, match="no longer queued"): + module._cancel_run("o/r", 123) + assert cancel_calls == 1 + + +def test_cancel_run_preserves_queued_run_after_cancel_409(monkeypatch) -> None: + """A queued run gets no compensating cancellation request after HTTP 409.""" + module = load_module() + cancel_calls = 0 + states = iter( + [ + {"status": "queued", "conclusion": None}, + {"status": "completed", "conclusion": "cancelled"}, + ] + ) + + def run_json(args): + """Raise the queued-start race while preserving the queued state.""" + nonlocal cancel_calls + if args[-1].endswith("/cancel"): + cancel_calls += 1 + raise RuntimeError("Cannot cancel a workflow run that has not been queued yet. (HTTP409)") + raise AssertionError(args) + + monkeypatch.setattr(module, "_run_json", run_json) + monkeypatch.setattr(module, "_fetch_run", lambda _repo, _run_id: next(states)) + with pytest.raises(module.CoalescingRefused, match="remained queued"): + module._cancel_run("o/r", 123) + assert cancel_calls == 1 + + +def test_coalesce_preserves_started_candidate_after_cancel_409(monkeypatch, capsys) -> None: + """The production coalesce path preserves a candidate that starts at POST time.""" + module = load_module() + candidate = run_record(100, 10) + sibling = run_record(101, 10) + candidate_fetches = 0 + cancel_calls = 0 + + monkeypatch.setattr(module, "_fetch_pr", lambda *_args: live_pr()) + monkeypatch.setattr(module, "_active_runs", lambda *_args: [candidate, sibling]) + + def fetch_run(_repo, run_id): + """Return the sibling or transition the candidate to in-progress.""" + nonlocal candidate_fetches + if run_id == 101: + return sibling + candidate_fetches += 1 + return candidate if candidate_fetches == 1 else run_record(100, 10, status="in_progress") + + def run_json(args): + """Raise the queued-start race without permitting unrelated commands.""" + nonlocal cancel_calls + if args[-1].endswith("/cancel"): + cancel_calls += 1 + raise RuntimeError("Cannot cancel a workflow run that has not been queued yet. (HTTP409)") + raise AssertionError(args) + + monkeypatch.setattr(module, "_fetch_run", fetch_run) + monkeypatch.setattr(module, "_run_json", run_json) + + assert module.coalesce( + "ContextualWisdomLab/.github", + 1, + "ContextualWisdomLab/.github", + "feature/current", + "a" * 40, + ) == [] + assert cancel_calls == 1 + assert "Preserving run 100" in capsys.readouterr().out + + +@pytest.mark.parametrize( + ("state", "error", "expected_posts", "expected_gets"), + [ + ({"status": "completed", "conclusion": "cancelled"}, None, 1, 1), + ({"status": "in_progress", "conclusion": None}, "no longer queued", 1, 1), + ({"status": "completed", "conclusion": "success"}, "no longer queued", 1, 1), + ({"status": "mystery", "conclusion": None}, "no longer queued", 1, 1), + ], +) +def test_cancel_run_409_state_gate_never_overclaims( + monkeypatch, state, error, expected_posts, expected_gets +) -> None: + """Only cancelled terminal evidence suppresses the preservation refusal.""" + module = load_module() + calls = {"post": 0, "get": 0} + + def run_json(args): + """Raise the cancellation race for each parameterized state.""" + if args[-1].endswith("/cancel"): + calls["post"] += 1 + raise RuntimeError("Cannot cancel a workflow run that has not been queued yet. (HTTP409)") + raise AssertionError(args) + + def fetch_run(_repo, _run_id): + """Return the parameterized authoritative post-409 state.""" + calls["get"] += 1 + return state + + monkeypatch.setattr(module, "_run_json", run_json) + monkeypatch.setattr(module, "_fetch_run", fetch_run) + if error: + with pytest.raises(module.CoalescingRefused, match=error): + module._cancel_run("o/r", 123) + else: + module._cancel_run("o/r", 123) + assert calls == {"post": expected_posts, "get": expected_gets} + + +def test_cancel_run_ignores_unrelated_error_without_recheck(monkeypatch) -> None: + """A non-409 cancellation error cannot trigger a compensating mutation.""" + module = load_module() + calls: list[str] = [] + + def run_json(args): + """Raise the unrelated cancellation failure without a second request.""" + calls.append("post") + raise RuntimeError("HTTP500 upstream failure") + + monkeypatch.setattr(module, "_run_json", run_json) + monkeypatch.setattr(module, "_fetch_run", lambda *_args: calls.append("get")) + with pytest.raises(RuntimeError, match="HTTP500"): + module._cancel_run("o/r", 123) + assert calls == ["post"] + + +def test_cancel_run_fails_closed_when_queued_after_queue_start_race(monkeypatch) -> None: + """A queued run after a startup race is preserved without a second POST.""" + module = load_module() + calls = {"post": 0} + + def run_json(args): + """Raise the queue-start race while counting cancellation posts.""" + if args[-1].endswith("/cancel"): + calls["post"] += 1 + raise RuntimeError("Cannot cancel a workflow run that has not been queued yet. (HTTP409)") + raise AssertionError(args) + + monkeypatch.setattr(module, "_run_json", run_json) + monkeypatch.setattr(module, "_fetch_run", lambda *_args: {"status": "queued", "conclusion": None}) + with pytest.raises(module.CoalescingRefused, match="remained queued"): + module._cancel_run("o/r", 123) + assert calls == {"post": 1} + + +def test_cancel_run_409_detection_does_not_depend_on_provider_english(monkeypatch) -> None: + """A bare HTTP 409 still preserves a queued run without a second POST.""" + module = load_module() + calls = {"post": 0} + + def run_json(args): + """Raise a bare HTTP 409 to test language-independent detection.""" + if args[-1].endswith("/cancel"): + calls["post"] += 1 + raise RuntimeError("HTTP 409 conflict") + raise AssertionError(args) + + monkeypatch.setattr(module, "_run_json", run_json) + monkeypatch.setattr(module, "_fetch_run", lambda *_args: {"status": "queued"}) + with pytest.raises(module.CoalescingRefused, match="remained queued"): + module._cancel_run("o/r", 123) + assert calls == {"post": 1} + + def test_cancel_run_fails_when_terminal_cancellation_is_unproven(monkeypatch) -> None: """An accepted cancellation is not reported complete while GitHub stays active.""" module = load_module() @@ -577,6 +765,7 @@ def test_coalesce_refetches_candidate_last_and_preserves_started_run(monkeypatch monkeypatch.setattr(module, "_active_runs", lambda *_args: [candidate, sibling]) def fetch_run(_repo: str, run_id: int): + """Return the sibling while showing the candidate started meanwhile.""" return sibling if run_id == 101 else run_record(100, 10, status="in_progress") monkeypatch.setattr(module, "_fetch_run", fetch_run) @@ -673,6 +862,7 @@ def test_main_treats_coalescing_refused_as_a_safe_no_op(monkeypatch, capsys) -> ] def refuse(*_args: object) -> list[int]: + """Raise the safe coalescing refusal handled by the CLI entrypoint.""" raise module.CoalescingRefused("pull request head moved before duplicate classification") monkeypatch.setattr(module, "coalesce", refuse) From 509fa2a44dba6c8c8718ff35bf2161c127125cca Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 22:00:48 +0900 Subject: [PATCH 78/80] chore(sidecar): advance contextual-orchestrator pin to 767e67fb (no implicit model timeout) Advance ORCHESTRATOR_PIN_SHA from 414f2297 to protected main@767e67fbc6b881a452761f32abb69b9971b9b03b, the merge of contextual-orchestrator#1053. Under the old pin ModelClient defaulted to timeout=90 and the review gateway inherited it, so NVIDIA NIM google/gemma-4-31b-it attempts in the Noema sidecar timed out at exactly 90 s (15 of 27 attempts in fast-mlsirm#1860 run 34748511702) and the gateway answered 502 provider_connection_error after ~20 min of circuit retries. #1053 makes the model timeout null by default and administrator configured per model. Update the sidecar contract test constant, ADR-0003 (current pin and a 2026-09-13 amendment), and CHANGELOG. tests/test_contextual_orchestrator_* and sidecar contract tests: 225 passed. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 4 ++++ ...03-contextual-orchestrator-vendored-free-zdr.md | 14 +++++++++++++- .../ci/contextual_orchestrator_review_sidecar.sh | 2 +- ...textual_orchestrator_review_sidecar_contract.py | 2 +- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50e3fb6443..d118a4a47a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Contextual-orchestrator pin advance removes the implicit 90 s model request timeout + +- Advanced the central sidecar's pinned immutable CO revision from `414f2297` to protected `main@767e67fbc6b881a452761f32abb69b9971b9b03b`, carrying contextual-orchestrator#1053 into Strix, OpenCode, and Noema. Root cause: `ModelClient.__init__` defaulted `timeout=90`, and the review gateway constructed its client without a timeout, so long structured-output completions on NVIDIA NIM (`google/gemma-4-31b-it`) hit `TimeoutError` at exactly 90 s on every attempt; the orchestrator then cycled circuit open/reset on the same route for ~20 min and answered `502 provider_connection_error` (fast-mlsirm#1860 run 34748511702, sidecar artifact 10315556637: 15 of 27 failed attempts at 90.0 s; fast-mlsirm#1825 run 34752130895 same signature). #1053 removes the implicit deadline (null by default, administrator `model_timeout_seconds` per model) and was merged under the infrastructure exception because the pre-fix sidecar was failing its own Noema/OpenCode gates. Hosted acceptance is the first Noema/OpenCode/Strix run on this pin against a consumer PR; not claimed here. Refs ContextualWisdomLab/contextual-orchestrator#1053, ContextualWisdomLab/fast-mlsirm#1860. + ### Pingora edge policy admits HWPX evidence documents without UTF-8 decoding - `scripts/ci/pingora_edge_policy.py`'s `BINARY_DOCUMENT_MAGIC` only knew `.pdf` and `.png`, and `_is_binary_documentation_asset` only admitted a `doc`/`docs`/`documentation` directory, so a ZIP-based `.hwpx` evidence attachment under `evidence/` matched neither rule and fell through to the strict UTF-8 decode every other candidate gets. Observed on ContextualWisdomLab/late-life-anxiety-reanalysis#10, head `a1cd5bc6783c6510dfcf937f523c733366e82213`, run `34700409497`, job `103571044859`: "Pingora edge policy could not establish complete evidence: Runtime policy candidate evidence/reviewer_response_draft.hwpx is not valid UTF-8". The fix adds `.hwpx` (`PK\x03\x04`) to `BINARY_DOCUMENT_MAGIC` and extends `_is_binary_documentation_asset` to admit an `.hwpx` under an `evidence` path segment, gated on a bounded container check in the new `_is_complete_hwpx` -- unprefixed ZIP, exact EOCD record, unique members with `mimetype` first, a stored (not deflated) `mimetype` entry exactly `application/hwp+zip`, and a non-empty, unencrypted `Contents/content.hpf` manifest -- so no document body is ever parsed or rendered and no malware inspection is implied. The runtime-path guard and the Nginx-runtime-text fallback scan for disguised or malformed archives are unchanged. `tests/test_pingora_hwpx_evidence.py` runs the production policy boundary offline: RED (test-only apply) showed 3 failing / 19 passing; GREEN (full patch) showed 90 passing across that file plus `tests/test_pingora_edge_policy.py` and `tests/test_pingora_edge_workflow_contract.py`. Branch coverage of the touched module is 100% (388 statements, 174 branches, 0 missed) and `interrogate scripts/ci -q` reports 100.0% docstrings. Hosted acceptance still requires a newly loaded central source SHA to re-run the consumer's exact head bootstrap. Refs ContextualWisdomLab/.github#2116. diff --git a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md index 9b0749f258..6629675f14 100644 --- a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md +++ b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md @@ -24,7 +24,7 @@ all five, and auto-optimize routing by cost. 1. **Vendoring, pinned**: `scripts/ci/contextual_orchestrator_review_sidecar.sh` clones `ContextualWisdomLab/contextual-orchestrator` at an exact SHA - (`414f22973658c4ddc3d4320fcf7acd9b4e8ba991` today) into `RUNNER_TEMP`. The + (`767e67fbc6b881a452761f32abb69b9971b9b03b` today) into `RUNNER_TEMP`. The source's `requirements.lock` is installed with `--require-hashes` and `--no-deps`, so dependency resolution cannot silently move the reviewed runtime. @@ -259,6 +259,18 @@ all five, and auto-optimize routing by cost. fault. Accepted-size and tool-schema probes call the pinned client's deterministic mock response explicitly and therefore perform no provider call. +- **2026-09-13 amendment: advance the governed runtime pin to remove the + implicit 90 s model request timeout.** The vendored pin advances from + `414f22973658c4ddc3d4320fcf7acd9b4e8ba991` to + `767e67fbc6b881a452761f32abb69b9971b9b03b`, the commit that merges + `contextual-orchestrator#1053`. Under the previous pin `ModelClient` + defaulted to `timeout=90`, so every NVIDIA NIM `google/gemma-4-31b-it` + attempt in the Noema sidecar ended in `TimeoutError` at exactly 90 s (15 of + 27 attempts in fast-mlsirm#1860 run 34748511702) and the gateway surfaced + `502 provider_connection_error` after ~20 min of circuit retries. #1053 makes + the model timeout null by default and administrator-configured per model + (`model_timeout_seconds`), matching this ADR's rule that model inference + carries no wall-clock deadline. - **2026-09-06 amendment: advance the governed runtime pin to fix `orchestrator/free` retry-stacking.** The vendored pin advances from `2e414d15ba58f28597751b625a8a2f00fc9fadcf` to diff --git a/scripts/ci/contextual_orchestrator_review_sidecar.sh b/scripts/ci/contextual_orchestrator_review_sidecar.sh index 38d9551a32..3c2a1b51b9 100755 --- a/scripts/ci/contextual_orchestrator_review_sidecar.sh +++ b/scripts/ci/contextual_orchestrator_review_sidecar.sh @@ -14,7 +14,7 @@ # (fail-closed zero-cost) pool. set -euo pipefail -ORCHESTRATOR_PIN_SHA="${ORCHESTRATOR_PIN_SHA:-414f22973658c4ddc3d4320fcf7acd9b4e8ba991}" +ORCHESTRATOR_PIN_SHA="${ORCHESTRATOR_PIN_SHA:-767e67fbc6b881a452761f32abb69b9971b9b03b}" ORCHESTRATOR_GIT_URL="${ORCHESTRATOR_GIT_URL:-https://github.com/ContextualWisdomLab/contextual-orchestrator.git}" # The Strix gate and Noema SSRF guard accept this one process-local origin. # Keep it fixed so an environment override cannot create an unvalidated sidecar. diff --git a/tests/test_contextual_orchestrator_review_sidecar_contract.py b/tests/test_contextual_orchestrator_review_sidecar_contract.py index 31af868d45..b279d33a98 100644 --- a/tests/test_contextual_orchestrator_review_sidecar_contract.py +++ b/tests/test_contextual_orchestrator_review_sidecar_contract.py @@ -40,7 +40,7 @@ ) GATEWAY_MODEL = "contextual-orchestrator/orchestrator/free" -ORCH_PIN_SHA = "414f22973658c4ddc3d4320fcf7acd9b4e8ba991" +ORCH_PIN_SHA = "767e67fbc6b881a452761f32abb69b9971b9b03b" def _read(path: Path) -> str: From 04d0f65b7138d06d2643cd677047398a1e47d42e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 14 Sep 2026 02:51:42 +0900 Subject: [PATCH 79/80] fix(noema): extract office documents for review context (#2172) * fix(noema): extract office documents for review context * fix(noema): harden document XML extraction * fix(noema): install document XML dependency in quality CI --- .../agent-review-runtime-quality-ci.yml | 16 +- .github/workflows/noema-review.yml | 28 + requirements-noema-document-ci-hashes.txt | 5 + requirements-noema-document-ci.txt | 1 + .../noema-document-reader/package-lock.json | 1315 +++++++++++++++++ scripts/ci/noema-document-reader/package.json | 9 + scripts/ci/noema_hwp_mcp_reader.mjs | 38 + scripts/ci/noema_review_document.py | 223 +++ scripts/ci/noema_review_gate.py | 17 +- tests/test_noema_document_review_context.py | 249 ++++ 10 files changed, 1898 insertions(+), 3 deletions(-) create mode 100644 requirements-noema-document-ci-hashes.txt create mode 100644 requirements-noema-document-ci.txt create mode 100644 scripts/ci/noema-document-reader/package-lock.json create mode 100644 scripts/ci/noema-document-reader/package.json create mode 100644 scripts/ci/noema_hwp_mcp_reader.mjs create mode 100644 scripts/ci/noema_review_document.py create mode 100644 tests/test_noema_document_review_context.py diff --git a/.github/workflows/agent-review-runtime-quality-ci.yml b/.github/workflows/agent-review-runtime-quality-ci.yml index 6c6efc3dd1..6340b2e055 100644 --- a/.github/workflows/agent-review-runtime-quality-ci.yml +++ b/.github/workflows/agent-review-runtime-quality-ci.yml @@ -103,6 +103,8 @@ on: - "docs/doctoring/exact-artifact-sbom-quality-runner-consolidation-20260903.md" - "CHANGELOG.d/20260903-exact-artifact-quality-runner-consolidation.md" - "requirements-opencode-review-ci-hashes.txt" + - "requirements-noema-document-ci.txt" + - "requirements-noema-document-ci-hashes.txt" # PR validation only: a new head cancels only an older run of this workflow # for the same repository and pull request. @@ -138,7 +140,9 @@ jobs: with: python-version: "3.14" cache: pip - cache-dependency-path: requirements-opencode-review-ci-hashes.txt + cache-dependency-path: | + requirements-opencode-review-ci-hashes.txt + requirements-noema-document-ci-hashes.txt - name: Select affected contract suites id: affected_suites @@ -210,6 +214,10 @@ jobs: noema_suite=true opencode_suite=true ;; + requirements-noema-document-ci.txt|\ + requirements-noema-document-ci-hashes.txt) + noema_suite=true + ;; .github/workflows/pr-review-merge-scheduler.yml) queue_suite=true review_repair_suite=true @@ -327,6 +335,12 @@ jobs: python -m pip install --disable-pip-version-check --require-hashes -r requirements-opencode-review-ci-hashes.txt + - name: Install exact Noema document dependencies + if: steps.affected_suites.outputs.noema == 'true' + run: >- + python -m pip install --disable-pip-version-check --require-hashes --no-deps + -r requirements-noema-document-ci-hashes.txt + - name: Verify Noema token-lifetime contracts if: steps.affected_suites.outputs.noema == 'true' run: | diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index f8ab55c896..59bb11b8cc 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -650,6 +650,34 @@ jobs: set -euo pipefail bash "$GITHUB_WORKSPACE/scripts/ci/contextual_orchestrator_review_sidecar.sh" + - name: Provision local reviewed HWP document reader + if: env.PR_NUMBER != '' + env: + NPM_CONFIG_IGNORE_SCRIPTS: "true" + run: | + set -euo pipefail + node_major="$(node -p 'process.versions.node.split(".")[0]')" + case "$node_major" in + 20|22) ;; + *) + echo "::error::Noema HWP reader requires Node.js 20 or 22; found ${node_major:-missing}." + exit 1 + ;; + esac + python3 -m pip install --quiet --require-hashes --no-deps \ + -r "$GITHUB_WORKSPACE/requirements-noema-document-ci-hashes.txt" + reader_root="${RUNNER_TEMP}/noema-document-reader" + rm -rf "$reader_root" + mkdir -p "$reader_root" + cp "$GITHUB_WORKSPACE/scripts/ci/noema-document-reader/package.json" \ + "$GITHUB_WORKSPACE/scripts/ci/noema-document-reader/package-lock.json" \ + "$reader_root/" + ( + cd "$reader_root" + npm ci --ignore-scripts --omit=dev --no-audit --no-fund + ) + echo "NOEMA_HWP_MCP_SOURCE=$reader_root/node_modules/hwp-mcp" >>"$GITHUB_ENV" + - name: Prepare Noema model verdict if: env.PR_NUMBER != '' id: noema_prepare diff --git a/requirements-noema-document-ci-hashes.txt b/requirements-noema-document-ci-hashes.txt new file mode 100644 index 0000000000..0fd5dd54d7 --- /dev/null +++ b/requirements-noema-document-ci-hashes.txt @@ -0,0 +1,5 @@ +# Generated with uv pip compile --generate-hashes --python-version 3.12 +# requirements-noema-document-ci.txt +defusedxml==0.7.1 \ + --hash=sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69 \ + --hash=sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61 diff --git a/requirements-noema-document-ci.txt b/requirements-noema-document-ci.txt new file mode 100644 index 0000000000..09dd20d248 --- /dev/null +++ b/requirements-noema-document-ci.txt @@ -0,0 +1 @@ +defusedxml==0.7.1 diff --git a/scripts/ci/noema-document-reader/package-lock.json b/scripts/ci/noema-document-reader/package-lock.json new file mode 100644 index 0000000000..1026fd79a3 --- /dev/null +++ b/scripts/ci/noema-document-reader/package-lock.json @@ -0,0 +1,1315 @@ +{ + "name": "noema-document-reader-runtime", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "noema-document-reader-runtime", + "version": "1.0.0", + "dependencies": { + "@rhwp/core": "0.7.7", + "hwp-mcp": "0.3.0" + } + }, + "node_modules/@hono/node-server": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.1.tgz", + "integrity": "sha512-ELuehkj5VCBdgEw9zs+ivkKwyzzUCSQuE96YmiPvn1ECBoZCczbFXJLeEGMTYjphP6gydh4pHMqEYPVMYUVgQg==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "hono": "^4" + } + }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.30.0.tgz", + "integrity": "sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA==", + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9 || ^2.0.5", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/@rhwp/core": { + "version": "0.7.7", + "resolved": "https://registry.npmjs.org/@rhwp/core/-/core-0.7.7.tgz", + "integrity": "sha512-FHWTdOO+YPY4SSOaFrGGc98AkzrnQy+IIZYng3C00Wqg3+BcaN0uk0cYx0YS4bwtefrPsT6b15fcG6rpNU8iyw==", + "license": "MIT" + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, + "node_modules/cors": { + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", + "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.7.0.tgz", + "integrity": "sha512-hOwV7WOxXfjRpAM1DSJWZDXx3GhplwD8IfwuwvogD8i1Qnkgosw/H45s4ZnFAUHDAhPjlY9hLBvJhKmGMyY26g==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.7.tgz", + "integrity": "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hono": { + "version": "4.13.7", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.7.tgz", + "integrity": "sha512-c8/gF9ac8Y78/agExVocyLevgR+JlpNB444Py0FSX8pJoPdYUfUzRcXtYEYGwt6l19qIlVZPN5Mfsw9jFShmQQ==", + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/hwp-mcp": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/hwp-mcp/-/hwp-mcp-0.3.0.tgz", + "integrity": "sha512-+CYrAT5cKOpf6GCHyXRvw4SY/Z5GqBxO+Za92PyHCwseU8C6T3A3AR2bVv6hNQbG4SL6GJsgyfeaKjmpMut/Fw==", + "license": "MIT", + "dependencies": { + "@modelcontextprotocol/sdk": "^1.0.0", + "@rhwp/core": "0.7.x", + "jszip": "^3.10.1" + }, + "bin": { + "hwp-mcp": "dist/server.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "license": "MIT" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ip-address": { + "version": "10.7.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.0.tgz", + "integrity": "sha512-BGFsyJd5mpXp3rK6jIdADLNgpJUK1jnjzvYF8lK+VyDab9JAmqN0YOKDdP17HlgKb2+ehPgDc8EtnRLbGCAMhA==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jose": { + "version": "6.2.12", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.12.tgz", + "integrity": "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "license": "BSD-2-Clause" + }, + "node_modules/jszip": { + "version": "3.10.2", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.2.tgz", + "integrity": "sha512-3l+rb15IOWtUhU0H5MFqES/T6Kh7abYwjosBey/vD6hDt8zoEffkSC5Ws5SGtgVw3gBx2NEbhTeSW1+kWkpyTQ==", + "license": "(MIT OR GPL-3.0-or-later)", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "license": "MIT", + "dependencies": { + "immediate": "~3.0.5" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz", + "integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==", + "license": "MIT", + "dependencies": { + "content-type": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/negotiator/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "license": "MIT" + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/zod": { + "version": "4.6.4", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.4.tgz", + "integrity": "sha512-AXSD6hvGdvRjajG/l1cC+d6IrhH+sjmPKtYeQdJIK8MFJl3LyClzS+o/YsVC+zQZPupAaeH5skwwm8YqYH7BqA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + } + } +} diff --git a/scripts/ci/noema-document-reader/package.json b/scripts/ci/noema-document-reader/package.json new file mode 100644 index 0000000000..aa4fc0d3ff --- /dev/null +++ b/scripts/ci/noema-document-reader/package.json @@ -0,0 +1,9 @@ +{ + "name": "noema-document-reader-runtime", + "private": true, + "version": "1.0.0", + "dependencies": { + "@rhwp/core": "0.7.7", + "hwp-mcp": "0.3.0" + } +} diff --git a/scripts/ci/noema_hwp_mcp_reader.mjs b/scripts/ci/noema_hwp_mcp_reader.mjs new file mode 100644 index 0000000000..37373cbbf6 --- /dev/null +++ b/scripts/ci/noema_hwp_mcp_reader.mjs @@ -0,0 +1,38 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +const [, , sourceRoot, filePath] = process.argv; +if (!sourceRoot || !filePath || !existsSync(sourceRoot)) { + process.stderr.write("hwp-mcp source directory and document path are required\n"); + process.exit(2); +} + +try { + const hwpPackage = JSON.parse(readFileSync(join(sourceRoot, "package.json"), "utf8")); + const require = createRequire(pathToFileURL(join(sourceRoot, "package.json"))); + const rhwpPackage = JSON.parse( + readFileSync(require.resolve("@rhwp/core/package.json"), "utf8"), + ); + if (hwpPackage.name !== "hwp-mcp" || hwpPackage.version !== "0.3.0") { + throw new Error("unexpected hwp-mcp package identity"); + } + if (rhwpPackage.name !== "@rhwp/core" || rhwpPackage.version !== "0.7.7") { + throw new Error("unexpected rhwp package identity"); + } + const documentModule = await import(pathToFileURL(join(sourceRoot, "dist/core/document.js"))); + const toolsModule = await import(pathToFileURL(join(sourceRoot, "dist/tools/read.js"))); + const document = await documentModule.openDocument(filePath); + documentModule.closeDocument(document); + const text = await toolsModule.readHwp({ file_path: filePath }); + if (!text || /(?:파일 읽기 오류|File not found|text extraction error)/i.test(text)) { + throw new Error("hwp-mcp returned an extraction error"); + } + process.stdout.write(`${text}\n`); +} catch (error) { + process.stderr.write("hwp-mcp/rhwp document extraction failed\n"); + process.exit(1); +} diff --git a/scripts/ci/noema_review_document.py b/scripts/ci/noema_review_document.py new file mode 100644 index 0000000000..f0211cf035 --- /dev/null +++ b/scripts/ci/noema_review_document.py @@ -0,0 +1,223 @@ +"""Extract bounded review text from office documents without model access. + +DOCX is a ZIP/XML container whose text can be read with the Python standard +library. HWP and HWPX stay delegated to the reviewed hwp-mcp/rhwp reader; this +module only supplies a temporary local file and validates the subprocess +contract. +""" + +from __future__ import annotations + +import io +import os +import subprocess +import tempfile +import zipfile +from pathlib import PurePosixPath + +from defusedxml import ElementTree as ET + + +MAX_DOCUMENT_BYTES = 8 * 1024 * 1024 +MAX_DOCUMENT_ZIP_ENTRIES = 2048 +MAX_DOCUMENT_ZIP_UNCOMPRESSED_BYTES = 64 * 1024 * 1024 +MAX_DOCUMENT_TEXT_BYTES = 256 * 1024 +HWP_READER_ENV = "NOEMA_HWP_MCP_SOURCE" +HWP_READER_TIMEOUT_SECONDS = 45 + +W_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" +M_NS = "http://schemas.openxmlformats.org/officeDocument/2006/math" +W = f"{{{W_NS}}}" +M = f"{{{M_NS}}}" + + +class DocumentReadError(RuntimeError): + """A document could not be converted to bounded review text.""" + + +def extract_review_document(path: str, raw: bytes) -> str: + """Return text for one supported document path or fail closed. + + The input bytes are obtained from the exact GitHub content ref by the + caller. HWP/HWPX bytes are never decoded as UTF-8 and never sent to an + external service; the configured reader runs as a local subprocess only. + """ + if len(raw) > MAX_DOCUMENT_BYTES: + raise DocumentReadError("document exceeds the bounded 8 MiB review input") + suffix = PurePosixPath(path).suffix.lower() + if suffix == ".docx": + return _extract_docx(raw) + if suffix in {".hwp", ".hwpx"}: + return _extract_hwp_with_reviewed_reader(path, raw) + raise DocumentReadError(f"unsupported review document format: {suffix or ''}") + + +def _extract_docx(raw: bytes) -> str: + """Extract paragraphs, tables, and Office Math text from one DOCX.""" + try: + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + infos = archive.infolist() + if len(infos) > MAX_DOCUMENT_ZIP_ENTRIES: + raise DocumentReadError("DOCX archive has too many entries") + if ( + sum(info.file_size for info in infos) + > MAX_DOCUMENT_ZIP_UNCOMPRESSED_BYTES + ): + raise DocumentReadError( + "DOCX archive exceeds the bounded unpacked size" + ) + try: + document_xml = archive.read("word/document.xml") + except KeyError as exc: + raise DocumentReadError( + "DOCX archive has no word/document.xml" + ) from exc + except DocumentReadError: + raise + except (zipfile.BadZipFile, OSError, ValueError) as exc: + raise DocumentReadError("DOCX archive is malformed") from exc + + try: + root = ET.fromstring(document_xml) + except ET.ParseError as exc: + raise DocumentReadError("DOCX document.xml is malformed") from exc + + body = root.find(f"{W}body") + if body is None: + raise DocumentReadError("DOCX document.xml has no document body") + + sections: list[str] = [] + table_number = 0 + for child in body: + if child.tag == f"{W}p": + text = _paragraph_text(child) + if text: + sections.append(text) + elif child.tag == f"{W}tbl": + table_number += 1 + table = _table_markdown(child, table_number) + if table: + sections.append(table) + + text = "\n\n".join(sections).strip() + if not text: + raise DocumentReadError("DOCX contains no readable text") + return _bounded_text(text) + + +def _paragraph_text(paragraph: ET.Element) -> str: + """Keep visible Word text, tabs, breaks, and Office Math runs.""" + parts: list[str] = [] + for element in paragraph.iter(): + if element.tag in {f"{W}t", f"{W}instrText", f"{M}t"}: + parts.append(element.text or "") + elif element.tag == f"{W}tab": + parts.append("\t") + elif element.tag in {f"{W}br", f"{W}cr"}: + parts.append("\n") + return "".join(parts).strip() + + +def _table_markdown(table: ET.Element, table_number: int) -> str: + """Render a DOCX table as bounded, reviewer-readable Markdown.""" + rows: list[list[str]] = [] + for row in table.findall(f"{W}tr"): + cells: list[str] = [] + for cell in row.findall(f"{W}tc"): + paragraphs = [_paragraph_text(p) for p in cell.findall(f".//{W}p")] + value = "\n".join(text for text in paragraphs if text).strip() + cells.append(value.replace("|", "\\|")) + if cells: + rows.append(cells) + if not rows: + return "" + + width = max(len(row) for row in rows) + normalized = [row + [""] * (width - len(row)) for row in rows] + lines = [f"### Table {table_number} ({len(normalized)} rows x {width} columns)"] + lines.append("| " + " | ".join(normalized[0]) + " |") + lines.append("| " + " | ".join("---" for _ in range(width)) + " |") + lines.extend("| " + " | ".join(row) + " |" for row in normalized[1:]) + return "\n".join(lines) + + +def _extract_hwp_with_reviewed_reader(path: str, raw: bytes) -> str: + """Delegate HWP/HWPX parsing to the reviewed hwp-mcp/rhwp source tree.""" + source = os.environ.get(HWP_READER_ENV, "").strip() + if not source: + raise DocumentReadError( + "reviewed hwp-mcp/rhwp reader is not configured; " + f"set {HWP_READER_ENV} to its trusted source directory" + ) + reader = os.path.join(os.path.dirname(__file__), "noema_hwp_mcp_reader.mjs") + with tempfile.NamedTemporaryFile( + prefix="noema-document-", suffix=PurePosixPath(path).suffix + ) as handle: + handle.write(raw) + handle.flush() + try: + completed = subprocess.run( + ["node", reader, source, handle.name], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + shell=False, + timeout=HWP_READER_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired as exc: + raise DocumentReadError( + "reviewed hwp-mcp/rhwp reader timed out after " + f"{HWP_READER_TIMEOUT_SECONDS} seconds" + ) from exc + except OSError as exc: + raise DocumentReadError( + "reviewed hwp-mcp/rhwp reader could not start" + ) from exc + if completed.returncode != 0: + raise DocumentReadError( + f"reviewed hwp-mcp/rhwp reader failed (exit {completed.returncode})" + ) + if len(completed.stdout) > MAX_DOCUMENT_TEXT_BYTES: + raise DocumentReadError( + "reviewed hwp-mcp/rhwp reader exceeded the bounded output" + ) + try: + text = completed.stdout.decode("utf-8").strip() + except UnicodeDecodeError as exc: + raise DocumentReadError( + "reviewed hwp-mcp/rhwp reader returned non-UTF-8 text" + ) from exc + if not text: + raise DocumentReadError("reviewed hwp-mcp/rhwp reader returned empty text") + return _bounded_text(text) + + +def _bounded_text(text: str) -> str: + """Bound reader output before it enters the review prompt.""" + encoded = text.encode("utf-8") + if len(encoded) <= MAX_DOCUMENT_TEXT_BYTES: + return text + clipped = encoded[:MAX_DOCUMENT_TEXT_BYTES].decode("utf-8", errors="ignore") + omitted = len(encoded) - len(clipped.encode("utf-8")) + return f"{clipped}\n[document text truncated; {omitted} bytes omitted]" + + +def _main() -> int: + """Provide a local, byte-safe smoke-test CLI for one document.""" + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("path") + args = parser.parse_args() + try: + with open(args.path, "rb") as handle: + text = extract_review_document(args.path, handle.read()) + except (OSError, DocumentReadError) as exc: + print(str(exc), file=os.sys.stderr) + return 1 + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 5ab7e830f3..a133bc3f30 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -6,6 +6,7 @@ import argparse import ast import base64 +import binascii import hashlib import http.client import ipaddress @@ -20,9 +21,11 @@ import urllib.parse import urllib.request from collections.abc import Sequence +from pathlib import PurePosixPath from typing import Any from scripts.ci.opencode_review_normalize_output import changed_file_is_material +from scripts.ci.noema_review_document import DocumentReadError, extract_review_document PRIMARY_REVIEW_AUTHORS = { @@ -736,7 +739,7 @@ def fetch_changed_files(repo: str, number: int) -> list[tuple[str, str]]: def fetch_file_content_at_ref(repo: str, path: str, ref: str) -> str: - """Fetch one repository text file at an exact Git ref through GitHub.""" + """Fetch one repository file at an exact Git ref through GitHub.""" encoded_path = urllib.parse.quote(path, safe="/") encoded_ref = urllib.parse.quote(ref, safe="") content = run( @@ -751,7 +754,17 @@ def fetch_file_content_at_ref(repo: str, path: str, ref: str) -> str: compact = "".join(content.split()) if not compact: return "" - return base64.b64decode(compact).decode("utf-8", errors="replace") + try: + raw = base64.b64decode(compact, validate=True) + except (binascii.Error, ValueError) as exc: + raise RuntimeError("GitHub content response contained malformed base64") from exc + suffix = PurePosixPath(path).suffix.lower() + if suffix in {".docx", ".hwp", ".hwpx"}: + try: + return extract_review_document(path, raw) + except DocumentReadError as exc: + raise RuntimeError(f"document extraction failed: {exc}") from exc + return raw.decode("utf-8", errors="replace") def fetch_merge_base_sha(repo: str, base_sha: str, head_sha: str) -> str: diff --git a/tests/test_noema_document_review_context.py b/tests/test_noema_document_review_context.py new file mode 100644 index 0000000000..1cfa6c40b1 --- /dev/null +++ b/tests/test_noema_document_review_context.py @@ -0,0 +1,249 @@ +"""Regression tests for binary document input on the canonical Noema path.""" + +from __future__ import annotations + +import base64 +import io +import json +import os +import zipfile +from pathlib import Path + +import pytest + +from scripts.ci import noema_review_document as document +from scripts.ci import noema_review_gate as noema + + +def _docx_bytes(*, malformed: bool = False) -> bytes: + """Build a synthetic DOCX containing body, table, and Office Math text.""" + if malformed: + return b"not a zip archive" + xml = """ + + + DOCX-REVIEW-MARKERx+y + table-cell-a + table-cell-b + +""" + output = io.BytesIO() + with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("word/document.xml", xml) + return output.getvalue() + + +def _pr() -> dict[str, object]: + return { + "headRefOid": "head", + "baseRefOid": "base", + "title": "document review input", + "reviewThreads": {"nodes": []}, + } + + +def test_hosted_reader_bundle_is_pinned_and_local(): + """The hosted workflow must install only the reviewed local reader bundle.""" + repository_root = Path(__file__).resolve().parents[1] + workflow = (repository_root / ".github/workflows/noema-review.yml").read_text( + encoding="utf-8" + ) + quality_workflow = ( + repository_root + / ".github/workflows/agent-review-runtime-quality-ci.yml" + ).read_text(encoding="utf-8") + package = json.loads( + (repository_root / "scripts/ci/noema-document-reader/package.json").read_text( + encoding="utf-8" + ) + ) + lock = json.loads( + ( + repository_root / "scripts/ci/noema-document-reader/package-lock.json" + ).read_text(encoding="utf-8") + ) + + assert "Provision local reviewed HWP document reader" in workflow + assert 'NPM_CONFIG_IGNORE_SCRIPTS: "true"' in workflow + assert "npm ci --ignore-scripts --omit=dev --no-audit --no-fund" in workflow + assert "NOEMA_HWP_MCP_SOURCE=$reader_root/node_modules/hwp-mcp" in workflow + assert package["dependencies"] == {"@rhwp/core": "0.7.7", "hwp-mcp": "0.3.0"} + assert lock["packages"]["node_modules/hwp-mcp"]["version"] == "0.3.0" + assert lock["packages"]["node_modules/@rhwp/core"]["version"] == "0.7.7" + assert "requirements-noema-document-ci-hashes.txt" in workflow + assert "python3 -m pip install --quiet --require-hashes --no-deps" in workflow + assert "requirements-noema-document-ci-hashes.txt" in quality_workflow + assert "Install exact Noema document dependencies" in quality_workflow + + +def test_docx_text_reaches_the_actual_reviewer_payload(monkeypatch): + """The extracted document context must be inside the model request body.""" + raw = _docx_bytes() + encoded = base64.b64encode(raw).decode("ascii") + + def fake_run(args, stdin=None): + assert "contents/docs/review.docx?ref=head" in args[2] + return encoded + + monkeypatch.setattr(noema, "run", fake_run) + context = noema.build_review_context( + "owner/repo", 7, _pr(), [("docs/review.docx", "modified")] + ) + assert "DOCX-REVIEW-MARKER" in context + assert "x+y" in context + assert "table-cell-a" in context + assert "table-cell-b" in context + + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example.test/chat") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + monkeypatch.setattr(noema, "validate_substantive_verdict", lambda *_args: None) + captured: dict[str, object] = {} + + class Response: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def read(self): + verdict = {"decision": "comment", "summary": "checked", "findings": []} + return json.dumps( + {"choices": [{"message": {"content": json.dumps(verdict)}}]} + ).encode() + + class Opener: + def open(self, request): + captured.update(json.loads(request.data.decode())) + return Response() + + monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *_args: Opener()) + noema.call_llm( + "owner/repo", + 7, + _pr(), + "diff --git a/docs/review.docx b/docs/review.docx\n+binary\n", + False, + "head", + context, + ("docs/review.docx",), + ) + prompt = captured["messages"][1]["content"] + assert "DOCX-REVIEW-MARKER" in prompt + assert "table-cell-a" in prompt + + +def test_malformed_docx_is_explicit_in_review_context(monkeypatch): + """Malformed document bytes are reported instead of UTF-8 replacement text.""" + encoded = base64.b64encode(_docx_bytes(malformed=True)).decode("ascii") + monkeypatch.setattr(noema, "run", lambda _args, stdin=None: encoded) + + context = noema.changed_file_context( + "owner/repo", 7, "head", changed_files=[("docs/broken.docx", "modified")] + ) + + assert "### docs/broken.docx" in context + assert "document extraction failed: DOCX archive is malformed" in context + assert "not a zip archive" not in context + + +def test_hwp_reader_contract_is_local_and_fail_closed(monkeypatch): + """HWP/HWPX use the configured local adapter and reject failed readers.""" + monkeypatch.setenv(document.HWP_READER_ENV, "/trusted/hwp-mcp-source") + completed = document.subprocess.CompletedProcess( + ["node"], 0, stdout=b"HWP-REVIEW-MARKER\n", stderr=b"" + ) + monkeypatch.setattr(document.subprocess, "run", lambda *args, **kwargs: completed) + assert ( + document.extract_review_document("docs/review.hwpx", b"binary") + == "HWP-REVIEW-MARKER" + ) + + failed = document.subprocess.CompletedProcess( + ["node"], 1, stdout=b"", stderr=b"private parser details" + ) + monkeypatch.setattr(document.subprocess, "run", lambda *args, **kwargs: failed) + try: + document.extract_review_document("docs/broken.hwp", b"binary") + except document.DocumentReadError as exc: + assert str(exc) == "reviewed hwp-mcp/rhwp reader failed (exit 1)" + else: + raise AssertionError("expected failed local HWP reader to fail closed") + + def timed_out(*args, **kwargs): + raise document.subprocess.TimeoutExpired(args[0], kwargs["timeout"]) + + monkeypatch.setattr(document.subprocess, "run", timed_out) + try: + document.extract_review_document("docs/slow.hwpx", b"binary") + except document.DocumentReadError as exc: + assert "timed out after" in str(exc) + else: + raise AssertionError("expected hung local HWP reader to fail closed") + + +@pytest.mark.parametrize( + ("fixture_name", "expected_text"), + [("simple.hwp", "안녕하세요 hwp-mcp."), ("text_only.hwpx", "hwpx 텍스트.")], +) +def test_real_hwp_mcp_fixture_text_reaches_reviewer_payload( + monkeypatch, fixture_name, expected_text +): + """The reviewed local hwp-mcp/rhwp fixture reaches the Noema request.""" + source = Path(os.environ.get(document.HWP_READER_ENV, "")) + fixture = source / "test" / "fixtures" / fixture_name + if not fixture.is_file(): + pytest.skip( + "NOEMA_HWP_MCP_SOURCE is not configured with local reviewed fixtures" + ) + + monkeypatch.setenv(document.HWP_READER_ENV, str(source)) + encoded = base64.b64encode(fixture.read_bytes()).decode("ascii") + monkeypatch.setattr(noema, "run", lambda _args, stdin=None: encoded) + context = noema.build_review_context( + "owner/repo", 7, _pr(), [(f"docs/{fixture_name}", "modified")] + ) + assert expected_text in context + if fixture_name == "simple.hwp": + assert "| 이름 | 회사 |" in context + assert "| 남대현 | 포텐랩 |" in context + + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example.test/chat") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + monkeypatch.setattr(noema, "validate_substantive_verdict", lambda *_args: None) + captured: dict[str, object] = {} + + class Response: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def read(self): + verdict = {"decision": "comment", "summary": "checked", "findings": []} + return json.dumps( + {"choices": [{"message": {"content": json.dumps(verdict)}}]} + ).encode() + + class Opener: + def open(self, request): + captured.update(json.loads(request.data.decode())) + return Response() + + monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *_args: Opener()) + noema.call_llm( + "owner/repo", + 7, + _pr(), + f"diff --git a/docs/{fixture_name} b/docs/{fixture_name}\n+binary\n", + False, + "head", + context, + (f"docs/{fixture_name}",), + ) + prompt = captured["messages"][1]["content"] + assert expected_text in prompt + if fixture_name == "simple.hwp": + assert "| 이름 | 회사 |" in prompt From 828eaaefb0cc97bba4da63eb9270447476d26710 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 14 Sep 2026 02:57:18 +0900 Subject: [PATCH 80/80] fix(noema): close document reader review gaps (#2178) Follow-up to #2172. The required hosted workflows remained queued repository-wide with zero available Actions runners; local focused, fixture, security, and syntax validation passed for the exact head. Apply the reviewed document-reader fixes. --- .../agent-review-runtime-quality-ci.yml | 16 +++++++++-- scripts/ci/noema_hwp_mcp_reader.mjs | 5 +++- scripts/ci/noema_review_document.py | 3 +- tests/test_noema_document_review_context.py | 28 +++++++++++++++++++ 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/.github/workflows/agent-review-runtime-quality-ci.yml b/.github/workflows/agent-review-runtime-quality-ci.yml index 6340b2e055..3976424bfb 100644 --- a/.github/workflows/agent-review-runtime-quality-ci.yml +++ b/.github/workflows/agent-review-runtime-quality-ci.yml @@ -11,6 +11,11 @@ on: - "tests/test_noema_two_phase_handoff.py" - "tests/test_noema_refreshed_app_identity.py" - "tests/test_noema_token_lifetime_stale_run_contract.py" + - "scripts/ci/noema_review_document.py" + - "scripts/ci/noema_hwp_mcp_reader.mjs" + - "scripts/ci/noema-document-reader/package.json" + - "scripts/ci/noema-document-reader/package-lock.json" + - "tests/test_noema_document_review_context.py" - "docs/doctoring/noema-review-token-lifetime.md" - "docs/product-technical-gap-baseline.md" - ".github/workflows/opencode-review-dispatch.yml" @@ -185,6 +190,11 @@ jobs: tests/test_noema_two_phase_handoff.py|\ tests/test_noema_refreshed_app_identity.py|\ tests/test_noema_token_lifetime_stale_run_contract.py|\ + scripts/ci/noema_review_document.py|\ + scripts/ci/noema_hwp_mcp_reader.mjs|\ + scripts/ci/noema-document-reader/package.json|\ + scripts/ci/noema-document-reader/package-lock.json|\ + tests/test_noema_document_review_context.py|\ docs/doctoring/noema-review-token-lifetime.md) noema_suite=true ;; @@ -349,13 +359,15 @@ jobs: tests/test_noema_reviewer_token_lifetime.py \ tests/test_noema_two_phase_handoff.py \ tests/test_noema_refreshed_app_identity.py \ - tests/test_noema_token_lifetime_stale_run_contract.py + tests/test_noema_token_lifetime_stale_run_contract.py \ + tests/test_noema_document_review_context.py python -m compileall -q \ .github/actions/noema-review/two_phase.py \ tests/test_noema_reviewer_token_lifetime.py \ tests/test_noema_two_phase_handoff.py \ tests/test_noema_refreshed_app_identity.py \ - tests/test_noema_token_lifetime_stale_run_contract.py + tests/test_noema_token_lifetime_stale_run_contract.py \ + tests/test_noema_document_review_context.py - name: Verify OpenCode Rust coverage toolchain contract if: steps.affected_suites.outputs.opencode == 'true' diff --git a/scripts/ci/noema_hwp_mcp_reader.mjs b/scripts/ci/noema_hwp_mcp_reader.mjs index 37373cbbf6..ccd32683b7 100644 --- a/scripts/ci/noema_hwp_mcp_reader.mjs +++ b/scripts/ci/noema_hwp_mcp_reader.mjs @@ -28,7 +28,10 @@ try { const document = await documentModule.openDocument(filePath); documentModule.closeDocument(document); const text = await toolsModule.readHwp({ file_path: filePath }); - if (!text || /(?:파일 읽기 오류|File not found|text extraction error)/i.test(text)) { + if ( + !text || + /^(?:파일 읽기 오류|File not found|텍스트 추출 오류|text extraction error)/i.test(text) + ) { throw new Error("hwp-mcp returned an extraction error"); } process.stdout.write(`${text}\n`); diff --git a/scripts/ci/noema_review_document.py b/scripts/ci/noema_review_document.py index f0211cf035..17d3ca603c 100644 --- a/scripts/ci/noema_review_document.py +++ b/scripts/ci/noema_review_document.py @@ -16,6 +16,7 @@ from pathlib import PurePosixPath from defusedxml import ElementTree as ET +from defusedxml.common import DefusedXmlException MAX_DOCUMENT_BYTES = 8 * 1024 * 1024 @@ -79,7 +80,7 @@ def _extract_docx(raw: bytes) -> str: try: root = ET.fromstring(document_xml) - except ET.ParseError as exc: + except (ET.ParseError, DefusedXmlException) as exc: raise DocumentReadError("DOCX document.xml is malformed") from exc body = root.find(f"{W}body") diff --git a/tests/test_noema_document_review_context.py b/tests/test_noema_document_review_context.py index 1cfa6c40b1..e6ec2e6d70 100644 --- a/tests/test_noema_document_review_context.py +++ b/tests/test_noema_document_review_context.py @@ -34,6 +34,19 @@ def _docx_bytes(*, malformed: bool = False) -> bytes: return output.getvalue() +def _docx_entity_bytes() -> bytes: + """Build a DOCX whose entity declaration must be rejected safely.""" + xml = """ +]> + + &expansion; +""" + output = io.BytesIO() + with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("word/document.xml", xml) + return output.getvalue() + + def _pr() -> dict[str, object]: return { "headRefOid": "head", @@ -75,6 +88,15 @@ def test_hosted_reader_bundle_is_pinned_and_local(): assert "python3 -m pip install --quiet --require-hashes --no-deps" in workflow assert "requirements-noema-document-ci-hashes.txt" in quality_workflow assert "Install exact Noema document dependencies" in quality_workflow + for path in ( + "scripts/ci/noema_review_document.py", + "scripts/ci/noema_hwp_mcp_reader.mjs", + "scripts/ci/noema-document-reader/package.json", + "scripts/ci/noema-document-reader/package-lock.json", + "tests/test_noema_document_review_context.py", + ): + assert path in quality_workflow + assert "tests/test_noema_document_review_context.py" in quality_workflow def test_docx_text_reaches_the_actual_reviewer_payload(monkeypatch): @@ -148,6 +170,12 @@ def test_malformed_docx_is_explicit_in_review_context(monkeypatch): assert "not a zip archive" not in context +def test_forbidden_docx_entities_are_explicitly_rejected(): + """Defused XML entity failures become the same bounded reader error.""" + with pytest.raises(document.DocumentReadError, match="DOCX document.xml is malformed"): + document.extract_review_document("docs/entity.docx", _docx_entity_bytes()) + + def test_hwp_reader_contract_is_local_and_fail_closed(monkeypatch): """HWP/HWPX use the configured local adapter and reject failed readers.""" monkeypatch.setenv(document.HWP_READER_ENV, "/trusted/hwp-mcp-source")