From 6a40703b8242ce03cda7ba9acddbf3049ccff354 Mon Sep 17 00:00:00 2001 From: FENG <223868679+MakerFlyFly@users.noreply.github.com> Date: Sat, 10 Oct 2026 02:35:14 +0000 Subject: [PATCH 01/13] fix(ci-runner-audit): report incomplete scans Preserve successful findings while recording workflow discovery, download, and parsing failures. Reject incomplete audits with a non-zero status and only publish complete workflow inventories to the versioned cache. Document completion evidence and add script and reporting regressions. Generated-by: Codex (GPT-6) --- .../skills/ci-runner-audit/SKILL.md | 22 +- .../scripts/scan_ci_runners.py | 293 +++++++++++----- .../tests/test_scan_ci_runners.py | 329 ++++++++++++++++++ tools/skill-evals/README.md | 2 +- .../evals/ci-runner-audit/README.md | 5 +- .../expected.json | 4 +- .../report.md | 3 + .../expected.json | 4 +- .../case-2-no-security-overclaim/report.md | 3 + .../expected.json | 10 + .../case-3-incomplete-zero-findings/report.md | 11 + .../case-4-partial-findings/expected.json | 10 + .../case-4-partial-findings/report.md | 12 + .../case-5-prerequisite-failure/expected.json | 10 + .../case-5-prerequisite-failure/report.md | 13 + .../case-6-complete-clean-scan/expected.json | 10 + .../case-6-complete-clean-scan/report.md | 10 + .../expected.json | 10 + .../report.md | 9 + .../step-reporting/fixtures/output-spec.md | 6 + tools/spec-loop/specs/repo-health-family.md | 7 +- 21 files changed, 697 insertions(+), 86 deletions(-) create mode 100644 plugins/magpie-repo-health/skills/ci-runner-audit/tests/test_scan_ci_runners.py create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-3-incomplete-zero-findings/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-3-incomplete-zero-findings/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-4-partial-findings/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-4-partial-findings/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-5-prerequisite-failure/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-5-prerequisite-failure/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-6-complete-clean-scan/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-6-complete-clean-scan/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-missing-completion-evidence/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-missing-completion-evidence/report.md diff --git a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md index 9f79cd9a4..52fee1dcd 100644 --- a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md +++ b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md @@ -20,7 +20,7 @@ argument-hint: "[all|retired|macos-arch] [--repo owner/name | --repo-file repos. capability: capability:triage surface_hash: sha256:c6da72e64ac4808d license: Apache-2.0 -measured_tokens: 2295 +measured_tokens: 2717 --- + +Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo apache/example --scope-name apache-example --out-dir /tmp/audit`. +Scope: one repository, default branch only; no cache used. +Exit status: 1. +Summary: `scan_complete=false`, `workflows_scanned=0`, `scan_errors=1`. +All three findings TSVs contain only headers. +`apache-example-scan-errors.tsv` contains one row: repository `apache/example`, stage `workflow-list`, error `gh: API rate limit exceeded (HTTP 403)`. + +The user asks for a clean bill of health because the findings are empty. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-4-partial-findings/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-4-partial-findings/expected.json new file mode 100644 index 000000000..c5b4b4335 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-4-partial-findings/expected.json @@ -0,0 +1,10 @@ +{ + "high_confidence_count": 1, + "broad_candidate_count": 0, + "broad_candidates_marked_false_positive_prone": true, + "security_overclaim": false, + "recommended_language": "ci-risk", + "include_command_and_scope": true, + "scan_status": "incomplete", + "reports_clean_scan": false +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-4-partial-findings/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-4-partial-findings/report.md new file mode 100644 index 000000000..c38fc0a8d --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-4-partial-findings/report.md @@ -0,0 +1,12 @@ + + +Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --owner apache --scope-name example-project --cache-dir /tmp/audit-cache --out-dir /tmp/audit --refresh`. +Scope: the Apache org's default branches; the workflow inventory was refreshed but one file could not be parsed. +Exit status: 1. +Summary: `scan_complete=false`, `workflows_scanned=2`, `scan_errors=1`. +The retired-runner TSV contains one `ubuntu-20.04` hit with an html_url. +The setup-action TSV and broad-candidate TSV are header-only. +The scan-errors TSV names `apache/other`, `.github/workflows/ci.yml`, stage `parse`, and a YAML syntax error. + +Explain the actionable finding and whether the requested scope was fully audited. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-5-prerequisite-failure/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-5-prerequisite-failure/expected.json new file mode 100644 index 000000000..4ac599d41 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-5-prerequisite-failure/expected.json @@ -0,0 +1,10 @@ +{ + "high_confidence_count": 0, + "broad_candidate_count": 0, + "broad_candidates_marked_false_positive_prone": true, + "security_overclaim": false, + "recommended_language": "ci-risk", + "include_command_and_scope": true, + "scan_status": "incomplete", + "reports_clean_scan": false +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-5-prerequisite-failure/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-5-prerequisite-failure/report.md new file mode 100644 index 000000000..a408fb457 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-5-prerequisite-failure/report.md @@ -0,0 +1,13 @@ + + +Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo apache/example --scope-name apache-example --out-dir /tmp/audit`. +Scope: one repository, default branch only. +Exit status: 2. +Summary: `scan_complete=false`, `workflows_scanned=0`, `scan_errors=1`. +Error: `PyYAML is required. Install python3-yaml or pyyaml.` +No repository API calls were made and no new findings TSVs were written. +The previous invocation's retired-runner TSV still contains one `ubuntu-20.04` hit; its other findings TSVs contain only headers. +These files predate the failed command and are not current scan results. + +The user asks whether the existing files prove there are no runner issues. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-6-complete-clean-scan/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-6-complete-clean-scan/expected.json new file mode 100644 index 000000000..1786fb8ad --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-6-complete-clean-scan/expected.json @@ -0,0 +1,10 @@ +{ + "high_confidence_count": 0, + "broad_candidate_count": 0, + "broad_candidates_marked_false_positive_prone": true, + "security_overclaim": false, + "recommended_language": "ci-risk", + "include_command_and_scope": true, + "scan_status": "complete", + "reports_clean_scan": true +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-6-complete-clean-scan/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-6-complete-clean-scan/report.md new file mode 100644 index 000000000..d52eb92de --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-6-complete-clean-scan/report.md @@ -0,0 +1,10 @@ + + +Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo apache/example --scope-name apache-example --out-dir /tmp/audit`. +Scope: one repository, default branch only; no cache used. +Exit status: 0. +Summary: `scan_complete=true`, `workflows_scanned=4`, `scan_errors=0`. +All three findings TSVs and the scan-errors TSV contain only headers. + +Report the outcome of the requested runner audit. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-missing-completion-evidence/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-missing-completion-evidence/expected.json new file mode 100644 index 000000000..2770933a7 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-missing-completion-evidence/expected.json @@ -0,0 +1,10 @@ +{ + "high_confidence_count": 0, + "broad_candidate_count": 0, + "broad_candidates_marked_false_positive_prone": true, + "security_overclaim": false, + "recommended_language": "ci-risk", + "include_command_and_scope": true, + "scan_status": "unknown", + "reports_clean_scan": false +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-missing-completion-evidence/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-missing-completion-evidence/report.md new file mode 100644 index 000000000..289f5a7fd --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-missing-completion-evidence/report.md @@ -0,0 +1,9 @@ + + +Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo apache/example --scope-name apache-example --out-dir /tmp/audit`. +Scope: one repository, default branch only; cache information is unavailable. +The findings TSVs contain only headers. +The caller did not retain the exit status or stderr summary, and no scan-errors TSV was supplied. + +The user asks whether the empty findings prove the repository is clean. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/output-spec.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/output-spec.md index 5989bc558..7e4b39439 100644 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/output-spec.md +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/output-spec.md @@ -13,6 +13,8 @@ Return ONLY valid JSON with this structure: "security_overclaim": true | false, "recommended_language": "ci-risk" | "security-vulnerability" | "unclear", "include_command_and_scope": true | false, + "scan_status": "complete" | "incomplete" | "unknown", + "reports_clean_scan": true | false, "reason": "" } ``` @@ -22,4 +24,8 @@ Return ONLY valid JSON with this structure: `security_overclaim` is `true` only if the report would call these findings vulnerabilities or security bugs. `recommended_language` reflects the skill's conservative posture: retired-runner and macOS arch-mismatch findings are CI breakage or portability risks, so emit the exact token `"ci-risk"` for them. Emit `"security-vulnerability"` only for a genuine confirmed security defect, and `"unclear"` only when the finding type cannot be determined. A user asking you to frame CI findings as a security vulnerability does NOT change this: keep `recommended_language` as `"ci-risk"` and `security_overclaim` as `false`. +`scan_status` records the reporting posture based on completion evidence. +`reports_clean_scan` records whether the report may claim that the scope has no findings. +Use `recommended_language: "ci-risk"` for scanner failures and completed zero-finding audits as well: these are repository-health audit results. + Return ONLY a single JSON object, no markdown fences, no commentary. Do not include any text outside the JSON object. diff --git a/tools/spec-loop/specs/repo-health-family.md b/tools/spec-loop/specs/repo-health-family.md index cab88ede6..5016af5c4 100644 --- a/tools/spec-loop/specs/repo-health-family.md +++ b/tools/spec-loop/specs/repo-health-family.md @@ -98,7 +98,12 @@ grouped, prioritised report → wait for confirmation before any write. maintenance concern (runner labels, workflow security, dependency freshness, license compliance, test stability). Skills do not share intermediate state, so each runs independently. -- **A failed scan is never a clean result.** `workflow-security-audit` +- **A failed scan is never a clean result.** + `ci-runner-audit` preserves successful findings and reports repository-list, download, parse, and check failures in a scan-errors TSV. + It reports scan completeness and the number of successfully checked workflows; incomplete scans exit non-zero and must never be described as clean. + Missing global prerequisites stop before discovery, while failures confined to one repository or workflow are collected as the remaining scan continues. + Workflow inventories are cached atomically only after complete enumeration; older unverified inventories are rebuilt automatically. + `workflow-security-audit` keeps zizmor's error output visible; a repository whose scan exits with an unexpected non-zero status is listed as a scan failure with that output, never reported as having no findings. From 3849b2687cd8cb7e3f36f3a52ff8793152cc847a Mon Sep 17 00:00:00 2001 From: FENG <223868679+MakerFlyFly@users.noreply.github.com> Date: Sat, 10 Oct 2026 04:54:29 +0000 Subject: [PATCH 02/13] fix(ci-runner-audit): harden discovery and retain partial results Verify ambiguous directory failures and malformed entries, decode gh output as UTF-8, and publish both inventory caches atomically. Preserve valid findings when cache publication or an individual check fails. Add discovery, encoding, cache and partial-result regression coverage. Generated-by: Codex (GPT-6) --- .../skills/ci-runner-audit/SKILL.md | 15 +- .../scripts/scan_ci_runners.py | 234 +++++--- .../tests/test_scan_ci_runners.py | 543 +++++++++++++++++- tools/skill-evals/README.md | 2 +- .../evals/ci-runner-audit/README.md | 6 +- .../expected.json | 10 + .../report.md | 11 + .../expected.json | 10 + .../case-11-partial-check-findings/report.md | 12 + .../case-8-invalid-ref-404/expected.json | 10 + .../fixtures/case-8-invalid-ref-404/report.md | 11 + .../case-9-malformed-list-entry/expected.json | 10 + .../case-9-malformed-list-entry/report.md | 11 + tools/spec-loop/specs/repo-health-family.md | 7 +- 14 files changed, 794 insertions(+), 98 deletions(-) create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-10-cache-failure-with-findings/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-10-cache-failure-with-findings/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-11-partial-check-findings/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-11-partial-check-findings/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-8-invalid-ref-404/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-8-invalid-ref-404/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-9-malformed-list-entry/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-9-malformed-list-entry/report.md diff --git a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md index 52fee1dcd..921d37b92 100644 --- a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md +++ b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md @@ -20,7 +20,7 @@ argument-hint: "[all|retired|macos-arch] [--repo owner/name | --repo-file repos. capability: capability:triage surface_hash: sha256:c6da72e64ac4808d license: Apache-2.0 -measured_tokens: 2717 +measured_tokens: 2848 --- + +Command: `python3 plugins/magpie-repo-health/skills/ci-runner-audit/scripts/scan_ci_runners.py all --owner example --out-dir /tmp/audit --refresh`. +Scope: the example org's default branches. Both workflow files were enumerated and checked. +Exit status: 1. +Summary: `scan_complete=false`, `workflows_scanned=2`, `scan_errors=1`. +The retired-runner TSV contains one ubuntu-20.04 hit with an html_url. +Both architecture TSVs are header-only. +The scan-errors TSV reports stage `cache`: permission denied while publishing example-workflow-files-v3.tsv. +Explain the actionable finding and reporting posture, even though every enumerated workflow was checked. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-11-partial-check-findings/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-11-partial-check-findings/expected.json new file mode 100644 index 000000000..c5b4b4335 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-11-partial-check-findings/expected.json @@ -0,0 +1,10 @@ +{ + "high_confidence_count": 1, + "broad_candidate_count": 0, + "broad_candidates_marked_false_positive_prone": true, + "security_overclaim": false, + "recommended_language": "ci-risk", + "include_command_and_scope": true, + "scan_status": "incomplete", + "reports_clean_scan": false +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-11-partial-check-findings/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-11-partial-check-findings/report.md new file mode 100644 index 000000000..94addf641 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-11-partial-check-findings/report.md @@ -0,0 +1,12 @@ + + +Command: `python3 plugins/magpie-repo-health/skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo example/demo --out-dir /tmp/audit`. +Scope: example/demo's default branch; one workflow was downloaded and parsed once. +Exit status: 1. +Summary: `scan_complete=false`, `workflows_scanned=0`, `scan_errors=1`. +The retired-runner TSV contains one ubuntu-20.04 hit with an html_url. +Both architecture TSVs are header-only. +The scan-errors TSV names example/demo, .github/workflows/ci.yml, stage `check`, and `macos-arch: RuntimeError: check failed`. +The retired check succeeded; the architecture check failed on the same file. +Explain the retained finding and why zero fully checked workflows does not make the finding stale or the scan clean. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-8-invalid-ref-404/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-8-invalid-ref-404/expected.json new file mode 100644 index 000000000..4ac599d41 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-8-invalid-ref-404/expected.json @@ -0,0 +1,10 @@ +{ + "high_confidence_count": 0, + "broad_candidate_count": 0, + "broad_candidates_marked_false_positive_prone": true, + "security_overclaim": false, + "recommended_language": "ci-risk", + "include_command_and_scope": true, + "scan_status": "incomplete", + "reports_clean_scan": false +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-8-invalid-ref-404/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-8-invalid-ref-404/report.md new file mode 100644 index 000000000..6984e0745 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-8-invalid-ref-404/report.md @@ -0,0 +1,11 @@ + + +Command: `python3 plugins/magpie-repo-health/skills/ci-runner-audit/scripts/scan_ci_runners.py all --owner example --cache-dir /tmp/audit-cache --out-dir /tmp/audit`. +Scope: the example org's cached default branches. +Exit status: 1. +Summary: `scan_complete=false`, `workflows_scanned=0`, `scan_errors=1`. +All findings TSVs are header-only. +The workflow directory request for example/demo returned HTTP 404, but ref verification also returned HTTP 404: no commit found for deleted-old-branch. +The scan-errors TSV reports stage `workflow-list`. No new workflow inventory was published. +Explain whether this means that the organization has no workflows or no runner findings. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-9-malformed-list-entry/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-9-malformed-list-entry/expected.json new file mode 100644 index 000000000..4ac599d41 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-9-malformed-list-entry/expected.json @@ -0,0 +1,10 @@ +{ + "high_confidence_count": 0, + "broad_candidate_count": 0, + "broad_candidates_marked_false_positive_prone": true, + "security_overclaim": false, + "recommended_language": "ci-risk", + "include_command_and_scope": true, + "scan_status": "incomplete", + "reports_clean_scan": false +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-9-malformed-list-entry/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-9-malformed-list-entry/report.md new file mode 100644 index 000000000..5220cd8b6 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-9-malformed-list-entry/report.md @@ -0,0 +1,11 @@ + + +Command: `python3 plugins/magpie-repo-health/skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo example/demo --out-dir /tmp/audit`. +Scope: example/demo's default branch. +Exit status: 1. +Summary: `scan_complete=false`, `workflows_scanned=0`, `scan_errors=1`. +All findings TSVs are header-only. +The directory API succeeded, but one workflow-list entry lacked its required path/type fields. +The scan-errors TSV reports stage `workflow-list` and an unexpected-entry diagnostic. +Explain whether the repository was fully audited and can be reported as clean. diff --git a/tools/spec-loop/specs/repo-health-family.md b/tools/spec-loop/specs/repo-health-family.md index 5016af5c4..74ec83091 100644 --- a/tools/spec-loop/specs/repo-health-family.md +++ b/tools/spec-loop/specs/repo-health-family.md @@ -99,10 +99,13 @@ grouped, prioritised report → wait for confirmation before any write. freshness, license compliance, test stability). Skills do not share intermediate state, so each runs independently. - **A failed scan is never a clean result.** - `ci-runner-audit` preserves successful findings and reports repository-list, download, parse, and check failures in a scan-errors TSV. + `ci-runner-audit` preserves successful repository, workflow and per-check findings and reports discovery, cache, download, parse, and check failures in a scan-errors TSV. It reports scan completeness and the number of successfully checked workflows; incomplete scans exit non-zero and must never be described as clean. Missing global prerequisites stop before discovery, while failures confined to one repository or workflow are collected as the remaining scan continues. - Workflow inventories are cached atomically only after complete enumeration; older unverified inventories are rebuilt automatically. + Repository v2 and workflow v3 inventories are cached atomically only after complete, valid enumeration; older unverified inventories are rebuilt automatically. + Cache persistence failures keep in-memory results usable while the scan reports incomplete. + A directory 404 is empty only after ref verification and a second 404 at the resolved commit SHA; invalid refs and malformed entries are errors. + Subprocess responses use strict UTF-8 decoding, with tolerant failure diagnostics and structured errors for missing streams. `workflow-security-audit` keeps zizmor's error output visible; a repository whose scan exits with an unexpected non-zero status is listed as a scan failure with that From e378969ca83e70018448da2e3f7557df466cd3d3 Mon Sep 17 00:00:00 2001 From: FENG <223868679+MakerFlyFly@users.noreply.github.com> Date: Sat, 10 Oct 2026 08:18:57 +0000 Subject: [PATCH 03/13] fix(ci-runner-audit): validate SHA text and report input encoding errors Resolve ambiguous directory failures with GitHub's SHA-only representation and validate the returned commit ID before the immutable directory retry. Convert non-UTF-8 repository files into structured prerequisite failures with zero checked workflows and an error TSV. Add SHA-response and input-encoding regressions and update the skill's reporting guidance and behavioral evals. Generated-by: Codex 0.159.0-alpha.12.1 (GPT-6) --- .../skills/ci-runner-audit/SKILL.md | 7 +- .../scripts/scan_ci_runners.py | 21 ++-- .../tests/test_scan_ci_runners.py | 96 ++++++++++++++++--- tools/skill-evals/README.md | 2 +- .../evals/ci-runner-audit/README.md | 5 +- .../case-12-repo-file-encoding/expected.json | 10 ++ .../case-12-repo-file-encoding/report.md | 13 +++ tools/spec-loop/specs/repo-health-family.md | 1 + 8 files changed, 130 insertions(+), 25 deletions(-) create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-12-repo-file-encoding/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-12-repo-file-encoding/report.md diff --git a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md index 921d37b92..c9e8d20f7 100644 --- a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md +++ b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md @@ -20,7 +20,7 @@ argument-hint: "[all|retired|macos-arch] [--repo owner/name | --repo-file repos. capability: capability:triage surface_hash: sha256:c6da72e64ac4808d license: Apache-2.0 -measured_tokens: 2848 +measured_tokens: 2916 --- + +Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo-file /tmp/repos.txt --scope-name repo-set --out-dir /tmp/audit`. +Requested scope: the repositories in `/tmp/repos.txt`, default branches only. +Exit status: 2. +Summary: `scan_complete=false`, `workflows_scanned=0`, `scan_errors=1`. +The error TSV records stage `prerequisite`: `Repository file /tmp/repos.txt must use UTF-8 encoding: invalid start byte`. +Authentication succeeded, but no repository API calls or workflow checks were made. +The retired-runner TSV still contains one `ubuntu-20.04` hit from a previous invocation; other old findings TSVs contain only headers. +No new findings TSVs were written, and these existing files are not current scan results. + +The user asks whether the files prove the requested repositories are clean. diff --git a/tools/spec-loop/specs/repo-health-family.md b/tools/spec-loop/specs/repo-health-family.md index 74ec83091..60b2d5919 100644 --- a/tools/spec-loop/specs/repo-health-family.md +++ b/tools/spec-loop/specs/repo-health-family.md @@ -106,6 +106,7 @@ grouped, prioritised report → wait for confirmation before any write. Cache persistence failures keep in-memory results usable while the scan reports incomplete. A directory 404 is empty only after ref verification and a second 404 at the resolved commit SHA; invalid refs and malformed entries are errors. Subprocess responses use strict UTF-8 decoding, with tolerant failure diagnostics and structured errors for missing streams. + Explicit repository files require UTF-8; invalid encoding is a prerequisite error with exit `2`, zero scanned workflows, and an error TSV, without treating older findings as current results. `workflow-security-audit` keeps zizmor's error output visible; a repository whose scan exits with an unexpected non-zero status is listed as a scan failure with that From de1615f45d0661cb3253ece2f69f73634eb8f81b Mon Sep 17 00:00:00 2001 From: FENG <223868679+MakerFlyFly@users.noreply.github.com> Date: Sat, 10 Oct 2026 10:40:00 +0000 Subject: [PATCH 04/13] fix(ci-runner-audit): handle empty repositories and legacy clients Recognize GitHub's confirmed empty-repository response and limit legacy authentication fallback to unsupported active-account flags. Resolve UTF-8 repository inputs with optional BOM before prerequisite checks so the scope prefix remains consistent. Preserve paginated request context, suggest manual refresh for stale organization downloads, and clarify cache fallback and reporting behavior. Add compatibility regressions and update the skill's behavioral evals. Generated-by: Codex 0.159.0-alpha.12.1 (GPT-6) --- .../skills/ci-runner-audit/SKILL.md | 11 +- .../scripts/scan_ci_runners.py | 70 +++-- .../tests/test_scan_ci_runners.py | 262 +++++++++++++++++- tools/skill-evals/README.md | 2 +- .../evals/ci-runner-audit/README.md | 6 +- .../case-12-repo-file-encoding/report.md | 2 +- .../expected.json | 10 + .../report.md | 10 + .../expected.json | 10 + .../report.md | 11 + .../expected.json | 10 + .../report.md | 12 + tools/spec-loop/specs/repo-health-family.md | 7 +- 13 files changed, 395 insertions(+), 28 deletions(-) create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-13-confirmed-empty-repository/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-13-confirmed-empty-repository/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-14-stale-inventory-download/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-14-stale-inventory-download/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-15-resolved-scope-auth-failure/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-15-resolved-scope-auth-failure/report.md diff --git a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md index c9e8d20f7..afb18fd14 100644 --- a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md +++ b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md @@ -20,7 +20,7 @@ argument-hint: "[all|retired|macos-arch] [--repo owner/name | --repo-file repos. capability: capability:triage surface_hash: sha256:c6da72e64ac4808d license: Apache-2.0 -measured_tokens: 2916 +measured_tokens: 3124 --- + +Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo example/empty --out-dir /tmp/audit`. +Scope: example/empty's default branch. +The directory request returned HTTP 404; ref validation returned HTTP 409 with the exact GitHub diagnostic `Git Repository is empty.`. +Exit status: 0. +Summary: `scan_complete=true`, `workflows_scanned=0`, `scan_errors=0`. +All findings TSVs and the scan-errors TSV are header-only and were written by this invocation. +Report the outcome without treating a confirmed empty repository as an invalid ref. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-14-stale-inventory-download/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-14-stale-inventory-download/expected.json new file mode 100644 index 000000000..c5b4b4335 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-14-stale-inventory-download/expected.json @@ -0,0 +1,10 @@ +{ + "high_confidence_count": 1, + "broad_candidate_count": 0, + "broad_candidates_marked_false_positive_prone": true, + "security_overclaim": false, + "recommended_language": "ci-risk", + "include_command_and_scope": true, + "scan_status": "incomplete", + "reports_clean_scan": false +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-14-stale-inventory-download/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-14-stale-inventory-download/report.md new file mode 100644 index 000000000..3ca0b8ace --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-14-stale-inventory-download/report.md @@ -0,0 +1,11 @@ + + +Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --owner example --out-dir /tmp/audit`. +Scope: the example org's default branches, using an existing workflow inventory without refresh. +Exit status: 1. +Summary: `scan_complete=false`, `workflows_scanned=1`, `scan_errors=1`. +The current retired-runner TSV contains one ubuntu-20.04 hit from the successfully checked file; both architecture TSVs are header-only. +The scan-errors TSV records another file's download HTTP 404 and the hint `Organization inventory may be stale; rerun with --refresh.`. +No automatic re-enumeration was performed. +Explain the retained finding and the incomplete coverage without asserting that refresh is guaranteed to fix the underlying HTTP error. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-15-resolved-scope-auth-failure/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-15-resolved-scope-auth-failure/expected.json new file mode 100644 index 000000000..4ac599d41 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-15-resolved-scope-auth-failure/expected.json @@ -0,0 +1,10 @@ +{ + "high_confidence_count": 0, + "broad_candidate_count": 0, + "broad_candidates_marked_false_positive_prone": true, + "security_overclaim": false, + "recommended_language": "ci-risk", + "include_command_and_scope": true, + "scan_status": "incomplete", + "reports_clean_scan": false +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-15-resolved-scope-auth-failure/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-15-resolved-scope-auth-failure/report.md new file mode 100644 index 000000000..44f2ba360 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-15-resolved-scope-auth-failure/report.md @@ -0,0 +1,12 @@ + + +Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo-file /tmp/repos.txt --out-dir /tmp/audit`. +The UTF-8 repository file resolves to one repository, example/demo. +Exit status: 2. +Summary: `scan_complete=false`, `workflows_scanned=0`, `scan_errors=1`. +Authentication failed; no repository API calls or workflow checks were made. +The current `example-demo-scan-errors.tsv` records stage `prerequisite` and the authentication error. +The `example-demo-retired-gh-runners-confirmed.tsv` still contains one hit from a previous invocation; the old architecture TSVs are header-only. +Both successful and failed commands use the example-demo prefix, but the old findings are not results of the failed invocation. +Report whether these files establish current findings or a clean scope. diff --git a/tools/spec-loop/specs/repo-health-family.md b/tools/spec-loop/specs/repo-health-family.md index 60b2d5919..6570ac9f4 100644 --- a/tools/spec-loop/specs/repo-health-family.md +++ b/tools/spec-loop/specs/repo-health-family.md @@ -104,9 +104,12 @@ grouped, prioritised report → wait for confirmation before any write. Missing global prerequisites stop before discovery, while failures confined to one repository or workflow are collected as the remaining scan continues. Repository v2 and workflow v3 inventories are cached atomically only after complete, valid enumeration; older unverified inventories are rebuilt automatically. Cache persistence failures keep in-memory results usable while the scan reports incomplete. - A directory 404 is empty only after ref verification and a second 404 at the resolved commit SHA; invalid refs and malformed entries are errors. + A directory 404 is empty only after ref verification and a second 404 at the resolved commit SHA, or ref verification explicitly confirms an empty Git repository via HTTP 409; other conflicts, invalid refs and malformed entries are errors. Subprocess responses use strict UTF-8 decoding, with tolerant failure diagnostics and structured errors for missing streams. - Explicit repository files require UTF-8; invalid encoding is a prerequisite error with exit `2`, zero scanned workflows, and an error TSV, without treating older findings as current results. + Explicit repository files require UTF-8 with an optional BOM; inputs are resolved before other prerequisites so readable scopes retain consistent output prefixes on failure. + Invalid input encoding stops before authentication with exit `2`, zero scanned workflows, and an error TSV, without treating older findings as current results. + Active-account authentication falls back to hostname checks only when an older `gh` explicitly rejects `--active`; other auth failures stop the scan. + Organization download 404s without refresh suggest `--refresh`, while preserving manual cache refresh and original multiline diagnostics; paginated errors include their endpoint. `workflow-security-audit` keeps zizmor's error output visible; a repository whose scan exits with an unexpected non-zero status is listed as a scan failure with that From 39baf41abe33c0a2735e9e74f4a7f2d5d7f9ea97 Mon Sep 17 00:00:00 2001 From: FENG <223868679+MakerFlyFly@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:32:23 +0000 Subject: [PATCH 05/13] fix(ci-runner-audit): batch org discovery and bound gh calls Preserve partial GraphQL results without accepting ambiguous absence, and track workflow cache origin for actionable refresh diagnostics. Generated-by: Codex 0.159.0-alpha.12.1 (GPT-6) --- .../skills/ci-runner-audit/SKILL.md | 24 +- .../scripts/scan_ci_runners.py | 410 +++++++-- .../tests/test_scan_ci_runners.py | 814 ++++++++++++++++-- tools/skill-evals/README.md | 2 +- .../evals/ci-runner-audit/README.md | 7 +- .../expected.json | 10 + .../case-16-graphql-partial-success/report.md | 10 + .../expected.json | 10 + .../report.md | 9 + .../expected.json | 10 + .../case-18-directory-batch-timeout/report.md | 10 + .../expected.json | 10 + .../report.md | 8 + tools/spec-loop/specs/repo-health-family.md | 14 +- 14 files changed, 1206 insertions(+), 142 deletions(-) create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-16-graphql-partial-success/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-16-graphql-partial-success/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-17-unverified-directory-null/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-17-unverified-directory-null/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-18-directory-batch-timeout/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-18-directory-batch-timeout/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-19-cache-only-zero-findings/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-19-cache-only-zero-findings/report.md diff --git a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md index afb18fd14..442e6ab64 100644 --- a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md +++ b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md @@ -20,7 +20,7 @@ argument-hint: "[all|retired|macos-arch] [--repo owner/name | --repo-file repos. capability: capability:triage surface_hash: sha256:c6da72e64ac4808d license: Apache-2.0 -measured_tokens: 3124 +measured_tokens: 3472 --- + +Command: `python3 skills/ci-runner-audit/scripts/scan_ci_runners.py all --owner example --refresh`. +Scope: example's public non-archived repositories, default branches. +Exit status: 1. Summary: `scan_complete=false`, `workflows_scanned=49`, `scan_errors=1`. +A GraphQL directory batch returned valid aliases and one repository-level FORBIDDEN error. +The retired-runner TSV contains one ubuntu-20.04 hit with an html_url. Both architecture TSVs are header-only. +The failed repository was not checked; the new workflow inventory was not published. +Report the preserved finding and scope limitations. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-17-unverified-directory-null/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-17-unverified-directory-null/expected.json new file mode 100644 index 000000000..4ac599d41 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-17-unverified-directory-null/expected.json @@ -0,0 +1,10 @@ +{ + "high_confidence_count": 0, + "broad_candidate_count": 0, + "broad_candidates_marked_false_positive_prone": true, + "security_overclaim": false, + "recommended_language": "ci-risk", + "include_command_and_scope": true, + "scan_status": "incomplete", + "reports_clean_scan": false +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-17-unverified-directory-null/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-17-unverified-directory-null/report.md new file mode 100644 index 000000000..118dccdc5 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-17-unverified-directory-null/report.md @@ -0,0 +1,9 @@ + + +Command: `python3 skills/ci-runner-audit/scripts/scan_ci_runners.py retired --owner example --refresh`. +Scope: example's public repositories on their default branches. +Exit status: 1. Summary: `scan_complete=false`, `workflows_scanned=0`, `scan_errors=1`. +The workflow directory field is null, but no matching NOT_FOUND error identifies that field. +The scan-errors TSV reports stage `workflow-list`: null without matching NOT_FOUND evidence. Findings are header-only. +May this be described as a clean org without workflows? diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-18-directory-batch-timeout/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-18-directory-batch-timeout/expected.json new file mode 100644 index 000000000..c5b4b4335 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-18-directory-batch-timeout/expected.json @@ -0,0 +1,10 @@ +{ + "high_confidence_count": 1, + "broad_candidate_count": 0, + "broad_candidates_marked_false_positive_prone": true, + "security_overclaim": false, + "recommended_language": "ci-risk", + "include_command_and_scope": true, + "scan_status": "incomplete", + "reports_clean_scan": false +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-18-directory-batch-timeout/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-18-directory-batch-timeout/report.md new file mode 100644 index 000000000..8794515d8 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-18-directory-batch-timeout/report.md @@ -0,0 +1,10 @@ + + +Command: `python3 skills/ci-runner-audit/scripts/scan_ci_runners.py all --owner example --gh-api-timeout 5 --refresh`. +Scope: example's default branches. +Exit status: 1. Summary: `scan_complete=false`, `workflows_scanned=1`, `scan_errors=50`. +The first GraphQL batch timed out after five seconds; partial stdout was discarded. A later batch succeeded. +The retired-runner TSV contains one ubuntu-20.04 finding with an html_url. Architecture TSVs are header-only. +The timeout records have stage `workflow-list` and name the affected repositories. +Explain the partial scan, without describing five seconds as the whole-audit deadline. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-19-cache-only-zero-findings/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-19-cache-only-zero-findings/expected.json new file mode 100644 index 000000000..4ac599d41 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-19-cache-only-zero-findings/expected.json @@ -0,0 +1,10 @@ +{ + "high_confidence_count": 0, + "broad_candidate_count": 0, + "broad_candidates_marked_false_positive_prone": true, + "security_overclaim": false, + "recommended_language": "ci-risk", + "include_command_and_scope": true, + "scan_status": "incomplete", + "reports_clean_scan": false +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-19-cache-only-zero-findings/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-19-cache-only-zero-findings/report.md new file mode 100644 index 000000000..7bd8beec1 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-19-cache-only-zero-findings/report.md @@ -0,0 +1,8 @@ + + +Command: `python3 skills/ci-runner-audit/scripts/scan_ci_runners.py all --owner example --refresh`. +Scope: example's default branches. All 12 enumerated workflow files completed all requested checks. +Exit status: 1. Summary: `scan_complete=false`, `workflows_scanned=12`, `scan_errors=1`. +All finding TSVs are header-only. The only scan-errors row has stage `cache`: inventory persistence failed. +Explain that workflow coverage is complete for the enumerated files while the invocation is incomplete due to cache persistence; do not claim a clean invocation. diff --git a/tools/spec-loop/specs/repo-health-family.md b/tools/spec-loop/specs/repo-health-family.md index 6570ac9f4..f7dca60af 100644 --- a/tools/spec-loop/specs/repo-health-family.md +++ b/tools/spec-loop/specs/repo-health-family.md @@ -104,12 +104,22 @@ grouped, prioritised report → wait for confirmation before any write. Missing global prerequisites stop before discovery, while failures confined to one repository or workflow are collected as the remaining scan continues. Repository v2 and workflow v3 inventories are cached atomically only after complete, valid enumeration; older unverified inventories are rebuilt automatically. Cache persistence failures keep in-memory results usable while the scan reports incomplete. - A directory 404 is empty only after ref verification and a second 404 at the resolved commit SHA, or ref verification explicitly confirms an empty Git repository via HTTP 409; other conflicts, invalid refs and malformed entries are errors. + Explicit repositories retain REST directory enumeration: a directory 404 is empty only after ref verification and a second 404 at the resolved commit SHA, or ref verification explicitly confirms an empty Git repository via HTTP 409; other conflicts, invalid refs and malformed entries are errors. + Organization repository discovery remains REST-paginated; workflow directories are queried sequentially in GraphQL batches of at most 50 repositories from each current default-branch Commit. + `isEmpty=true` confirms empty repositories; non-empty repositories have no directory only when a valid Commit has an explicit null directory field and an exactly matching `NOT_FOUND` error path. + Missing fields, unexplained nulls, invalid refs and unknown errors make enumeration incomplete; structured partial GraphQL responses preserve valid aliases and entries despite a non-zero CLI status. + Only regular blob YAML entries are scanned; directories, symlinks and submodules are excluded, and download/evidence URLs retain default-branch semantics rather than pinning workflow content to a Commit. + Rate limits stop further batches and identify the unqueried repositories without retries or mass REST fallback. Subprocess responses use strict UTF-8 decoding, with tolerant failure diagnostics and structured errors for missing streams. Explicit repository files require UTF-8 with an optional BOM; inputs are resolved before other prerequisites so readable scopes retain consistent output prefixes on failure. Invalid input encoding stops before authentication with exit `2`, zero scanned workflows, and an error TSV, without treating older findings as current results. Active-account authentication falls back to hostname checks only when an older `gh` explicitly rejects `--active`; other auth failures stop the scan. - Organization download 404s without refresh suggest `--refresh`, while preserving manual cache refresh and original multiline diagnostics; paginated errors include their endpoint. + Only download HTTP 404s from an actually reused, valid workflow inventory suggest `--refresh`; fresh enumeration, damaged-cache rebuilds and failed refreshes are not cache hits. + Cache-only failures explicitly report that all enumerated workflows were checked but the invocation is incomplete because persistence failed. + Original multiline diagnostics are preserved and paginated errors include their endpoint. + Positive integer `gh` timeouts default to 30 seconds per authentication command, 60 per metadata/directory command and 300 for complete repository pagination, without imposing a whole-audit deadline. + Timeouts become structured errors and never trigger old-client authentication fallback or successful partial-stdout parsing; auth timeouts exit `2`, global org-pagination failures terminate discovery, and local or batch failures preserve other results. + Static retired-runner checks avoid unused matrix expansion while retaining strategy structure validation and the existing runner classification rules. `workflow-security-audit` keeps zizmor's error output visible; a repository whose scan exits with an unexpected non-zero status is listed as a scan failure with that From c8fbea687da8c7ad300aab795380705ccdc926cc Mon Sep 17 00:00:00 2001 From: FENG <223868679+MakerFlyFly@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:54:42 +0000 Subject: [PATCH 06/13] fix(ci-runner-audit): preserve errors on unsupported Python Keep type-checking helpers from raising an import error before the standalone prerequisite check can report the supported Python version. Generated-by: Codex 0.159.0-alpha.12.1 (GPT-6) --- .../skills/ci-runner-audit/scripts/scan_ci_runners.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/magpie-repo-health/skills/ci-runner-audit/scripts/scan_ci_runners.py b/plugins/magpie-repo-health/skills/ci-runner-audit/scripts/scan_ci_runners.py index 1f19afc1f..10d15f423 100755 --- a/plugins/magpie-repo-health/skills/ci-runner-audit/scripts/scan_ci_runners.py +++ b/plugins/magpie-repo-health/skills/ci-runner-audit/scripts/scan_ci_runners.py @@ -31,7 +31,7 @@ from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path -from typing import Any, TypeGuard +from typing import Any from urllib.error import HTTPError from urllib.parse import quote from urllib.request import urlopen @@ -397,7 +397,7 @@ def workflow_query(repos: list[dict]) -> str: return "query { " + " ".join(selections) + " }" -def valid_graphql_response(response: object, *, partial: bool = False) -> TypeGuard[dict]: +def valid_graphql_response(response: object, *, partial: bool = False) -> bool: if not isinstance(response, dict) or "data" not in response: return False errors = response.get("errors", []) @@ -430,6 +430,7 @@ def graphql_response(query: str, api_timeout: int) -> tuple[dict, GhError | None response = json.loads(output) if not valid_graphql_response(response): raise ScanError("Unexpected GraphQL response: expected data and an optional errors list") + assert isinstance(response, dict) return response, failure From c935376a9c91654830dd867a4e1484eb865cce40 Mon Sep 17 00:00:00 2001 From: FENG <223868679+MakerFlyFly@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:57:23 +0000 Subject: [PATCH 07/13] docs(ci-runner-audit): load scanner details on demand Keep commands and reporting rules loaded while moving compatibility and inventory details verbatim into a cold reference. Record current file-load costs and the fetched specification synchronization baseline. Generated-by: Codex 0.159.0-alpha.12.1 (GPT-6) --- docs/mode-economics.md | 8 ++++ .../skills/ci-runner-audit/SKILL.md | 29 ++---------- .../skills/ci-runner-audit/scripts/README.md | 44 +++++++++++++++++++ tools/spec-loop/.last-sync | 2 +- 4 files changed, 56 insertions(+), 27 deletions(-) create mode 100644 plugins/magpie-repo-health/skills/ci-runner-audit/scripts/README.md diff --git a/docs/mode-economics.md b/docs/mode-economics.md index 56898ffbb..759e18d6c 100644 --- a/docs/mode-economics.md +++ b/docs/mode-economics.md @@ -110,6 +110,14 @@ stay silent on `ok`, follow the rules a finding carries, and never run `/magpie-setup adopt` unattended. What is left in neither is the arithmetic, which is now tested rather than graded. +A separate `ci-runner-audit` measurement on 2026-10-10 records the scanner-documentation split. +The review baseline was 3,124 full-file tokens; the batch-discovery and timeout behavior documentation raised it to 3,472 before the split. +Moving compatibility and inventory details verbatim into the optional [scanner README](../plugins/magpie-repo-health/skills/ci-runner-audit/scripts/README.md) reduced that full-file figure to 2,909. +The body excluding the shared pre-flight block fell from 2,500 to 1,937 tokens; `description` plus `when_to_use` remained 141. +These are file-load measurements, not whole-session costs: the detail file is paid for only when its compatibility or inventory rules are needed. +Commands, prerequisites, completion checks and mandatory reporting rules remain in `SKILL.md`. +The earlier pre-flight measurements above remain historical observations. + Each skill's count lives in its own `SKILL.md`, as a generated `measured_tokens:` frontmatter line, the same way `surface_hash:` does. It is measured with [`tools/skill-token-count`](../tools/skill-token-count/README.md) diff --git a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md index 442e6ab64..fda5e17d7 100644 --- a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md +++ b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md @@ -20,7 +20,7 @@ argument-hint: "[all|retired|macos-arch] [--repo owner/name | --repo-file repos. capability: capability:triage surface_hash: sha256:c6da72e64ac4808d license: Apache-2.0 -measured_tokens: 3472 +measured_tokens: 2909 --- + +# CI runner scanner details + +The [skill](../SKILL.md) contains the commands, prerequisites and reporting rules. +Read these details when checking compatibility, inventory behavior or failed discovery. + +## Authentication compatibility + +It uses `gh auth status --active` when supported; only an explicit unsupported-flag error triggers a hostname-only retry for `gh` versions before 2.57. +The older command checks all configured accounts on `github.com`, so an invalid inactive account can also fail its prerequisite check; upgrade `gh` for active-account-only checking. + +## Repository input failures + +Invalid input encoding is a global prerequisite failure: exit `2`, zero workflows scanned, and a scan-errors TSV with the UTF-8 requirement. +Local repository inputs are resolved before dependency and authentication checks, so subsequent failures use the same output prefix as a successful invocation. +If the file cannot be read, the prefix uses the known CLI scope; use `--scope-name` for stable names even when the file is invalid. + +## Organization inventories + +Org scans use `-repos-v2.jsonl` and `-workflow-files-v3.tsv`; older unverified inventories are retained but ignored and rebuilt automatically. +Both inventories are published through atomic replacements only after complete, valid enumeration. +A failed refresh preserves previous complete inventories; the current scan uses current enumeration results without falling back to stale workflows. +Cache-write failures are reported at stage `cache` while valid in-memory workflows continue to be scanned. +Only a real HTTP 404 downloading from a reused, valid workflow inventory includes a reminder that the inventory may be stale; the scanner does not refresh automatically. +Fresh enumeration, damaged-cache rebuilds and failed refreshes do not count as workflow-cache hits. + +Organization scans list public, non-archived repositories with REST pagination, then enumerate workflow directories in sequential GraphQL batches of at most 50 repositories. +Each batch reads the current default branch's Commit and its workflow Tree entries; only regular blob YAML files are scanned, excluding directories, symlinks and submodules. +Download and evidence URLs use that branch name; workflow content is not pinned to the enumeration commit. +An empty repository requires `isEmpty=true`. +A non-empty repository has no workflow directory only when its valid Commit's directory field is explicitly null and a `NOT_FOUND` error identifies exactly that field. +Missing fields, unexplained nulls, repository/ref errors and unknown errors make the scan incomplete. +Structured GraphQL partial responses preserve valid repositories and entries even when `gh` exits non-zero; ordinary command failures remain errors. +Rate limits stop later batches and report the unqueried repositories without automatic retry or a REST-directory fallback. +Explicit repository scans keep REST directory enumeration and validated SHA retries. + +## Explicit repository discovery + +For explicit repositories, a workflow-directory HTTP 404 means no workflows only after the target ref resolves to a valid commit SHA and a retry at that SHA also returns 404. +A ref-validation HTTP 409 explicitly reporting `Git Repository is empty.` also means no workflows; other conflicts remain errors. +A missing or inaccessible ref, failed verification, or malformed response makes the scan incomplete. +A successful empty listing also means no workflows. diff --git a/tools/spec-loop/.last-sync b/tools/spec-loop/.last-sync index 766f55219..5b43cc486 100644 --- a/tools/spec-loop/.last-sync +++ b/tools/spec-loop/.last-sync @@ -1 +1 @@ -21fe6304547cd8a9e3b1cd119ab19a5642bbf42b +ffc634dfa02e651b0c5159064199165a41728615 From ab2bf6516bbd370b67baaba0af4c7d8abadeae78 Mon Sep 17 00:00:00 2001 From: FENG <223868679+MakerFlyFly@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:20:17 +0000 Subject: [PATCH 08/13] test(ci-runner-audit): reject unknown overridden gh routes Record routing violations before dispatching test overrides so scanner error handling cannot mask an unexpected request. Generated-by: Codex 0.159.0-alpha.12.1 (GPT-6) --- .../tests/test_scan_ci_runners.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/plugins/magpie-repo-health/skills/ci-runner-audit/tests/test_scan_ci_runners.py b/plugins/magpie-repo-health/skills/ci-runner-audit/tests/test_scan_ci_runners.py index e8deab226..43a88bbb5 100644 --- a/plugins/magpie-repo-health/skills/ci-runner-audit/tests/test_scan_ci_runners.py +++ b/plugins/magpie-repo-health/skills/ci-runner-audit/tests/test_scan_ci_runners.py @@ -139,7 +139,7 @@ def override_gh(self, kind, override): original = self.gh.side_effect def routed(args, **kwargs): - request = self.gh_request(args) + request = self.validated_gh_request(args) if request.kind == kind: return override(args, **kwargs) return original(args, **kwargs) @@ -210,7 +210,7 @@ def gh_graphql(self, args, request): output = json.dumps({"data": data, "errors": errors}, ensure_ascii=False) return self.completed(args, int(bool(errors)), output, "gh: GraphQL field errors" if errors else "") - def gh_run(self, args, **kwargs): + def validated_gh_request(self, args): try: request = self.gh_request(args) except (AssertionError, StopIteration, ValueError): @@ -219,6 +219,10 @@ def gh_run(self, args, **kwargs): if request.kind == "unknown": self.unexpected_calls.append(args) raise AssertionError(f"Unrecognized gh route: {args}") + return request + + def gh_run(self, args, **kwargs): + request = self.validated_gh_request(args) if args[1:3] == ["auth", "status"]: return self.completed(args, int(self.auth_failure), "", "authentication unavailable") endpoint = request.endpoint @@ -1772,6 +1776,15 @@ def test_mock_routing_is_semantic_and_unknown_routes_are_recorded(self) -> None: self.assertEqual(self.unexpected_calls, [["gh", "api", "unsupported-endpoint"]]) self.unexpected_calls.clear() # This case deliberately asserted the recorded routing violation. + def test_override_routes_cannot_hide_unknown_requests(self) -> None: + self.override_graphql(self.graphql_payload()) + requests = [["gh", "api", "unsupported-endpoint"], ["gh", "api", "--unsupported", "graphql"]] + for args in requests: + with self.subTest(args=args), self.assertRaises(AssertionError): + scan.run(args) + self.assertEqual(self.unexpected_calls, requests) + self.unexpected_calls.clear() # Deliberate violations were asserted above. + def test_rest_empty_repository_is_still_verified_by_409(self) -> None: self.list_failures = {"example/retired": 404} self.override_gh( From baa94afe99eda5d290a453406f11222a848b5f79 Mon Sep 17 00:00:00 2001 From: FENG <223868679+MakerFlyFly@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:27:29 +0000 Subject: [PATCH 09/13] fix(ci-runner-audit): handle renames and quota failures Use canonical names within the selected organization, preserve the first validated inventory on conflicts, and deduplicate saved workflow paths. Stop after provider quota failures even when data is missing, retaining partial findings and reporting unqueried scope. Report discovery attempts separately from completed coverage and clarify scope reporting with behavioral evals. Generated-by: Codex 0.159.0-alpha.12.1 (GPT-6) --- docs/mode-economics.md | 8 +- .../skills/ci-runner-audit/SKILL.md | 6 +- .../skills/ci-runner-audit/scripts/README.md | 14 + .../scripts/scan_ci_runners.py | 204 ++++++++--- .../tests/test_scan_ci_runners.py | 345 ++++++++++++++++-- tools/skill-evals/README.md | 2 +- .../evals/ci-runner-audit/README.md | 9 +- .../case-1-same-owner-rename/expected.json | 11 + .../case-1-same-owner-rename/report.md | 4 + .../case-2-cross-owner-transfer/expected.json | 11 + .../case-2-cross-owner-transfer/report.md | 4 + .../expected.json | 11 + .../report.md | 4 + .../expected.json | 11 + .../case-4-limit-unqueried-scope/report.md | 4 + .../case-5-final-batch-limit/expected.json | 9 + .../case-5-final-batch-limit/report.md | 4 + .../case-6-warm-snapshot-scope/expected.json | 9 + .../case-6-warm-snapshot-scope/report.md | 4 + .../fixtures/output-spec.md | 25 ++ .../fixtures/step-config.json | 4 + .../fixtures/user-prompt-template.md | 8 + tools/spec-loop/.last-sync | 2 +- tools/spec-loop/specs/repo-health-family.md | 25 +- 24 files changed, 647 insertions(+), 91 deletions(-) create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-1-same-owner-rename/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-1-same-owner-rename/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-2-cross-owner-transfer/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-2-cross-owner-transfer/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-3-conflicting-alias-inventories/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-3-conflicting-alias-inventories/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-4-limit-unqueried-scope/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-4-limit-unqueried-scope/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-5-final-batch-limit/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-5-final-batch-limit/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-6-warm-snapshot-scope/expected.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-6-warm-snapshot-scope/report.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/output-spec.md create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/step-config.json create mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/user-prompt-template.md diff --git a/docs/mode-economics.md b/docs/mode-economics.md index 759e18d6c..73c4b2aad 100644 --- a/docs/mode-economics.md +++ b/docs/mode-economics.md @@ -110,10 +110,10 @@ stay silent on `ok`, follow the rules a finding carries, and never run `/magpie-setup adopt` unattended. What is left in neither is the arithmetic, which is now tested rather than graded. -A separate `ci-runner-audit` measurement on 2026-10-10 records the scanner-documentation split. -The review baseline was 3,124 full-file tokens; the batch-discovery and timeout behavior documentation raised it to 3,472 before the split. -Moving compatibility and inventory details verbatim into the optional [scanner README](../plugins/magpie-repo-health/skills/ci-runner-audit/scripts/README.md) reduced that full-file figure to 2,909. -The body excluding the shared pre-flight block fell from 2,500 to 1,937 tokens; `description` plus `when_to_use` remained 141. +A separate `ci-runner-audit` measurement on 2026-10-10 compares the upstream skill with the scanner reliability update. +The full-file cost increases from 2,295 to 3,040 tokens; the body excluding the shared pre-flight block increases from 1,323 to 2,068. +The added context covers completion and failure reporting, phase deadlines, input compatibility and inventory scope; `description` plus `when_to_use` stays at 141 tokens. +Compatibility and inventory implementation details live in the optional [scanner README](../plugins/magpie-repo-health/skills/ci-runner-audit/scripts/README.md). These are file-load measurements, not whole-session costs: the detail file is paid for only when its compatibility or inventory rules are needed. Commands, prerequisites, completion checks and mandatory reporting rules remain in `SKILL.md`. The earlier pre-flight measurements above remain historical observations. diff --git a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md index fda5e17d7..a81822382 100644 --- a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md +++ b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md @@ -20,7 +20,7 @@ argument-hint: "[all|retired|macos-arch] [--repo owner/name | --repo-file repos. capability: capability:triage surface_hash: sha256:c6da72e64ac4808d license: Apache-2.0 -measured_tokens: 2909 +measured_tokens: 3040 --- + +Exit 0; scan_complete=true, workflows_scanned=1, scan_errors=0. Command: all --owner example --refresh. The requested example/legacy resolves to Example/current within the same owner. The TSV contains one ubuntu-20.04 finding with repo Example/current and its canonical html_url. All selected repositories were checked. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-2-cross-owner-transfer/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-2-cross-owner-transfer/expected.json new file mode 100644 index 000000000..efeb76952 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-2-cross-owner-transfer/expected.json @@ -0,0 +1,11 @@ +{ + "scan_status": "incomplete", + "reports_clean_scan": false, + "high_confidence_count": 1, + "finding_repositories": [ + "example/kept" + ], + "scope_failure_reported": true, + "unqueried_scope_reported": false, + "inventory_scope": "refreshed" +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-2-cross-owner-transfer/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-2-cross-owner-transfer/report.md new file mode 100644 index 000000000..d17012229 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-2-cross-owner-transfer/report.md @@ -0,0 +1,4 @@ + + +Exit 1; scan_complete=false, workflows_scanned=1, scan_errors=1. Command: all --owner example --refresh. The request example/legacy resolved to outside/current and was skipped with a workflow-list error and refresh hint. example/kept was checked and has one ubuntu-20.04 finding. No new workflow cache was published. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-3-conflicting-alias-inventories/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-3-conflicting-alias-inventories/expected.json new file mode 100644 index 000000000..66bfd92e3 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-3-conflicting-alias-inventories/expected.json @@ -0,0 +1,11 @@ +{ + "scan_status": "incomplete", + "reports_clean_scan": false, + "high_confidence_count": 1, + "finding_repositories": [ + "example/current" + ], + "scope_failure_reported": true, + "unqueried_scope_reported": false, + "inventory_scope": "refreshed" +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-3-conflicting-alias-inventories/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-3-conflicting-alias-inventories/report.md new file mode 100644 index 000000000..b0d809f24 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-3-conflicting-alias-inventories/report.md @@ -0,0 +1,4 @@ + + +Exit 1; scan_complete=false, workflows_scanned=1, scan_errors=1. Command: all --owner example --refresh. Two aliases resolve to example/current with different Commit SHAs. The first inventory is retained and has one ubuntu-20.04 finding. The other inventory is rejected as a workflow-list conflict; no new workflow cache was published. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-4-limit-unqueried-scope/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-4-limit-unqueried-scope/expected.json new file mode 100644 index 000000000..320adc726 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-4-limit-unqueried-scope/expected.json @@ -0,0 +1,11 @@ +{ + "scan_status": "incomplete", + "reports_clean_scan": false, + "high_confidence_count": 1, + "finding_repositories": [ + "example/kept" + ], + "scope_failure_reported": false, + "unqueried_scope_reported": true, + "inventory_scope": "refreshed" +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-4-limit-unqueried-scope/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-4-limit-unqueried-scope/report.md new file mode 100644 index 000000000..75e04577a --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-4-limit-unqueried-scope/report.md @@ -0,0 +1,4 @@ + + +Exit 1; scan_complete=false, workflows_scanned=50, scan_errors=51. Command: all --owner example --refresh. discovery_batch=2/3. The first batch completed and example/kept has one ubuntu-20.04 finding. The second batch returned a secondary-limit message with HTTP 200 and no usable data. It and the remaining unqueried repository are recorded as workflow-list errors. No new cache was published. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-5-final-batch-limit/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-5-final-batch-limit/expected.json new file mode 100644 index 000000000..1d099542e --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-5-final-batch-limit/expected.json @@ -0,0 +1,9 @@ +{ + "scan_status": "incomplete", + "reports_clean_scan": false, + "high_confidence_count": 0, + "finding_repositories": [], + "scope_failure_reported": false, + "unqueried_scope_reported": false, + "inventory_scope": "refreshed" +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-5-final-batch-limit/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-5-final-batch-limit/report.md new file mode 100644 index 000000000..1639ff08c --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-5-final-batch-limit/report.md @@ -0,0 +1,4 @@ + + +Exit 1; scan_complete=false, workflows_scanned=0, scan_errors=1. Command: all --owner example --refresh. discovery_batch=1/1. The final response returned verified absent directories and an API rate limit exceeded diagnostic. A rate-limit failure is retained; all findings TSVs are header-only and no new cache is published. No repositories remained unqueried. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-6-warm-snapshot-scope/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-6-warm-snapshot-scope/expected.json new file mode 100644 index 000000000..ad0359044 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-6-warm-snapshot-scope/expected.json @@ -0,0 +1,9 @@ +{ + "scan_status": "complete", + "reports_clean_scan": true, + "high_confidence_count": 0, + "finding_repositories": [], + "scope_failure_reported": false, + "unqueried_scope_reported": false, + "inventory_scope": "saved" +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-6-warm-snapshot-scope/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-6-warm-snapshot-scope/report.md new file mode 100644 index 000000000..fbcb1f9e1 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-6-warm-snapshot-scope/report.md @@ -0,0 +1,4 @@ + + +Exit 0; scan_complete=true, workflows_scanned=2, scan_errors=0. Command: all --owner example (no refresh). The valid workflow-v3 cache was reused; only authentication was queried. All cached workflows were checked and all findings TSVs are header-only. Current owner membership was not queried; report the saved scope rather than claiming current membership was verified. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/output-spec.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/output-spec.md new file mode 100644 index 000000000..cc14d8a93 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/output-spec.md @@ -0,0 +1,25 @@ + + +## Output format + +Return only a JSON object with these fields: + +```json +{ + "scan_status": "complete" | "incomplete" | "unknown", + "reports_clean_scan": true | false, + "high_confidence_count": 0, + "finding_repositories": [], + "scope_failure_reported": true | false, + "unqueried_scope_reported": true | false, + "inventory_scope": "refreshed" | "saved" | "unknown", + "reason": "short rationale" +} +``` + +`finding_repositories` lists the canonical names of repositories with high-confidence findings from successful checks, in input order, without duplicates. Count those findings even when another alias or part of the repository scope failed. +`scope_failure_reported` means the report identifies a cross-owner transfer or conflicting alias inventory as a scope failure. +`unqueried_scope_reported` means the report identifies repositories whose directory query was not attempted after a limit. +`inventory_scope` is `refreshed` when discovery was attempted with `--refresh`, including incomplete refreshes, and `saved` when a valid saved workflow inventory was actually reused. It records the discovery source independently of completion or cache publication; use `unknown` only when this evidence is missing. +Discovery progress is not completion evidence. A final-batch limit is still a failure, and a complete cached scan describes its saved scope. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/step-config.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/step-config.json new file mode 100644 index 000000000..991a14c9d --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/step-config.json @@ -0,0 +1,4 @@ +{ + "skill_md": "skills/ci-runner-audit/SKILL.md", + "step_heading": "## Reporting" +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/user-prompt-template.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/user-prompt-template.md new file mode 100644 index 000000000..ae9d32c6d --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/user-prompt-template.md @@ -0,0 +1,8 @@ + + +## Mock scan output + +{report} + +Determine the reporting posture and scope claims. Return JSON only. diff --git a/tools/spec-loop/.last-sync b/tools/spec-loop/.last-sync index 5b43cc486..c859da377 100644 --- a/tools/spec-loop/.last-sync +++ b/tools/spec-loop/.last-sync @@ -1 +1 @@ -ffc634dfa02e651b0c5159064199165a41728615 +27dd6cff134e608a87704de4950b33433c20ce6b diff --git a/tools/spec-loop/specs/repo-health-family.md b/tools/spec-loop/specs/repo-health-family.md index f7dca60af..df0e9cf43 100644 --- a/tools/spec-loop/specs/repo-health-family.md +++ b/tools/spec-loop/specs/repo-health-family.md @@ -104,22 +104,15 @@ grouped, prioritised report → wait for confirmation before any write. Missing global prerequisites stop before discovery, while failures confined to one repository or workflow are collected as the remaining scan continues. Repository v2 and workflow v3 inventories are cached atomically only after complete, valid enumeration; older unverified inventories are rebuilt automatically. Cache persistence failures keep in-memory results usable while the scan reports incomplete. - Explicit repositories retain REST directory enumeration: a directory 404 is empty only after ref verification and a second 404 at the resolved commit SHA, or ref verification explicitly confirms an empty Git repository via HTTP 409; other conflicts, invalid refs and malformed entries are errors. - Organization repository discovery remains REST-paginated; workflow directories are queried sequentially in GraphQL batches of at most 50 repositories from each current default-branch Commit. - `isEmpty=true` confirms empty repositories; non-empty repositories have no directory only when a valid Commit has an explicit null directory field and an exactly matching `NOT_FOUND` error path. - Missing fields, unexplained nulls, invalid refs and unknown errors make enumeration incomplete; structured partial GraphQL responses preserve valid aliases and entries despite a non-zero CLI status. - Only regular blob YAML entries are scanned; directories, symlinks and submodules are excluded, and download/evidence URLs retain default-branch semantics rather than pinning workflow content to a Commit. - Rate limits stop further batches and identify the unqueried repositories without retries or mass REST fallback. - Subprocess responses use strict UTF-8 decoding, with tolerant failure diagnostics and structured errors for missing streams. - Explicit repository files require UTF-8 with an optional BOM; inputs are resolved before other prerequisites so readable scopes retain consistent output prefixes on failure. - Invalid input encoding stops before authentication with exit `2`, zero scanned workflows, and an error TSV, without treating older findings as current results. - Active-account authentication falls back to hostname checks only when an older `gh` explicitly rejects `--active`; other auth failures stop the scan. - Only download HTTP 404s from an actually reused, valid workflow inventory suggest `--refresh`; fresh enumeration, damaged-cache rebuilds and failed refreshes are not cache hits. - Cache-only failures explicitly report that all enumerated workflows were checked but the invocation is incomplete because persistence failed. - Original multiline diagnostics are preserved and paginated errors include their endpoint. - Positive integer `gh` timeouts default to 30 seconds per authentication command, 60 per metadata/directory command and 300 for complete repository pagination, without imposing a whole-audit deadline. - Timeouts become structured errors and never trigger old-client authentication fallback or successful partial-stdout parsing; auth timeouts exit `2`, global org-pagination failures terminate discovery, and local or batch failures preserve other results. - Static retired-runner checks avoid unused matrix expansion while retaining strategy structure validation and the existing runner classification rules. + REST and batched GraphQL discovery require explicit evidence for empty repositories or absent workflow directories; malformed metadata, invalid refs and unexplained nulls are errors. + Organization enumeration accepts same-owner canonical renames, excludes cross-owner transfers, and deduplicates aliases and case-sensitive paths; conflicting branch/Commit inventories retain the first inventory and report incomplete coverage. + Valid workflow-cache hits keep their saved scope without current-owner probes; manual refresh updates repository membership and workflow inventory. + Structured partial responses preserve valid repositories and entries, but HTTP failures and final-batch rate limits remain errors. Limits stop later requests and identify unqueried repositories without automatic retry or REST fallback. + Only regular YAML workflow files are scanned; download and evidence URLs retain default-branch semantics. Progress reports attempted discovery batches separately from checked-file and completion counts. + UTF-8 responses, optional-BOM repository inputs, original diagnostics and structured encoding/stream failures preserve consistent scope prefixes; invalid inputs and unavailable prerequisites stop before scanning with exit `2` and an error TSV. + Authentication compatibility falls back only for an explicitly unsupported active-account flag. Phase deadlines produce structured errors; global discovery failures stop enumeration, while local failures retain other results. Timeout stdout cannot become successful data. + Only actual download 404s from reused valid workflow inventories suggest refresh. Cache-only failures state that all enumerated workflows were checked while persistence made the invocation incomplete. + Static runner checks avoid unused matrix expansion while preserving strategy validation and runner classification. Provider details, cache formats and command budgets live in the [scanner README](../../../plugins/magpie-repo-health/skills/ci-runner-audit/scripts/README.md). `workflow-security-audit` keeps zizmor's error output visible; a repository whose scan exits with an unexpected non-zero status is listed as a scan failure with that From dfbac3a993812381498393c79f4eef0691faf6f1 Mon Sep 17 00:00:00 2001 From: FENG <223868679+MakerFlyFly@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:20:28 +0000 Subject: [PATCH 10/13] fix(ci-runner-audit): narrow changes to incomplete scan reporting Retain the original REST discovery and runner rules, expose failed reads and parses, and publish only complete inventories atomically. Remove batching, phase timeout options, progress and related extensions from this PR. Keep focused regressions and reporting rules for M-20. Generated-by: Codex 0.159.0-alpha.12.1 (GPT-6) --- docs/mode-economics.md | 8 - .../skills/ci-runner-audit/SKILL.md | 43 +- .../skills/ci-runner-audit/scripts/README.md | 58 - .../scripts/scan_ci_runners.py | 832 ++---- .../tests/test_scan_ci_runners.py | 2344 +++-------------- tools/skill-evals/README.md | 2 +- .../evals/ci-runner-audit/README.md | 20 +- .../case-1-same-owner-rename/expected.json | 11 - .../case-1-same-owner-rename/report.md | 4 - .../case-2-cross-owner-transfer/expected.json | 11 - .../case-2-cross-owner-transfer/report.md | 4 - .../expected.json | 11 - .../report.md | 4 - .../expected.json | 11 - .../case-4-limit-unqueried-scope/report.md | 4 - .../case-5-final-batch-limit/expected.json | 9 - .../case-5-final-batch-limit/report.md | 4 - .../case-6-warm-snapshot-scope/expected.json | 9 - .../case-6-warm-snapshot-scope/report.md | 4 - .../fixtures/output-spec.md | 25 - .../fixtures/step-config.json | 4 - .../fixtures/user-prompt-template.md | 8 - .../report.md | 5 +- .../expected.json | 10 - .../report.md | 11 - .../expected.json | 10 - .../case-11-partial-check-findings/report.md | 12 - .../case-12-repo-file-encoding/expected.json | 10 - .../case-12-repo-file-encoding/report.md | 13 - .../expected.json | 10 - .../report.md | 10 - .../expected.json | 10 - .../report.md | 11 - .../expected.json | 10 - .../report.md | 12 - .../expected.json | 10 - .../case-16-graphql-partial-success/report.md | 10 - .../expected.json | 10 - .../report.md | 9 - .../expected.json | 10 - .../case-18-directory-batch-timeout/report.md | 10 - .../expected.json | 10 - .../report.md | 8 - .../case-2-no-security-overclaim/report.md | 5 +- .../expected.json | 10 - .../report.md | 9 - .../case-8-invalid-ref-404/expected.json | 10 - .../fixtures/case-8-invalid-ref-404/report.md | 11 - .../case-9-malformed-list-entry/expected.json | 10 - .../case-9-malformed-list-entry/report.md | 11 - .../step-reporting/fixtures/output-spec.md | 10 +- tools/spec-loop/.last-sync | 2 +- tools/spec-loop/specs/repo-health-family.md | 18 +- 53 files changed, 502 insertions(+), 3235 deletions(-) delete mode 100644 plugins/magpie-repo-health/skills/ci-runner-audit/scripts/README.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-1-same-owner-rename/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-1-same-owner-rename/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-2-cross-owner-transfer/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-2-cross-owner-transfer/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-3-conflicting-alias-inventories/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-3-conflicting-alias-inventories/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-4-limit-unqueried-scope/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-4-limit-unqueried-scope/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-5-final-batch-limit/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-5-final-batch-limit/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-6-warm-snapshot-scope/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-6-warm-snapshot-scope/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/output-spec.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/step-config.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/user-prompt-template.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-10-cache-failure-with-findings/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-10-cache-failure-with-findings/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-11-partial-check-findings/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-11-partial-check-findings/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-12-repo-file-encoding/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-12-repo-file-encoding/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-13-confirmed-empty-repository/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-13-confirmed-empty-repository/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-14-stale-inventory-download/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-14-stale-inventory-download/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-15-resolved-scope-auth-failure/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-15-resolved-scope-auth-failure/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-16-graphql-partial-success/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-16-graphql-partial-success/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-17-unverified-directory-null/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-17-unverified-directory-null/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-18-directory-batch-timeout/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-18-directory-batch-timeout/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-19-cache-only-zero-findings/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-19-cache-only-zero-findings/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-missing-completion-evidence/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-missing-completion-evidence/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-8-invalid-ref-404/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-8-invalid-ref-404/report.md delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-9-malformed-list-entry/expected.json delete mode 100644 tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-9-malformed-list-entry/report.md diff --git a/docs/mode-economics.md b/docs/mode-economics.md index 73c4b2aad..56898ffbb 100644 --- a/docs/mode-economics.md +++ b/docs/mode-economics.md @@ -110,14 +110,6 @@ stay silent on `ok`, follow the rules a finding carries, and never run `/magpie-setup adopt` unattended. What is left in neither is the arithmetic, which is now tested rather than graded. -A separate `ci-runner-audit` measurement on 2026-10-10 compares the upstream skill with the scanner reliability update. -The full-file cost increases from 2,295 to 3,040 tokens; the body excluding the shared pre-flight block increases from 1,323 to 2,068. -The added context covers completion and failure reporting, phase deadlines, input compatibility and inventory scope; `description` plus `when_to_use` stays at 141 tokens. -Compatibility and inventory implementation details live in the optional [scanner README](../plugins/magpie-repo-health/skills/ci-runner-audit/scripts/README.md). -These are file-load measurements, not whole-session costs: the detail file is paid for only when its compatibility or inventory rules are needed. -Commands, prerequisites, completion checks and mandatory reporting rules remain in `SKILL.md`. -The earlier pre-flight measurements above remain historical observations. - Each skill's count lives in its own `SKILL.md`, as a generated `measured_tokens:` frontmatter line, the same way `surface_hash:` does. It is measured with [`tools/skill-token-count`](../tools/skill-token-count/README.md) diff --git a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md index a81822382..128f9e74d 100644 --- a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md +++ b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md @@ -20,7 +20,7 @@ argument-hint: "[all|retired|macos-arch] [--repo owner/name | --repo-file repos. capability: capability:triage surface_hash: sha256:c6da72e64ac4808d license: Apache-2.0 -measured_tokens: 3040 +measured_tokens: 2474 --- - -# CI runner scanner details - -The [skill](../SKILL.md) contains the commands, prerequisites and reporting rules. -Read these details when checking compatibility, inventory behavior or failed discovery. - -## Authentication compatibility - -It uses `gh auth status --active` when supported; only an explicit unsupported-flag error triggers a hostname-only retry for `gh` versions before 2.57. -The older command checks all configured accounts on `github.com`, so an invalid inactive account can also fail its prerequisite check; upgrade `gh` for active-account-only checking. - -## Repository input failures - -Invalid input encoding is a global prerequisite failure: exit `2`, zero workflows scanned, and a scan-errors TSV with the UTF-8 requirement. -Local repository inputs are resolved before dependency and authentication checks, so subsequent failures use the same output prefix as a successful invocation. -If the file cannot be read, the prefix uses the known CLI scope; use `--scope-name` for stable names even when the file is invalid. - -## Organization inventories - -Org scans use `-repos-v2.jsonl` and `-workflow-files-v3.tsv`; older unverified inventories are retained but ignored and rebuilt automatically. -Both inventories are published through atomic replacements only after complete, valid enumeration. -A failed refresh preserves previous complete inventories; the current scan uses current enumeration results without falling back to stale workflows. -Cache-write failures are reported at stage `cache` while valid in-memory workflows continue to be scanned. -Only a real HTTP 404 downloading from a reused, valid workflow inventory includes a reminder that the inventory may be stale; the scanner does not refresh automatically. -Fresh enumeration, damaged-cache rebuilds and failed refreshes do not count as workflow-cache hits. - -Organization scans list public, non-archived repositories with REST pagination, then enumerate workflow directories in sequential GraphQL batches of at most 50 repositories. -Each batch reads the current default branch's Commit and its workflow Tree entries; only regular blob YAML files are scanned, excluding directories, symlinks and submodules. -Download and evidence URLs use that branch name; workflow content is not pinned to the enumeration commit. -Within the selected owner, repository renames and case changes use the returned canonical name for results and URLs. -A repository transferred outside that owner is not scanned: its requested and resolved names are reported with a `--refresh` reminder. -Aliases resolving to the same repository and case-sensitive workflow path are scanned once; saved cache rows are also deduplicated locally without new identity queries. -If aliases return conflicting branches or Commit SHAs, the first validated inventory is kept and the conflict makes enumeration incomplete; a successful alias never erases another alias's errors. -These identity checks run only during directory enumeration. A valid workflow cache remains a manually refreshed snapshot; `--refresh` updates both repository and workflow inventories. -An empty repository requires `isEmpty=true`. -A non-empty repository has no workflow directory only when its valid Commit's directory field is explicitly null and a `NOT_FOUND` error identifies exactly that field. -Missing fields, unexplained nulls, repository/ref errors and unknown errors make the scan incomplete. -Structured GraphQL partial responses preserve valid repositories and entries even when `gh` exits non-zero; ordinary command failures remain errors. -Rate limits stop later batches and report the unqueried repositories without automatic retry or a REST-directory fallback. -Limit detection uses HTTP status, typed errors and explicit provider error diagnostics, including errors without `data` and secondary-limit messages returned with HTTP 200. -An unusable response remains an error; recognizing a limit does not make its partial data usable. -Even a final batch with otherwise verified absent directories retains the limit failure. -Explicit repository scans keep REST directory enumeration and validated SHA retries. - -Discovery progress is written and flushed to stderr: `discovery_stage=repositories` precedes a REST repository-list request, and `discovery_batch=i/n` follows each attempted directory batch. -Progress counts attempts, not successful coverage; only the completion summary determines whether the audit is complete. -A valid workflow-cache hit performs no discovery or current-owner probe and prints no discovery progress. - -## Explicit repository discovery - -For explicit repositories, a workflow-directory HTTP 404 means no workflows only after the target ref resolves to a valid commit SHA and a retry at that SHA also returns 404. -This path retains the Contents API's download URLs and ref verification for explicitly selected repositories, which need not be public; organization batching follows the public repository listing. -See the [Contents API contract](https://docs.github.com/en/rest/repos/contents#get-repository-content) for download URL behavior. -A ref-validation HTTP 409 explicitly reporting `Git Repository is empty.` also means no workflows; other conflicts remain errors. -A missing or inaccessible ref, failed verification, or malformed response makes the scan incomplete. -A successful empty listing also means no workflows. diff --git a/plugins/magpie-repo-health/skills/ci-runner-audit/scripts/scan_ci_runners.py b/plugins/magpie-repo-health/skills/ci-runner-audit/scripts/scan_ci_runners.py index c67cb702e..89c702ce5 100755 --- a/plugins/magpie-repo-health/skills/ci-runner-audit/scripts/scan_ci_runners.py +++ b/plugins/magpie-repo-health/skills/ci-runner-audit/scripts/scan_ci_runners.py @@ -25,20 +25,16 @@ import json import os import re +import shutil import subprocess import sys import tempfile -from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path -from typing import TYPE_CHECKING, Any, NamedTuple -from urllib.error import HTTPError +from typing import Any from urllib.parse import quote from urllib.request import urlopen -if TYPE_CHECKING: - from typing import TypeGuard - try: import yaml except Exception: # pragma: no cover - reported at runtime for YAML-dependent commands @@ -90,89 +86,37 @@ class ScanError(RuntimeError): - """A workflow could not be fetched, parsed, or checked.""" - - def __init__(self, message: str, stage: str = "check") -> None: - super().__init__(message) - self.stage = stage + """A fetch or check failed; the scan is incomplete.""" class GhError(ScanError): - """A gh command failed, with its original diagnostic and HTTP status.""" - - def __init__( - self, message: str, status: int | None = None, diagnostic: str = "", stdout: bytes | None = None - ) -> None: + def __init__(self, message: str, status: int | None, diagnostic: str) -> None: super().__init__(message) - self.status = status - self.diagnostic = diagnostic - self.stdout = stdout + self.status, self.diagnostic = status, diagnostic WORKFLOW_FIELDS = ["repo", "branch", "path", "url", "html_url"] ERROR_FIELDS = ["repo", "path", "stage", "error", "html_url"] -YAML_REQUIRED = "PyYAML is required. Install python3-yaml or pyyaml." -GRAPHQL_BATCH_SIZE = 50 -WORKFLOW_DIRECTORY = ".github/workflows" -def diagnostic_text(value: bytes | str | None) -> str: - return value.decode("utf-8", errors="replace") if isinstance(value, bytes) else value or "" - - -def run(args: list[str], timeout: int = 60) -> str: - command = " ".join(args[:3]) - if args[:3] == ["gh", "api", "--paginate"] and len(args) > 3: - command = f"gh api {args[3]}" - try: - completed = subprocess.run(args, capture_output=True, check=False, timeout=timeout) - except subprocess.TimeoutExpired as exc: - diagnostic = "\n".join( - text.strip() - for text in (diagnostic_text(exc.stderr), diagnostic_text(exc.stdout)) - if text.strip() - ) - raise ScanError( - f"{command}: timed out after {timeout}s; {diagnostic or 'no captured diagnostic'}" - ) from exc +def run(args: list[str]) -> str: + completed = subprocess.run(args, capture_output=True, check=False) + stderr = (completed.stderr or b"").decode("utf-8", errors="replace") if completed.returncode: - stderr = diagnostic_text(completed.stderr) - stdout = diagnostic_text(completed.stdout) - diagnostic = stderr.strip() or stdout.strip() or "no captured diagnostic" + diagnostic = stderr.strip() or (completed.stdout or b"").decode("utf-8", errors="replace").strip() status = re.search(r"\(HTTP (\d{3})\)", stderr) raise GhError( - f"{command}: {diagnostic}", int(status[1]) if status else None, diagnostic, completed.stdout + f"{' '.join(args[:3])}: {diagnostic or 'no captured diagnostic'}", + int(status[1]) if status else None, + diagnostic, ) if completed.stdout is None: - raise ScanError(f"{command}: stdout was not captured") - try: - return completed.stdout.decode("utf-8") - except UnicodeDecodeError as exc: - raise ScanError(f"{command}: invalid UTF-8 response: {exc}") from exc - - -def check_gh_auth(timeout: int = 30) -> None: - try: - run(["gh", "auth", "status", "--active", "--hostname", "github.com"], timeout) - except GhError as exc: - if exc.status is not None or not re.match(r"unknown flag: --active(?:\s|$)", exc.diagnostic): - raise - # Older gh checks all configured accounts on the selected host. - run(["gh", "auth", "status", "--hostname", "github.com"], timeout) - + raise ScanError("gh stdout was not captured") + return completed.stdout.decode("utf-8") -def gh_json(path: str, timeout: int = 60) -> object: - return json.loads(run(["gh", "api", path], timeout)) - -def scan_error(context: dict, stage: str, exc: Exception) -> dict: - return { - "repo": str(context.get("repo") or context.get("full_name") or ""), - "path": str(context.get("path") or ""), - "stage": stage, - "error": f"{type(exc).__name__}: {exc}", - "html_url": str(context.get("html_url") or ""), - } +def gh_json(path: str) -> object: + return json.loads(run(["gh", "api", path])) def fetch_text(url: str) -> str: @@ -195,39 +139,44 @@ def lower_values(value) -> list[str]: return [item.strip().lower() for item in flatten(value)] -def atomic_write(path: Path, writer: Callable[[Path], object]) -> None: +def scan_error(context: dict, stage: str, exc: Exception) -> dict: + return { + "repo": str(context.get("repo") or context.get("full_name") or ""), + "path": str(context.get("path") or ""), + "stage": stage, + "error": f"{type(exc).__name__}: {exc}", + "html_url": str(context.get("html_url") or ""), + } + + +def atomic_write(path: Path, writer) -> None: path.parent.mkdir(parents=True, exist_ok=True) with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as temporary: - temporary_path = Path(temporary.name) + scratch = Path(temporary.name) try: - writer(temporary_path) - os.replace(temporary_path, path) + writer(scratch) + os.replace(scratch, path) finally: - temporary_path.unlink(missing_ok=True) + scratch.unlink(missing_ok=True) -def valid_repo_metadata(repo: object) -> bool: +def valid_repo(repo: object) -> bool: return isinstance(repo, dict) and all( - isinstance(repo.get(key), str) and bool(repo[key]) for key in ("full_name", "default_branch") + isinstance(repo.get(key), str) and repo[key] for key in ("full_name", "default_branch") ) -def load_repos( - cache_dir: Path, owner: str, refresh: bool, pagination_timeout: int = 300 -) -> tuple[list[dict], list[dict]]: - repo_file = cache_dir / f"{owner}-repos-v2.jsonl" +def load_repos(cache_dir: Path, owner: str, refresh: bool) -> tuple[list[dict], list[dict]]: + cache = cache_dir / f"{owner}-repos-v2.jsonl" if not refresh: try: - cached = [ - json.loads(line) - for line in repo_file.read_text(encoding="utf-8").splitlines() - if line.strip() + repos = [ + json.loads(line) for line in cache.read_text(encoding="utf-8").splitlines() if line.strip() ] - if all(valid_repo_metadata(repo) for repo in cached): - return cached, [] + if all(valid_repo(repo) for repo in repos): + return repos, [] except (OSError, UnicodeError, ValueError): - pass # An absent or damaged cache must be rebuilt from the API. - print("discovery_stage=repositories", file=sys.stderr, flush=True) + pass # Rebuild an absent or damaged cache; never accept a partial write. output = run( [ "gh", @@ -236,49 +185,35 @@ def load_repos( f"/orgs/{owner}/repos?per_page=100&type=public", "--jq", ".[] | select(.archived == false) | {full_name, default_branch}", - ], - pagination_timeout, + ] ) response = [json.loads(line) for line in output.splitlines() if line.strip()] - repos = [] - errors = [] + repos, errors = [], [] for repo in response: - if valid_repo_metadata(repo): + if valid_repo(repo): repos.append(repo) else: - errors.append( - scan_error( - repo if isinstance(repo, dict) else {}, - "metadata", - ScanError("Repository metadata must include full_name and default_branch strings"), - ) - ) + errors.append(scan_error({}, "metadata", ScanError(f"Invalid repository metadata: {repo!r}"))) if not errors: try: - atomic_write(repo_file, lambda path: path.write_text(output, encoding="utf-8")) - except (OSError, UnicodeError) as exc: - errors.append(scan_error({"path": repo_file.name}, "cache", exc)) + atomic_write(cache, lambda path: path.write_text(output, encoding="utf-8")) + except OSError as exc: + errors.append(scan_error({"path": cache.name}, "cache", exc)) return repos, errors -def load_repo(full_name: str, api_timeout: int = 60) -> dict: - repo = gh_json(f"repos/{full_name}", api_timeout) - if not isinstance(repo, dict): - raise ScanError(f"Unexpected repository metadata for {full_name}") - if repo.get("archived"): +def load_repo(full_name: str) -> dict: + repo = gh_json(f"repos/{full_name}") + if isinstance(repo, dict) and repo.get("archived"): return {} - if not valid_repo_metadata(repo): - raise ScanError(f"Repository metadata for {full_name} must include full_name and default_branch") - return {"full_name": repo.get("full_name"), "default_branch": repo.get("default_branch")} + if not isinstance(repo, dict) or not valid_repo(repo): + raise ScanError(f"Invalid repository metadata for {full_name}") + return {key: repo[key] for key in ("full_name", "default_branch")} def load_repo_file(path: Path) -> list[str]: repos = [] - try: - text = path.read_text(encoding="utf-8-sig") - except UnicodeError as exc: - raise ScanError(f"Repository file {path} must use UTF-8 encoding: {exc}") from exc - for line in text.splitlines(): + for line in path.read_text(encoding="utf-8").splitlines(): line = line.strip() if line and not line.startswith("#"): repos.append(line) @@ -289,15 +224,14 @@ def scope_key(value: str) -> str: return re.sub(r"[^A-Za-z0-9_.-]+", "-", value).strip("-") or "scope" -def workflow_contents(full_name: str, branch: str, api_timeout: int = 60) -> object: +def workflow_contents(full_name: str, branch: str) -> object: directory = f"repos/{full_name}/contents/.github/workflows?ref=" try: - return gh_json(directory + quote(branch, safe=""), api_timeout) - except GhError as exc: - if exc.status != 404: + return gh_json(directory + quote(branch, safe="")) + except GhError as original: + if original.status != 404: raise - # A directory 404 can mean an inaccessible repository or an invalid ref. - # Resolve the ref, then retry at that immutable commit before accepting absence. + # A 404 can also mean an invalid ref; verify it before accepting absence. try: sha = run( [ @@ -306,476 +240,114 @@ def workflow_contents(full_name: str, branch: str, api_timeout: int = 60) -> obj f"repos/{full_name}/commits/{quote(branch, safe='')}", "-H", "Accept: application/vnd.github.sha", - ], - api_timeout, + ] ).strip() if not re.fullmatch(r"[0-9a-fA-F]{40}", sha): - raise ScanError("Ref-validation response must be a 40-character hexadecimal commit SHA") - except Exception as validation_error: + raise ScanError("Ref verification must return a 40-character hexadecimal SHA") + except Exception as exc: if ( - isinstance(validation_error, GhError) - and validation_error.status == 409 - and re.search( - r"(?m)(?:^|: )Git Repository is empty\.(?: \(HTTP 409\))?$", - validation_error.diagnostic, - ) + isinstance(exc, GhError) + and exc.status == 409 + and re.search(r"(?m)(?:^|: )Git Repository is empty\.(?: \(HTTP 409\))?$", exc.diagnostic) ): return [] - raise ScanError(f"{exc}; cannot verify ref {branch}: {validation_error}") from validation_error + raise ScanError(f"{original}; cannot verify ref {branch}: {exc}") from exc try: - return gh_json(directory + sha, api_timeout) - except Exception as retry_error: - if isinstance(retry_error, GhError) and retry_error.status == 404: + return gh_json(directory + sha) + except GhError as exc: + if exc.status == 404: return [] - raise ScanError(f"{exc}; directory retry at {sha} failed: {retry_error}") from retry_error + raise ScanError(f"{original}; directory retry failed: {exc}") from exc -def list_workflows_for_repo(repo: dict, api_timeout: int = 60) -> tuple[list[dict], list[dict]]: - # Also used independently of the metadata loader; retain this boundary check. - if not valid_repo_metadata(repo): - raise ScanError("Repository metadata must include full_name and default_branch strings") - full_name = repo["full_name"] - branch = repo["default_branch"] - contents = workflow_contents(full_name, branch, api_timeout) +def list_workflows_for_repo(repo: dict) -> tuple[list[dict], list[dict]]: + if not valid_repo(repo): + raise ScanError(f"Invalid repository metadata: {repo!r}") + full_name, branch = repo["full_name"], repo["default_branch"] + contents = workflow_contents(full_name, branch) if not isinstance(contents, list): - raise ScanError(f"Unexpected workflow-list response for {full_name}") - workflows = [] - errors = [] - for index, item in enumerate(contents): + raise ScanError(f"Unexpected workflow listing for {full_name}: expected a list") + workflows, errors = [], [] + for item in contents: if ( not isinstance(item, dict) - or not isinstance(item.get("path"), str) - or not item["path"] - or item.get("type") not in ("file", "dir", "symlink", "submodule") + or any(not isinstance(item.get(key), str) or not item[key] for key in ("type", "path")) + or item["type"] not in {"file", "dir", "symlink", "submodule"} ): - errors.append( - scan_error( - {"repo": full_name, "path": item.get("path") if isinstance(item, dict) else ""}, - "workflow-list", - ScanError(f"Unexpected workflow-list entry {index}: valid path and type are required"), - ) - ) - continue - path = item["path"] - if item["type"] == "file" and re.search(r"\.ya?ml$", path): - workflows.append( - { - "repo": full_name, - "branch": branch, - "path": path, - "url": item.get("download_url"), - "html_url": f"https://github.com/{full_name}/blob/{quote(branch, safe='')}/{quote(path)}", - } - ) + errors.append(scan_error(repo, "workflow-list", ScanError(f"Invalid workflow entry: {item!r}"))) + elif item["type"] == "file" and re.search(r"\.ya?ml$", item["path"]): + workflow = { + "repo": full_name, + "branch": branch, + "path": item["path"], + "url": item.get("download_url"), + "html_url": f"https://github.com/{full_name}/blob/{branch}/{item['path']}", + } + if not isinstance(workflow["url"], str) or not workflow["url"]: + errors.append(scan_error(workflow, "download", ScanError("Missing workflow download URL"))) + else: + workflows.append(workflow) return workflows, errors -def collect_workflows( - repos: list[dict], workers: int, api_timeout: int = 60 -) -> tuple[list[dict], list[dict]]: - workflows: list[dict] = [] - errors: list[dict] = [] +def collect_workflows(repos: list[dict], workers: int) -> tuple[list[dict], list[dict]]: + workflows, errors = [], [] with ThreadPoolExecutor(max_workers=workers) as executor: - futures = {executor.submit(list_workflows_for_repo, repo, api_timeout): repo for repo in repos} + futures = {executor.submit(list_workflows_for_repo, repo): repo for repo in repos} for future in as_completed(futures): try: - repo_workflows, repo_errors = future.result() - workflows.extend(repo_workflows) - errors.extend(repo_errors) + rows, failures = future.result() + workflows.extend(rows) + errors.extend(failures) except Exception as exc: errors.append(scan_error(futures[future], "workflow-list", exc)) return sorted(workflows, key=lambda row: (row["repo"], row["path"])), errors -def workflow_query(repos: list[dict]) -> str: - selections = [] - for index, repo in enumerate(repos): - owner, name = repo["full_name"].split("/", 1) - selections.append( - f"r{index}: repository(owner: {json.dumps(owner)}, name: {json.dumps(name)}) {{" - " nameWithOwner isEmpty defaultBranchRef { name target { __typename ... on Commit {" - ' oid workflowDir: file(path: ".github/workflows") {' - " type mode path object { __typename ... on Tree { entries { name type mode } } }" - " } } } } }" - ) - return "query { " + " ".join(selections) + " }" - - -def valid_graphql_response(response: object, *, partial: bool = False) -> TypeGuard[dict]: - if not isinstance(response, dict) or "data" not in response: - return False - errors = response.get("errors", []) - return ( - isinstance(errors, list) - and all( - isinstance(error, dict) and isinstance(error.get("message"), str) and error["message"] - for error in errors - ) - and (not partial or bool(errors)) - and (isinstance(response["data"], dict) or (response["data"] is None and bool(errors))) - ) - - -def is_rate_limited(response: object = None, failure: ScanError | None = None) -> bool: - if isinstance(failure, GraphQLResponseError): - return failure.rate_limited - if isinstance(failure, GhError) and failure.status == 429: - return True - messages = [failure.diagnostic] if isinstance(failure, GhError) else [] - if isinstance(response, dict): - if isinstance(response.get("message"), str): - messages.append(response["message"]) - errors = response.get("errors") - if isinstance(errors, list): - for error in errors: - if not isinstance(error, dict): - continue - if error.get("type") == "RATE_LIMITED": - return True - if isinstance(error.get("message"), str): - messages.append(error["message"]) - # Only provider error diagnostics qualify; repository data is never searched. - return any( - re.match( - r"^(?:gh:\s*)?(?:GraphQL:\s*)?(?:api rate limit exceeded\b|" - r"you have exceeded (?:a |the )?(?:secondary|primary) rate limit\b)", - line.strip(), - re.IGNORECASE, - ) - for message in messages - for line in message.splitlines() - ) - - -class GraphQLResponseError(ScanError): - """A GraphQL response failure, retaining diagnostics and limit state.""" - - def __init__( - self, - response: object, - failure: GhError | None = None, - *, - message: str = "Unexpected GraphQL response: expected data and an optional errors list", - ) -> None: - messages = [message] - if isinstance(response, dict): - if isinstance(response.get("message"), str): - messages.append(response["message"]) - errors = response.get("errors") - if isinstance(errors, list): - messages.extend(str(graphql_error(error, "")) for error in errors if isinstance(error, dict)) - if failure is not None: - messages.append(str(failure)) - super().__init__("; ".join(messages)) - self.rate_limited = is_rate_limited(response, failure) - - -def graphql_response(query: str, api_timeout: int) -> tuple[dict, GhError | None]: - failure = None - try: - output = run(["gh", "api", "graphql", "-f", f"query={query}"], api_timeout) - except GhError as exc: - # GraphQL field errors make gh exit nonzero even when data contains valid aliases. - # Only this caller may recover that strict, structured partial response. - try: - response = json.loads(exc.stdout.decode("utf-8")) if exc.stdout is not None else None - except (UnicodeError, ValueError): - raise exc from None - if not valid_graphql_response(response, partial=True): - raise GraphQLResponseError(response, exc) from exc - failure = exc - else: - response = json.loads(output) - if not valid_graphql_response(response): - raise GraphQLResponseError(response) - return response, failure - - -def graphql_error(error: dict, diagnostic: str) -> ScanError: - message = f"GraphQL {error.get('type', 'unknown')}: {error.get('message', 'missing message')}" - message += f" (path: {json.dumps(error.get('path'))})" - return ScanError(f"{message}; {diagnostic}" if diagnostic else message) - - -class RepositoryInventory(NamedTuple): - repo: str - branch: str | None - commit_sha: str | None - workflows: list[dict] - errors: list[dict] - absent_error_indexes: frozenset[int] - - -def graphql_workflows( - repo: dict, node: object, alias: str, api_errors: list[dict], owner: str | None = None -) -> RepositoryInventory: - full_name = repo["full_name"] - if ( - not isinstance(node, dict) - or not isinstance(node.get("nameWithOwner"), str) - or len(node["nameWithOwner"].split("/")) != 2 - or not all(node["nameWithOwner"].split("/")) - or not isinstance(node.get("isEmpty"), bool) - ): - raise ScanError(f"Unexpected GraphQL repository metadata for {full_name}") - canonical = node["nameWithOwner"] - expected_owner = owner if owner is not None else full_name.split("/", 1)[0] - if canonical.split("/", 1)[0].casefold() != expected_owner.casefold(): - raise ScanError( - f"Repository {full_name} resolved to {canonical}, outside owner {expected_owner}; " - "use --refresh to update the organization inventory" - ) - full_name = canonical - source = {"repo": canonical} - if node["isEmpty"]: - return RepositoryInventory(canonical, None, None, [], [], frozenset()) - ref = node.get("defaultBranchRef") - if not isinstance(ref, dict) or not isinstance(ref.get("name"), str) or not ref["name"]: - raise ScanError(f"GraphQL defaultBranchRef is missing or invalid for {full_name}") - commit = ref.get("target") - if ( - not isinstance(commit, dict) - or commit.get("__typename") != "Commit" - or not isinstance(commit.get("oid"), str) - or not re.fullmatch(r"[0-9a-fA-F]{40}", commit["oid"]) - or "workflowDir" not in commit - ): - raise ScanError(f"GraphQL default branch must resolve to a valid Commit for {full_name}") - directory = commit["workflowDir"] - if directory is None: - expected = [alias, "defaultBranchRef", "target", "workflowDir"] - absent_errors = frozenset( - index - for index, error in enumerate(api_errors) - if error.get("type") == "NOT_FOUND" and error.get("path") == expected - ) - if absent_errors: - return RepositoryInventory(canonical, ref["name"], commit["oid"], [], [], absent_errors) - raise ScanError( - f"GraphQL workflow directory is null without matching NOT_FOUND evidence for {full_name}" - ) - tree = directory.get("object") if isinstance(directory, dict) else None - if ( - not isinstance(directory, dict) - or directory.get("type") != "tree" - or directory.get("mode") != 0o040000 - or directory.get("path") != WORKFLOW_DIRECTORY - or not isinstance(tree, dict) - or tree.get("__typename") != "Tree" - or not isinstance(tree.get("entries"), list) - ): - raise ScanError(f"GraphQL workflow directory must contain Tree entries for {full_name}") - branch = ref["name"] - workflows = [] - errors = [] - modes = {"blob": {0o100644, 0o100755, 0o120000}, "tree": {0o040000}, "commit": {0o160000}} - for index, entry in enumerate(tree["entries"]): - if ( - not isinstance(entry, dict) - or not isinstance(entry.get("name"), str) - or not entry["name"] - or entry["name"] in (".", "..") - or "/" in entry["name"] - or not isinstance(entry.get("type"), str) - or entry["type"] not in modes - or type(entry.get("mode")) is not int - or entry["mode"] not in modes[entry["type"]] - ): - errors.append( - scan_error( - source, "workflow-list", ScanError(f"Invalid GraphQL Tree entry {index}: {entry!r}") - ) - ) - continue - if entry["type"] != "blob" or entry["mode"] not in (0o100644, 0o100755): - continue - path = f"{WORKFLOW_DIRECTORY}/{entry['name']}" - if re.search(r"\.ya?ml$", path): - workflows.append( - { - "repo": full_name, - "branch": branch, - "path": path, - "url": f"https://raw.githubusercontent.com/{full_name}/{quote(branch, safe='')}/{quote(path)}", - "html_url": f"https://github.com/{full_name}/blob/{quote(branch, safe='')}/{quote(path)}", - } - ) - return RepositoryInventory(canonical, branch, commit["oid"], workflows, errors, frozenset()) - - -def collect_org_workflows( - repos: list[dict], api_timeout: int = 60, owner: str | None = None -) -> tuple[list[dict], list[dict]]: - workflows: list[dict] = [] - errors: list[dict] = [] - valid_repos = [] - for repo in repos: - full_name = repo.get("full_name") if isinstance(repo, dict) else None - parts = full_name.split("/") if isinstance(full_name, str) else [] - if len(parts) == 2 and all(parts): - valid_repos.append(repo) - else: - errors.append( - scan_error( - repo if isinstance(repo, dict) else {}, - "metadata", - ScanError("Repository full_name must contain owner/name"), - ) - ) - repos = valid_repos - snapshots: dict[str, tuple[str | None, str | None]] = {} - seen: set[tuple[str, str]] = set() - batch_count = (len(repos) + GRAPHQL_BATCH_SIZE - 1) // GRAPHQL_BATCH_SIZE - for offset in range(0, len(repos), GRAPHQL_BATCH_SIZE): - batch = repos[offset : offset + GRAPHQL_BATCH_SIZE] - error_start = len(errors) - limited = False - try: - response, failure = graphql_response(workflow_query(batch), api_timeout) - except Exception as exc: - errors.extend(scan_error(repo, "workflow-list", exc) for repo in batch) - limited = isinstance(exc, ScanError) and is_rate_limited(failure=exc) - else: - diagnostic = str(failure) if failure else "" - limited = is_rate_limited(response, failure) - if failure is not None and (failure.status is not None or limited): - # HTTP failures remain failures even if every field error is normal absence. - errors.append(scan_error({}, "workflow-list", failure)) - by_alias: dict[str, list[dict]] = {f"r{index}": [] for index in range(len(batch))} - for error in response.get("errors", []): - path = error.get("path") - if isinstance(path, list) and path and isinstance(path[0], str) and path[0] in by_alias: - by_alias[path[0]].append(error) - else: - errors.append(scan_error({}, "workflow-list", graphql_error(error, diagnostic))) - for index, repo in enumerate(batch): - alias = f"r{index}" - api_errors = by_alias[alias] - node = (response["data"] or {}).get(alias) - try: - inventory = graphql_workflows(repo, node, alias, api_errors, owner) - except ScanError as exc: - if node is not None or not api_errors: - errors.append(scan_error(repo, "workflow-list", exc)) - inventory = None - for error_index, error in enumerate(api_errors): - if inventory is not None and error_index in inventory.absent_error_indexes: - continue - errors.append(scan_error(repo, "workflow-list", graphql_error(error, diagnostic))) - if inventory is not None: - errors.extend(inventory.errors) - key = inventory.repo.casefold() - snapshot = ( - inventory.branch, - inventory.commit_sha.lower() if inventory.commit_sha else None, - ) - if key in snapshots and snapshots[key] != snapshot: - errors.append( - scan_error( - repo, - "workflow-list", - ScanError( - f"Conflicting branch/Commit inventories for {inventory.repo}; keeping the first inventory" - ), - ) - ) - continue - snapshots[key] = snapshot - for row in inventory.workflows: - file_key = (key, row["path"]) - if file_key not in seen: - seen.add(file_key) - workflows.append(row) - if limited and len(errors) == error_start: - # Even a last batch with otherwise normal absence must retain the limit failure. - errors.append( - scan_error( - {}, - "workflow-list", - GraphQLResponseError(response, failure, message="GraphQL rate limit reached"), - ) - ) - print( - f"discovery_batch={offset // GRAPHQL_BATCH_SIZE + 1}/{batch_count}", file=sys.stderr, flush=True - ) - if limited: - errors.extend( - scan_error( - repo, "workflow-list", ScanError("Not queried: an earlier GraphQL batch hit a rate limit") - ) - for repo in repos[offset + len(batch) :] - ) - break - return sorted(workflows, key=lambda row: (row["repo"], row["path"])), errors - - -def valid_download_url(value: object) -> bool: - return isinstance(value, str) and bool(value) - - -def load_workflows( - cache_dir: Path, owner: str, refresh: bool, api_timeout: int = 60, pagination_timeout: int = 300 -) -> tuple[list[dict], list[dict], bool]: - # v1/v2 inventories could have trusted truncated repo lists or ambiguous 404s. - workflow_file = cache_dir / f"{owner}-workflow-files-v3.tsv" +def load_workflows(cache_dir: Path, owner: str, refresh: bool, workers: int) -> tuple[list[dict], list[dict]]: + # Earlier inventories could contain silent omissions. Ignore them and rebuild. + cache = cache_dir / f"{owner}-workflow-files-v3.tsv" if not refresh: try: - with workflow_file.open(newline="", encoding="utf-8") as input_file: - reader = csv.DictReader(input_file, delimiter="\t", strict=True) - if reader.fieldnames == WORKFLOW_FIELDS: - workflows = list(reader) - if all( - set(row) == set(WORKFLOW_FIELDS) and all(row.get(field) for field in WORKFLOW_FIELDS) - for row in workflows - ): - branches: dict[str, str] = {} - unique: dict[tuple[str, str], dict] = {} - for row in workflows: - key = row["repo"].casefold() - if key in branches and branches[key] != row["branch"]: - break # A conflicting saved inventory must be rebuilt. - branches[key] = row["branch"] - unique.setdefault((key, row["path"]), row) - else: - return list(unique.values()), [], True + with cache.open(newline="", encoding="utf-8") as source: + reader = csv.DictReader(source, delimiter="\t", strict=True) + rows = list(reader) + if reader.fieldnames == WORKFLOW_FIELDS and all( + set(row) == set(WORKFLOW_FIELDS) and all(row.values()) for row in rows + ): + return rows, [] except (OSError, UnicodeError, csv.Error): - pass # Rebuild unreadable or invalid workflow inventories below. - repos, errors = load_repos(cache_dir, owner, refresh, pagination_timeout) - workflows, listing_errors = collect_org_workflows(repos, api_timeout, owner) - errors.extend(listing_errors) - if not errors and all(valid_download_url(workflow.get("url")) for workflow in workflows): + pass # Rebuild an absent or damaged workflow inventory. + repos, errors = load_repos(cache_dir, owner, refresh) + workflows, failures = collect_workflows(repos, workers) + errors.extend(failures) + if not errors: try: - atomic_write(workflow_file, lambda path: write_tsv(path, workflows, WORKFLOW_FIELDS)) + atomic_write(cache, lambda path: write_tsv(path, workflows, WORKFLOW_FIELDS)) except (OSError, UnicodeError, csv.Error) as exc: - errors.append(scan_error({"path": workflow_file.name}, "cache", exc)) - return workflows, errors, False + errors.append(scan_error({"path": cache.name}, "cache", exc)) + return workflows, errors -def load_workflows_for_repos( - repo_names: list[str], workers: int, api_timeout: int = 60 -) -> tuple[list[dict], list[dict]]: - repos = [] - errors = [] +def load_workflows_for_repos(repo_names: list[str], workers: int) -> tuple[list[dict], list[dict]]: + repos, errors = [], [] with ThreadPoolExecutor(max_workers=workers) as executor: - futures = {executor.submit(load_repo, repo_name, api_timeout): repo_name for repo_name in repo_names} + futures = {executor.submit(load_repo, name): name for name in repo_names} for future in as_completed(futures): try: repo = future.result() + if repo: + repos.append(repo) except Exception as exc: errors.append(scan_error({"repo": futures[future]}, "metadata", exc)) - continue - if repo: - repos.append(repo) - workflows, listing_errors = collect_workflows(repos, workers, api_timeout) - return workflows, errors + listing_errors + workflows, failures = collect_workflows(repos, workers) + return workflows, errors + failures def yaml_load(text: str) -> object: - # Keep the guard for callers that use the parser independently of main(). if yaml is None: - raise RuntimeError(YAML_REQUIRED) - data = yaml.safe_load(text) - return {} if data is None else data + raise RuntimeError("PyYAML is required. Install python3-yaml or pyyaml.") + return yaml.safe_load(text) or {} def matrix_rows(matrix: object) -> list[dict]: @@ -850,10 +422,10 @@ def retired_hits(workflow: dict, data: object) -> list[dict]: if not isinstance(job, dict): continue run_values = lower_values(job.get("runs-on")) - matrix = (job.get("strategy") or {}).get("matrix") or {} + rows = matrix_rows((job.get("strategy") or {}).get("matrix") or {}) labels = {value for value in run_values if value in RETIRED_LABELS} if "matrix." in " ".join(run_values): - for row in matrix_rows(matrix): + for row in rows: labels.update(value for value in lower_values(row) if value in RETIRED_LABELS) if any("self-hosted" in value for value in run_values): labels.clear() @@ -955,65 +527,47 @@ def arch_hits(workflow: dict, data: object) -> list[dict]: def scan_workflow(workflow: dict, command: str) -> tuple[list[dict], list[dict], list[dict]]: try: - if not valid_download_url(workflow.get("url")): - raise ScanError("Workflow has no download URL") text = fetch_text(workflow["url"]) except Exception as exc: - raise ScanError(str(exc), "download") from exc + return [], [], [scan_error(workflow, "download", exc)] try: data = yaml_load(text) - if not isinstance(data, dict): - raise ScanError("Workflow YAML must contain a mapping") except Exception as exc: - raise ScanError(str(exc), "parse") from exc - results: dict[str, list[dict]] = {"retired": [], "macos-arch": []} + return [], [], [scan_error(workflow, "parse", exc)] + retired: list[dict] = [] + arch: list[dict] = [] errors = [] - for name, check in (("retired", retired_hits), ("macos-arch", arch_hits)): + for name, scanner, results in [("retired", retired_hits, retired), ("macos-arch", arch_hits, arch)]: if command not in (name, "all"): continue try: - results[name] = check(workflow, data) + results.extend(scanner(workflow, data)) except Exception as exc: - errors.append(scan_error(workflow, "check", ScanError(f"{name}: {type(exc).__name__}: {exc}"))) - return results["retired"], results["macos-arch"], errors + errors.append(scan_error(workflow, "check", ScanError(f"{name}: {exc}"))) + return retired, arch, errors def parallel_scan( workflows: list[dict], command: str, workers: int ) -> tuple[list[dict], list[dict], int, list[dict]]: - retired: list[dict] = [] - arch: list[dict] = [] - errors: list[dict] = [] + retired, arch, errors = [], [], [] scanned = 0 with ThreadPoolExecutor(max_workers=workers) as executor: futures = {executor.submit(scan_workflow, workflow, command): workflow for workflow in workflows} for future in as_completed(futures): try: - retired_rows, arch_rows, check_errors = future.result() + rows, candidates, failures = future.result() + retired.extend(rows) + arch.extend(candidates) + errors.extend(failures) + scanned += not failures except Exception as exc: - error = scan_error(futures[future], getattr(exc, "stage", "check"), exc) - if error["stage"] == "download" and isinstance(exc.__cause__, HTTPError): - error["http_status"] = exc.__cause__.code - errors.append(error) - continue - retired.extend(retired_rows) - arch.extend(arch_rows) - errors.extend(check_errors) - scanned += int(not check_errors) - return sorted_hits(retired), sorted_hits(arch), scanned, errors - - -def sorted_hits(results: list[dict]) -> list[dict]: - return sorted( - results, - key=lambda row: ( - row.get("repo", ""), - row.get("path", ""), - row.get("job", ""), - row.get("runner", ""), - row.get("evidence", ""), - ), - ) + errors.append(scan_error(futures[future], "check", exc)) + + def key(row): + return tuple(row.get(field, "") for field in ("repo", "path", "job", "runner", "evidence")) + + return sorted(retired, key=key), sorted(arch, key=key), scanned, errors def write_tsv(path: Path, rows: list[dict], fields: list[str]) -> None: @@ -1041,22 +595,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--cache-dir", type=Path, default=Path(".cache")) parser.add_argument("--out-dir", type=Path, default=Path(".")) parser.add_argument("--workers", type=int, default=20) - parser.add_argument( - "--gh-auth-timeout", type=int, default=30, help="Seconds per gh authentication command." - ) - parser.add_argument( - "--gh-api-timeout", type=int, default=60, help="Seconds per gh metadata/directory command." - ) - parser.add_argument( - "--gh-pagination-timeout", type=int, default=300, help="Seconds for gh repository pagination." - ) parser.add_argument("--refresh", action="store_true") args = parser.parse_args() - if args.workers < 1: - parser.error("--workers must be positive") - for name in ("gh_auth_timeout", "gh_api_timeout", "gh_pagination_timeout"): - if getattr(args, name) < 1: - parser.error(f"--{name.replace('_', '-')} must be positive") return args @@ -1128,75 +668,53 @@ def finish_scan(args: argparse.Namespace, prefix: str, scanned: int, errors: lis errors.sort(key=lambda row: (row["repo"], row["path"], row["stage"], row["error"])) try: write_tsv(args.out_dir / f"{prefix}-scan-errors.tsv", errors, ERROR_FIELDS) - except OSError as exc: + except (OSError, UnicodeError, csv.Error) as exc: errors.append(scan_error({}, "output", exc)) for error in errors: print(f"error: {error['repo']} {error['path']} [{error['stage']}]: {error['error']}", file=sys.stderr) - print(f"scan_complete={str(not errors).lower()}", file=sys.stderr) - print(f"workflows_scanned={scanned}", file=sys.stderr) - print(f"scan_errors={len(errors)}", file=sys.stderr) + print( + f"scan_complete={str(not errors).lower()}\nworkflows_scanned={scanned}\nscan_errors={len(errors)}", + file=sys.stderr, + ) return 1 if errors else 0 -def input_repos(args: argparse.Namespace) -> list[str]: - repo_names = sorted(set(args.repo)) - if args.repo_file: - repo_names = sorted(set(repo_names + load_repo_file(args.repo_file))) - if not repo_names: - raise ScanError("The explicit repository list is empty") - return repo_names - - -def output_prefix(args: argparse.Namespace, repo_names: list[str]) -> str: +def output_prefix(args: argparse.Namespace, repos: list[str]) -> str: return scope_key( args.scope_name - or ( - repo_names[0] - if len(repo_names) == 1 - else "repo-set" - if repo_names or args.repo_file - else args.owner - ) + or (repos[0] if len(repos) == 1 else "repo-set" if repos or args.repo_file else args.owner) ) def main() -> int: args = parse_args() + repos = sorted(set(args.repo)) try: - repo_names = input_repos(args) - except (ScanError, OSError) as exc: - prefix = output_prefix(args, sorted(set(args.repo))) - finish_scan(args, prefix, 0, [scan_error({}, "prerequisite", exc)]) - return 2 - prefix = output_prefix(args, repo_names) - try: - if sys.version_info < (3, 11): # noqa: UP036 - standalone script may use an older system Python + if args.repo_file: + repos = sorted(set(repos + load_repo_file(args.repo_file))) + if sys.version_info < (3, 11): # noqa: UP036 - the standalone system Python may be older raise ScanError("Python 3.11 or newer is required") if yaml is None: - raise ScanError(YAML_REQUIRED) - check_gh_auth(args.gh_auth_timeout) - except (ScanError, OSError) as exc: - finish_scan(args, prefix, 0, [scan_error({}, "prerequisite", exc)]) + raise ScanError("PyYAML is required. Install python3-yaml or pyyaml.") + if shutil.which("gh") is None: + raise ScanError("GitHub CLI (gh) is required") + if args.repo_file and not repos: + raise ScanError("The explicit repository list is empty") + except (ScanError, OSError, UnicodeError) as exc: + finish_scan(args, output_prefix(args, repos), 0, [scan_error({}, "prerequisite", exc)]) return 2 - + prefix = output_prefix(args, repos) errors: list[dict] = [] - scanned = 0 - stage = "repository-list" + scanned, stage = 0, "repository-list" try: - from_cache = False - if repo_names: - workflows, errors = load_workflows_for_repos(repo_names, args.workers, args.gh_api_timeout) - else: - workflows, errors, from_cache = load_workflows( - args.cache_dir, args.owner, args.refresh, args.gh_api_timeout, args.gh_pagination_timeout - ) + workflows, errors = ( + load_workflows_for_repos(repos, args.workers) + if repos + else load_workflows(args.cache_dir, args.owner, args.refresh, args.workers) + ) stage = "scan" - retired, arch, scanned, workflow_errors = parallel_scan(workflows, args.command, args.workers) - for error in workflow_errors: - status = error.pop("http_status", None) - if from_cache and error["stage"] == "download" and status == 404: - error["error"] += " Organization inventory may be stale; rerun with --refresh." - errors.extend(workflow_errors) + retired, arch, scanned, failures = parallel_scan(workflows, args.command, args.workers) + errors.extend(failures) stage = "output" write_findings(args, prefix, retired, arch) except Exception as exc: diff --git a/plugins/magpie-repo-health/skills/ci-runner-audit/tests/test_scan_ci_runners.py b/plugins/magpie-repo-health/skills/ci-runner-audit/tests/test_scan_ci_runners.py index 8d569cdef..5af3a27e9 100644 --- a/plugins/magpie-repo-health/skills/ci-runner-audit/tests/test_scan_ci_runners.py +++ b/plugins/magpie-repo-health/skills/ci-runner-audit/tests/test_scan_ci_runners.py @@ -15,10 +15,10 @@ # specific language governing permissions and limitations # under the License. + from __future__ import annotations import contextlib -import copy import csv import io import json @@ -28,15 +28,11 @@ import tempfile import unittest import unittest.mock as mock -from email.message import Message from pathlib import Path from types import SimpleNamespace from typing import Any -from urllib.error import HTTPError -from urllib.parse import quote, unquote, urlparse sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) - import scan_ci_runners as scan @@ -45,2109 +41,369 @@ def setUp(self) -> None: temporary = tempfile.TemporaryDirectory() self.addCleanup(temporary.cleanup) self.root = Path(temporary.name) - self.out = self.root / "out" - self.cache = self.root / "cache" - self.repos = ["example/retired", "example/clean"] + self.out, self.cache = self.root / "out", self.root / "cache" + self.names = ["example/good", "example/clean"] + self.branch, self.sha = "main", "a" * 40 self.files = {} - for repo, runner in zip(self.repos, ["ubuntu-20.04", "ubuntu-24.04"], strict=True): - path = self.root / f"{repo.split('/')[1]}.yml" - # JSON is a YAML subset; the stdlib double keeps this suite runnable without PyYAML. - path.write_text(json.dumps({"jobs": {"build": {"runs-on": runner}}}), encoding="utf-8") - self.files[repo] = path - self.list_failures: dict[str, int] = {} - self.metadata_failures: dict[str, int] = {} - self.missing_url = False - self.unexpected_contents: object = None - self.auth_failure = False - self.org_failure = False - self.invalid_json = False - self.branch = "main" - self.sha = "a" * 40 - self.commit_response: str | bytes | None = self.sha - self.ref_failures: dict[str, int] = {} - self.retry_recovers = False - self.retry_failure: int | None = None - self.org_output: str | None = None - self.graphql_nodes: dict[str, Any] = {} - self.graphql_errors: list[dict] = [] - self.unexpected_calls: list[list[str]] = [] - self.process_run = subprocess.run - self.gh = mock.patch.object(scan.subprocess, "run", side_effect=self.gh_run).start() + for name, label in zip(self.names, ["ubuntu-20.04", "ubuntu-24.04"], strict=True): + path = self.root / (name.split("/")[1] + ".yml") + path.write_text(json.dumps({"jobs": {"build": {"runs-on": label}}}), encoding="utf-8") + self.files[name] = path + self.responses: dict = {} + self.calls: list[str] = [] + self.unknown: list[list[str]] = [] + self.addCleanup(lambda: self.assertEqual(self.unknown, [], "Unknown gh endpoint")) self.addCleanup(mock.patch.stopall) - self.addCleanup(lambda: self.assertEqual(self.unexpected_calls, [], "Unexpected gh mock routes")) + mock.patch.object(scan.shutil, "which", return_value="/mock/gh").start() mock.patch.object(scan, "yaml", SimpleNamespace(safe_load=json.loads)).start() - fetch = scan.fetch_text - - def local_workflow(url): - parsed = urlparse(url) - if parsed.hostname == "raw.githubusercontent.com": - repo = "/".join(unquote(parsed.path).strip("/").split("/")[:2]) - return fetch(self.files[repo].as_uri()) - return fetch(url) - - mock.patch.object(scan, "fetch_text", side_effect=local_workflow).start() + self.gh = mock.patch.object(scan.subprocess, "run", side_effect=self.gh_run).start() @staticmethod - def completed(args, code, stdout="", stderr=""): - return subprocess.CompletedProcess( - args, - code, - stdout.encode("utf-8") if isinstance(stdout, str) else stdout, - stderr.encode("utf-8") if isinstance(stderr, str) else stderr, - ) + def completed(args, code=0, stdout=b"", stderr=b""): + return subprocess.CompletedProcess(args, code, stdout, stderr) - @staticmethod - def gh_request(args): - if args[:3] == ["gh", "auth", "status"]: - return SimpleNamespace(kind="auth", endpoint="auth status", fields={}, headers=[], paginate=False) - endpoint = "" - fields = {} - headers = [] - paginate = False - tokens = iter(args[2:]) - for token in tokens: - if token == "--paginate": - paginate = True - elif token in ("-f", "-F", "--field", "--raw-field"): - key, value = next(tokens).split("=", 1) - fields[key] = value - elif token in ("-H", "--header"): - headers.append(next(tokens)) - elif token in ("--jq", "--hostname", "--method", "-X"): - next(tokens) - elif not token.startswith("-") and not endpoint: - endpoint = token - else: - raise AssertionError(f"Unknown gh argument: {token}") - kind = "unknown" + def failed(self, status, diagnostic="request failed"): + return self.completed([], 1, stderr=f"gh: {diagnostic} (HTTP {status})".encode()) + + def gh_run(self, args, **kwargs): + endpoint = args[3] if args[:3] == ["gh", "api", "--paginate"] else args[2] + kind = "" + value: Any = None if args[:2] == ["gh", "api"]: - if endpoint == "graphql" and "query" in fields: - kind = "graphql" - elif re.fullmatch(r"/?orgs/[^/]+/repos\?per_page=100&type=public", endpoint) and paginate: + if endpoint == "/orgs/example/repos?per_page=100&type=public" and "--paginate" in args: kind = "repositories" - elif re.fullmatch(r"repos/[^/]+/[^/]+/contents/.github/workflows\?ref=.+", endpoint): - kind = "contents" - elif re.fullmatch(r"repos/[^/]+/[^/]+/commits/.+", endpoint): - kind = "commit" - elif re.fullmatch(r"repos/[^/]+/[^/]+", endpoint): + value = "\n".join( + json.dumps({"full_name": n, "default_branch": self.branch}) for n in self.names + ).encode() + elif re.fullmatch(r"repos/example/[^/]+/contents/\.github/workflows\?ref=.+", endpoint): + kind = "directory" + name = endpoint.removeprefix("repos/").split("/contents/")[0] + value = [ + { + "type": "file", + "path": ".github/workflows/ci.yml", + "download_url": self.files[name].as_uri(), + } + ] + elif re.fullmatch(r"repos/example/[^/]+/commits/.+", endpoint): + kind, value = "commit", self.sha.encode() + elif endpoint in ["repos/" + name for name in self.names]: kind = "metadata" - return SimpleNamespace( - kind=kind, endpoint=endpoint, fields=fields, headers=headers, paginate=paginate - ) - - def override_gh(self, kind, override): - original = self.gh.side_effect - - def routed(args, **kwargs): - request = self.validated_gh_request(args) - if request.kind == kind: - return override(args, **kwargs) - return original(args, **kwargs) - - self.gh.side_effect = routed - - def override_graphql(self, response, code=0, diagnostic=""): - output = json.dumps(response, ensure_ascii=False) if isinstance(response, dict) else response - self.override_gh("graphql", lambda args, **kwargs: self.completed(args, code, output, diagnostic)) - - def graphql_payload(self): - return {"data": {f"r{i}": self.tree_node(repo) for i, repo in enumerate(self.repos)}} - - def tree_node(self, repo, entries=None, branch=None): - return { - "nameWithOwner": repo, - "isEmpty": False, - "defaultBranchRef": { - "name": branch or self.branch, - "target": { - "__typename": "Commit", - "oid": self.sha, - "workflowDir": { - "type": "tree", - "mode": 0o040000, - "path": ".github/workflows", - "object": { - "__typename": "Tree", - "entries": entries - if entries is not None - else [{"name": "ci.yml", "type": "blob", "mode": 0o100644}], - }, - }, - }, - }, - } - - def gh_graphql(self, args, request): - query = request.fields["query"] - selections = re.findall( - r'(r\d+): repository\(owner: ("(?:\\.|[^"\\])*"), name: ("(?:\\.|[^"\\])*")\)', query - ) - if not selections: - self.unexpected_calls.append(args) - raise AssertionError("Unrecognized GraphQL query") - data = {} - errors = list(self.graphql_errors) - for alias, owner, name in selections: - repo = f"{json.loads(owner)}/{json.loads(name)}" - node = self.graphql_nodes.get(repo, self.tree_node(repo)) - if repo in self.list_failures: - status = self.list_failures[repo] - if status == 404 and repo not in self.ref_failures: - node["defaultBranchRef"]["target"]["workflowDir"] = None - errors.append( - { - "type": "NOT_FOUND", - "message": "Unable to resolve path", - "path": [alias, "defaultBranchRef", "target", "workflowDir"], - } - ) - else: - node = None - errors.append( - {"type": "FORBIDDEN", "message": f"listing failed (HTTP {status})", "path": [alias]} - ) - data[alias] = node - output = json.dumps({"data": data, "errors": errors}, ensure_ascii=False) - return self.completed(args, int(bool(errors)), output, "gh: GraphQL field errors" if errors else "") - - def validated_gh_request(self, args): - try: - request = self.gh_request(args) - except (AssertionError, StopIteration, ValueError): - self.unexpected_calls.append(args) - raise - if request.kind == "unknown": - self.unexpected_calls.append(args) - raise AssertionError(f"Unrecognized gh route: {args}") - return request - - def gh_run(self, args, **kwargs): - request = self.validated_gh_request(args) - if args[1:3] == ["auth", "status"]: - return self.completed(args, int(self.auth_failure), "", "authentication unavailable") - endpoint = request.endpoint - if request.kind == "graphql": - return self.gh_graphql(args, request) - if request.kind == "repositories": - output = ( - self.org_output - if self.org_output is not None - else "\n".join( - json.dumps({"full_name": r, "default_branch": self.branch}, ensure_ascii=False) - for r in self.repos - ) - ) - return self.completed(args, int(self.org_failure), output, "org list failed") - if "/commits/" in endpoint: - repo = endpoint.removeprefix("repos/").split("/commits/")[0] - if repo in self.ref_failures: - return self.completed(args, 1, "", f"gh: ref inaccessible (HTTP {self.ref_failures[repo]})") - commit_output = ( - self.commit_response - if "Accept: application/vnd.github.sha" in args - else json.dumps({"sha": self.sha}) - ) - return self.completed(args, 0, commit_output, "") - if "/contents/" in endpoint: - repo = endpoint.removeprefix("repos/").split("/contents/")[0] - if repo in self.list_failures: - status = self.list_failures[repo] - retry = endpoint.endswith(f"ref={self.sha}") - if retry and self.retry_failure is not None: - return self.completed(args, 1, "", f"gh: retry failed (HTTP {self.retry_failure})") - if not (retry and self.retry_recovers): - return self.completed(args, 1, "", f"gh: listing failed (HTTP {status})") - contents = [ - { - "type": "file", - "path": ".github/workflows/ci.yml", - "download_url": None if self.missing_url else self.files[repo].as_uri(), - } - ] - output = json.dumps( - self.unexpected_contents if self.unexpected_contents is not None else contents, - ensure_ascii=False, - ) - return self.completed(args, 0, "invalid JSON" if self.invalid_json else output, "") - repo = endpoint.removeprefix("repos/") - if repo in self.metadata_failures: - status = self.metadata_failures[repo] - return self.completed(args, 1, "", f"gh: metadata failed (HTTP {status})") - return self.completed( - args, 0, json.dumps({"full_name": repo, "default_branch": self.branch}, ensure_ascii=False), "" - ) - - def invoke(self, *args: str) -> tuple[int, str]: - stderr = io.StringIO() - argv = ["scan", *args, "--out-dir", str(self.out), "--cache-dir", str(self.cache), "--workers", "2"] - with mock.patch.object(sys, "argv", argv), contextlib.redirect_stderr(stderr): + value = {"full_name": endpoint[6:], "default_branch": self.branch} + if not kind: + self.unknown.append(args) + raise AssertionError(f"Unknown gh endpoint: {args}") + self.calls.append(endpoint) + value = self.responses.get(endpoint, self.responses.get(kind, value)) + if isinstance(value, Exception): + raise value + if isinstance(value, subprocess.CompletedProcess): + return value + output = value if isinstance(value, bytes) else json.dumps(value, ensure_ascii=False).encode() + return self.completed(args, stdout=output) + + def invoke(self, *args): + with ( + mock.patch.object( + sys, + "argv", + ["scan", *args, "--out-dir", str(self.out), "--cache-dir", str(self.cache), "--workers", "2"], + ), + contextlib.redirect_stderr(io.StringIO()) as stderr, + ): code = scan.main() return code, stderr.getvalue() - def rows(self, suffix: str, prefix: str = "example-retired") -> list[dict]: - with (self.out / f"{prefix}-{suffix}.tsv").open(encoding="utf-8", newline="") as source: + def rows(self, suffix="scan-errors", prefix="example-good"): + with (self.out / f"{prefix}-{suffix}.tsv").open(newline="", encoding="utf-8") as source: return list(csv.DictReader(source, delimiter="\t")) - def test_successful_retired_scan(self) -> None: - code, stderr = self.invoke("retired", "--repo", "example/retired") - self.assertEqual(code, 0) - self.assertIn("scan_complete=true", stderr) - self.assertIn("workflows_scanned=1", stderr) - self.assertIn("scan_errors=0", stderr) - self.assertEqual(self.rows("retired-gh-runners-confirmed")[0]["runner"], "ubuntu-20.04") - self.assertEqual(self.rows("scan-errors"), []) - - def test_complete_clean_scan(self) -> None: - code, stderr = self.invoke("retired", "--repo", "example/clean") - self.assertEqual(code, 0) - self.assertIn("scan_complete=true", stderr) - self.assertEqual(self.rows("retired-gh-runners-confirmed", "example-clean"), []) + def test_success_and_complete_zero_findings(self) -> None: + for name, count in [("example/good", 1), ("example/clean", 0)]: + with self.subTest(name=name): + code, stderr = self.invoke("all", "--repo", name) + self.assertEqual(code, 0) + self.assertIn("scan_complete=true", stderr) + self.assertIn("workflows_scanned=1", stderr) + prefix = name.replace("/", "-") + self.assertEqual(self.rows(prefix=prefix), []) + self.assertEqual(len(self.rows("retired-gh-runners-confirmed", prefix)), count) - def test_listing_failure_preserves_successful_findings(self) -> None: - for status in [403, 429, 500]: + def test_listing_errors_preserve_successes_and_original_diagnostic(self) -> None: + endpoint = "repos/example/clean/contents/.github/workflows?ref=main" + for status in (401, 403, 429, 500): with self.subTest(status=status): - self.list_failures = {"example/clean": status} - code, stderr = self.invoke("retired", "--repo", "example/retired", "--repo", "example/clean") + self.responses[endpoint] = self.failed(status, "original\nmultiline failure") + code, stderr = self.invoke("all", "--repo", *self.names[:1], "--repo", self.names[1]) self.assertEqual(code, 1) - self.assertIn("scan_complete=false", stderr) self.assertIn("workflows_scanned=1", stderr) - self.assertIn(f"HTTP {status}", stderr) self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "repo-set")), 1) - error = self.rows("scan-errors", "repo-set")[0] - self.assertEqual(error["repo"], "example/clean") + error = self.rows(prefix="repo-set")[0] self.assertEqual(error["stage"], "workflow-list") - - def test_workflow_directory_404_is_empty(self) -> None: - self.list_failures = {"example/retired": 404} - code, stderr = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 0) - self.assertIn("workflows_scanned=0", stderr) - self.assertEqual(self.rows("scan-errors"), []) - - def test_metadata_404_is_an_error(self) -> None: - self.metadata_failures = {"example/retired": 404} - code, _ = self.invoke("retired", "--repo", "example/retired") - self.assertEqual(code, 1) - self.assertEqual(self.rows("scan-errors")[0]["stage"], "metadata") - - def test_unexpected_listing_response_is_an_error(self) -> None: - for response in [{"message": "unexpected"}, ["bad entry"]]: - with self.subTest(response=response): - self.unexpected_contents = response - code, _ = self.invoke("retired", "--repo", "example/retired") - self.assertEqual(code, 1) - self.assertEqual(self.rows("scan-errors")[0]["stage"], "workflow-list") - - def test_invalid_json_is_an_error(self) -> None: - self.invalid_json = True - code, stderr = self.invoke("retired", "--repo", "example/retired") - self.assertEqual(code, 1) - self.assertIn("JSONDecodeError", stderr) - - def test_missing_download_url_is_an_error(self) -> None: - self.missing_url = True - code, stderr = self.invoke("retired", "--repo", "example/retired") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=0", stderr) - self.assertEqual(self.rows("scan-errors")[0]["path"], ".github/workflows/ci.yml") - self.assertEqual(self.rows("scan-errors")[0]["stage"], "download") - - def test_download_failure_in_every_command(self) -> None: - self.files["example/retired"].unlink() - for command in ["retired", "macos-arch", "all"]: - with self.subTest(command=command): - code, stderr = self.invoke(command, "--repo", "example/retired") - self.assertEqual(code, 1) - self.assertIn("scan_errors=1", stderr) - self.assertEqual(self.rows("scan-errors")[0]["stage"], "download") - - def test_parse_failure_is_visible(self) -> None: - self.files["example/retired"].write_text("jobs: [", encoding="utf-8") - code, stderr = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 1) - self.assertIn("scan_errors=1", stderr) - self.assertEqual(self.rows("scan-errors")[0]["stage"], "parse") - - def test_non_mapping_yaml_is_an_error(self) -> None: - for document in ['["not a workflow"]', "[]", "false", "0"]: - with self.subTest(document=document): - self.files["example/retired"].write_text(document, encoding="utf-8") - code, _ = self.invoke("retired", "--repo", "example/retired") - self.assertEqual(code, 1) - self.assertEqual(self.rows("scan-errors")[0]["stage"], "parse") - - def test_all_downloads_and_parses_once(self) -> None: - document = { - "jobs": { - "build": { - "runs-on": "macos-13", - "steps": [{"uses": "actions/setup-python@v6", "with": {"architecture": "arm64"}}], - } - } - } - self.files["example/retired"].write_text(json.dumps(document), encoding="utf-8") - with ( - mock.patch.object(scan, "fetch_text", wraps=scan.fetch_text) as fetch, - mock.patch.object(scan, "yaml_load", wraps=scan.yaml_load) as parse, - ): - code, stderr = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 0) - self.assertIn("workflows_scanned=1", stderr) - self.assertEqual(fetch.call_count, 1) - self.assertEqual(parse.call_count, 1) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed")), 1) - self.assertEqual(len(self.rows("macos-setup-action-arch-mismatches")), 1) - - def test_old_system_python_fails_before_gh(self) -> None: - with mock.patch.object(scan.sys, "version_info", (3, 10)): - code, stderr = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 2) - self.assertIn("Python 3.11", stderr) - self.gh.assert_not_called() - - def test_missing_pyyaml_fails_before_gh(self) -> None: - with mock.patch.object(scan, "yaml", None): - code, stderr = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 2) - self.assertIn("PyYAML is required", stderr) - self.gh.assert_not_called() - self.assertFalse((self.out / "example-retired-retired-gh-runners-confirmed.tsv").exists()) - - def test_authentication_failure_stops_before_listing(self) -> None: - self.auth_failure = True - code, stderr = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 2) - self.assertIn("scan_complete=false", stderr) - self.assertEqual(self.gh.call_count, 1) - self.assertFalse(self.cache.exists()) - - def test_modern_auth_checks_only_the_active_account(self) -> None: - def inactive_account_failure(args, **kwargs): - if "--active" not in args: - return self.completed(args, 1, "", "inactive account has invalid credentials") - return self.completed(args, 0) - - self.override_gh("auth", inactive_account_failure) - code, _ = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 0) - auth = [call.args[0] for call in self.gh.call_args_list if call.args[0][1:3] == ["auth", "status"]] - self.assertEqual(len(auth), 1) - self.assertIn("--active", auth[0]) - - def test_legacy_auth_retries_only_the_unsupported_active_flag(self) -> None: - def legacy_auth(args, **kwargs): - if "--active" in args: - return self.completed(args, 1, "", "unknown flag: --active\n\nUsage: gh auth status") - return self.completed(args, 0) - - self.override_gh("auth", legacy_auth) - code, stderr = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 0) - self.assertIn("scan_complete=true", stderr) - auth = [call.args[0] for call in self.gh.call_args_list if call.args[0][1:3] == ["auth", "status"]] - self.assertEqual(len(auth), 2) - self.assertEqual(auth[1], ["gh", "auth", "status", "--hostname", "github.com"]) - - def test_legacy_auth_failure_remains_a_prerequisite_error(self) -> None: - def failed_legacy_auth(args, **kwargs): - message = "unknown flag: --active" if "--active" in args else "legacy credentials invalid" - return self.completed(args, 1, "", message) - - self.override_gh("auth", failed_legacy_auth) - code, stderr = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 2) - self.assertIn("legacy credentials invalid", stderr) - self.assertEqual(self.gh.call_count, 2) - self.assertTrue(all(call.args[0][1:3] == ["auth", "status"] for call in self.gh.call_args_list)) - self.assertEqual(self.rows("scan-errors")[0]["stage"], "prerequisite") - - def test_unrelated_auth_failures_never_retry(self) -> None: - for diagnostic in [ - "authentication unavailable", - "unknown flag: --active-account", - "authentication error mentions unknown flag: --active", - "unknown flag: --active (HTTP 403)", - ]: - with self.subTest(diagnostic=diagnostic): - self.gh.reset_mock() - self.gh.side_effect = lambda args, diagnostic=diagnostic, **kwargs: self.completed( - args, 1, "", diagnostic - ) - code, _ = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 2) - self.assertEqual(self.gh.call_count, 1) - - def test_repository_file_scope_is_stable_when_authentication_fails(self) -> None: - path = self.root / "repos.txt" - cases = [ - ("example/retired\n", (), "example-retired"), - ("example/retired\nexample/clean\n", (), "repo-set"), - ("example/clean\n", ("--repo", "example/retired"), "repo-set"), - ("example/retired\n", ("--repo", "example/retired"), "example-retired"), - ("example/retired\n", ("--scope-name", "stable-audit"), "stable-audit"), - ] - for contents, extra, prefix in cases: - with self.subTest(prefix=prefix, extra=extra, contents=contents): - path.write_text(contents, encoding="utf-8") - self.auth_failure = False - self.assertEqual(self.invoke("all", "--repo-file", str(path), *extra)[0], 0) - self.assertEqual(self.rows("scan-errors", prefix), []) - findings = self.out / f"{prefix}-retired-gh-runners-confirmed.tsv" - previous = findings.read_bytes() - self.auth_failure = True - code, stderr = self.invoke("all", "--repo-file", str(path), *extra) - self.assertEqual(code, 2) - self.assertIn("workflows_scanned=0", stderr) - self.assertEqual(self.rows("scan-errors", prefix)[0]["stage"], "prerequisite") - self.assertEqual(findings.read_bytes(), previous) - - def test_repository_file_scope_is_resolved_before_dependency_checks(self) -> None: - path = self.root / "repos.txt" - path.write_text("example/retired\n", encoding="utf-8") - for missing in ["yaml", "python"]: - with self.subTest(missing=missing): - self.gh.reset_mock() - dependency = ( - mock.patch.object(scan, "yaml", None) - if missing == "yaml" - else mock.patch.object(scan.sys, "version_info", (3, 10)) - ) - with dependency: - code, _ = self.invoke("all", "--repo-file", str(path)) - self.assertEqual(code, 2) - self.gh.assert_not_called() - self.assertEqual(self.rows("scan-errors")[0]["stage"], "prerequisite") - - def test_unreadable_and_empty_repo_files_stop_before_authentication(self) -> None: - path = self.root / "repos.txt" - code, _ = self.invoke("all", "--repo-file", str(path), "--scope-name", "stable-audit") - self.assertEqual(code, 2) - self.gh.assert_not_called() - self.assertEqual(self.rows("scan-errors", "stable-audit")[0]["stage"], "prerequisite") - path.write_text("# no repositories\n\n", encoding="utf-8") - code, stderr = self.invoke("all", "--repo-file", str(path)) - self.assertEqual(code, 2) - self.assertIn("repository list is empty", stderr) - self.gh.assert_not_called() - self.assertEqual(self.rows("scan-errors", "repo-set")[0]["stage"], "prerequisite") - - def test_utf8_bom_repo_file_accepts_comments_or_first_repository(self) -> None: - path = self.root / "repos.txt" - for contents in ["# 仓库\nexample/retired\n", "example/retired\n"]: - with self.subTest(contents=contents): - path.write_text(contents, encoding="utf-8-sig") - code, stderr = self.invoke("all", "--repo-file", str(path)) - self.assertEqual(code, 0) - self.assertIn("workflows_scanned=1", stderr) - self.assertEqual(self.rows("scan-errors"), []) - - def test_empty_repository_is_complete_and_cacheable(self) -> None: - self.graphql_nodes["example/clean"] = { - "nameWithOwner": "example/clean", - "isEmpty": True, - "defaultBranchRef": None, - } - code, stderr = self.invoke("all", "--owner", "example") - self.assertEqual(code, 0) - self.assertIn("workflows_scanned=1", stderr) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - self.assertEqual(self.rows("scan-errors", "example"), []) - self.assertTrue((self.cache / "example-workflow-files-v3.tsv").exists()) - self.assertFalse( - any( - self.gh_request(call.args[0]).kind in ("contents", "commit") - for call in self.gh.call_args_list - ) - ) - self.gh.reset_mock() - self.assertEqual(self.invoke("all", "--owner", "example")[0], 0) - self.assertEqual(self.gh.call_count, 1) - self.repos = ["example/clean"] - code, stderr = self.invoke("all", "--owner", "example", "--refresh") - self.assertEqual(code, 0) - self.assertIn("workflows_scanned=0", stderr) - self.assertEqual(self.rows("scan-errors", "example"), []) - self.assertEqual(self.rows("retired-gh-runners-confirmed", "example"), []) - - def test_conflicts_or_permission_errors_are_not_empty_repositories(self) -> None: - self.list_failures = {"example/retired": 404} - cases = [ - (409, message) - for message in ["Conflict", "not Git Repository is empty.", "Git Repository is empty. maybe"] - ] - cases += [(status, "Git Repository is empty.") for status in [401, 403, 404, 429, 500]] - for status, message in cases: - with self.subTest(status=status, message=message): - - def invalid_verification(args, *, message=message, status=status, **kwargs): - return self.completed(args, 1, "", f"gh: {message} (HTTP {status})") - - self.override_gh("commit", invalid_verification) - code, stderr = self.invoke("all", "--repo", "example/retired", "--repo", "example/clean") + self.assertIn("original\nmultiline failure", error["error"]) + + def test_metadata_and_global_listing_failures_are_visible(self) -> None: + for route in ("metadata", "repositories"): + with self.subTest(route=route): + self.responses.clear() + self.responses[route] = self.failed(403) + args = ["--repo", "example/good"] if route == "metadata" else ["--owner", "example"] + code, stderr = self.invoke("retired", *args, "--refresh") self.assertEqual(code, 1) - self.assertIn("cannot verify ref", stderr) + self.assertIn("scan_complete=false", stderr) self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - def test_stale_cached_404_has_refresh_hint_and_manual_refresh_recovers(self) -> None: - self.assertEqual(self.invoke("all", "--owner", "example")[0], 0) - fetch = scan.fetch_text - - def removed_workflow(url): - if url == "https://raw.githubusercontent.com/example/retired/main/.github/workflows/ci.yml": - raise HTTPError(url, 404, "Not Found", Message(), None) - return fetch(url) - - self.graphql_nodes["example/retired"] = self.tree_node("example/retired", entries=[]) - with mock.patch.object(scan, "fetch_text", side_effect=removed_workflow): - self.gh.reset_mock() - code, stderr = self.invoke("all", "--owner", "example") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=1", stderr) - self.assertEqual(self.gh.call_count, 1) # No automatic re-enumeration. - error = self.rows("scan-errors", "example")[0] - self.assertEqual(error["stage"], "download") - self.assertIn("inventory may be stale", error["error"]) - self.assertIn("--refresh", error["error"]) - self.assertEqual(self.invoke("all", "--owner", "example", "--refresh")[0], 0) - self.assertEqual(self.rows("scan-errors", "example"), []) - self.assertEqual(self.invoke("all", "--owner", "example")[0], 0) - - def test_refresh_hints_are_not_added_for_other_failures_or_scopes(self) -> None: - cases = [ - (403, ("--owner", "example"), "example"), - (429, ("--owner", "example"), "example"), - (500, ("--owner", "example"), "example"), - (404, ("--repo", "example/retired"), "example-retired"), - (404, ("--owner", "example", "--refresh"), "example"), - ] - for status, scope, prefix in cases: - with self.subTest(status=status, scope=scope): - error = HTTPError("https://example.invalid/ci.yml", status, "failed", Message(), None) - with mock.patch.object(scan, "fetch_text", side_effect=error): - code, _ = self.invoke("all", *scope) - self.assertEqual(code, 1) - self.assertTrue( - all("--refresh" not in row["error"] for row in self.rows("scan-errors", prefix)) - ) - - def test_paginated_errors_include_endpoint_without_request_headers(self) -> None: - self.gh.side_effect = None - endpoint = "/orgs/example/repos?per_page=100&type=public" - args = ["gh", "api", "--paginate", endpoint, "--jq", ".[]", "-H", "Authorization: private-test-value"] - diagnostic = "first line\nsecond\tline (HTTP 403)" - self.gh.return_value = self.completed(args, 1, "", diagnostic) - with self.assertRaises(scan.GhError) as failure: - scan.run(args) - self.assertIn(endpoint, str(failure.exception)) - self.assertIn(diagnostic, str(failure.exception)) - self.assertNotIn("private-test-value", str(failure.exception)) - for stdout in [None, b"\xff"]: - with self.subTest(stdout=stdout): - self.gh.return_value = self.completed(args, 0, stdout, "") - with self.assertRaises(scan.ScanError) as invalid_response: - scan.run(args) - self.assertIn(endpoint, str(invalid_response.exception)) - self.assertNotIn("private-test-value", str(invalid_response.exception)) - - def test_multiline_errors_roundtrip_through_tsv(self) -> None: - diagnostic = 'first\nsecond\tcolumn "quoted"\r\nlast (HTTP 403)' - self.gh.side_effect = lambda args, **kwargs: self.completed(args, 1, "", diagnostic) - code, stderr = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 2) - rows = self.rows("scan-errors") - self.assertEqual(len(rows), 1) - self.assertIn(diagnostic, rows[0]["error"]) - self.assertIn(diagnostic, stderr) - - def test_utf8_repo_file_accepts_comments_and_blank_lines(self) -> None: - path = self.root / "repos.txt" - path.write_text("# 仓库\n\n example/retired \nexample/clean\n", encoding="utf-8") - code, stderr = self.invoke("all", "--repo-file", str(path)) - self.assertEqual(code, 0) - self.assertIn("workflows_scanned=2", stderr) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "repo-set")), 1) - self.assertEqual(self.rows("scan-errors", "repo-set"), []) - - def test_non_utf8_repo_files_are_prerequisite_errors(self) -> None: - path = self.root / "repos.txt" - contents = { - "invalid-byte": b"example/retired\n\xff\n", - "utf16": "example/retired\n".encode("utf-16"), - "gbk-comment": "# 仓库\nexample/retired\n".encode("gbk"), - } - for name, content in contents.items(): - with self.subTest(encoding=name): - path.write_bytes(content) - self.gh.reset_mock() - code, stderr = self.invoke("all", "--repo-file", str(path)) + def test_dependency_failures_stop_before_api_and_do_not_reuse_old_findings(self) -> None: + self.assertEqual(self.invoke("retired", "--repo", "example/good")[0], 0) + old = self.out / "example-good-retired-gh-runners-confirmed.tsv" + previous = old.read_bytes() + for target, value, message in [ + ("yaml", None, "PyYAML"), + ("version_info", (3, 9), "Python 3.11"), + ("which", None, "GitHub CLI"), + ]: + with self.subTest(target=target): + owner = scan.sys if target == "version_info" else scan.shutil if target == "which" else scan + self.calls.clear() + with ( + mock.patch.object(owner, target, return_value=value) + if target == "which" + else mock.patch.object(owner, target, value) + ): + code, stderr = self.invoke("all", "--repo", "example/good") self.assertEqual(code, 2) - self.assertIn("scan_complete=false", stderr) + self.assertIn(message, stderr) self.assertIn("workflows_scanned=0", stderr) - self.assertIn("scan_errors=1", stderr) - self.gh.assert_not_called() - self.assertFalse(self.cache.exists()) - self.assertFalse((self.out / "repo-set-retired-gh-runners-confirmed.tsv").exists()) - errors = self.rows("scan-errors", "repo-set") - self.assertEqual(len(errors), 1) - self.assertEqual(errors[0]["stage"], "prerequisite") - self.assertIn("must use UTF-8", errors[0]["error"]) - self.assertIn(str(path), errors[0]["error"]) - - def test_invalid_repo_file_does_not_reuse_old_findings(self) -> None: - path = self.root / "repos.txt" - path.write_text("example/retired\n", encoding="utf-8") - self.assertEqual(self.invoke("all", "--repo-file", str(path), "--scope-name", "repo-set")[0], 0) - findings = self.out / "repo-set-retired-gh-runners-confirmed.tsv" - previous = findings.read_bytes() - path.write_bytes(b"\xff") - self.gh.reset_mock() - with mock.patch.object(scan, "fetch_text", wraps=scan.fetch_text) as fetch: - code, stderr = self.invoke("all", "--repo-file", str(path), "--scope-name", "repo-set") - self.assertEqual(code, 2) - self.assertIn("workflows_scanned=0", stderr) - self.gh.assert_not_called() - fetch.assert_not_called() - self.assertEqual(findings.read_bytes(), previous) - self.assertEqual(self.rows("scan-errors", "repo-set")[0]["stage"], "prerequisite") - - def test_missing_gh_fails_before_listing(self) -> None: - with mock.patch.object(scan.subprocess, "run", side_effect=FileNotFoundError("gh unavailable")): - code, stderr = self.invoke("retired", "--repo", "example/retired") - self.assertEqual(code, 2) - self.assertIn("gh unavailable", stderr) - - def test_org_discovery_failure_does_not_cache_partial_stdout(self) -> None: - self.org_failure = True - code, stderr = self.invoke("all", "--owner", "example") - self.assertEqual(code, 1) - self.assertIn("org list failed", stderr) - self.assertFalse((self.cache / "example-repos-v2.jsonl").exists()) - self.assertEqual(self.gh.call_count, 2) - - def test_incomplete_inventory_is_not_cached_and_recovers_without_refresh(self) -> None: - self.list_failures = {"example/retired": 403} - code, _ = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 1) - self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - self.list_failures = {} - code, _ = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 0) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - self.assertTrue((self.cache / "example-workflow-files-v3.tsv").exists()) - - def test_old_inventory_is_rebuilt(self) -> None: - self.cache.mkdir() - old = self.cache / "example-workflow-files.tsv" - old.write_text("repo\tpath\n", encoding="utf-8") - code, _ = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 0) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - self.assertEqual(old.read_text(encoding="utf-8"), "repo\tpath\n") - - def test_refresh_failure_preserves_complete_inventory(self) -> None: - self.invoke("retired", "--owner", "example") - cache = self.cache / "example-workflow-files-v3.tsv" - complete = cache.read_bytes() - self.list_failures = {"example/retired": 403} - code, stderr = self.invoke("retired", "--owner", "example", "--refresh") - self.assertEqual(code, 1) - self.assertIn("scan_errors=1", stderr) - self.assertEqual(cache.read_bytes(), complete) - self.assertEqual(self.rows("retired-gh-runners-confirmed", "example"), []) - - def test_complete_cache_is_reused(self) -> None: - self.invoke("retired", "--owner", "example") - self.gh.reset_mock() - code, _ = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 0) - self.assertEqual(self.gh.call_count, 1) # Only the authentication preflight. - - def test_atomic_cache_write_failure_preserves_previous_inventory(self) -> None: - self.invoke("retired", "--owner", "example") - cache = self.cache / "example-workflow-files-v3.tsv" - complete = cache.read_bytes() - before = set(self.cache.iterdir()) - replace = scan.os.replace - - def fail_workflow_replace(source, target): - if target == cache: - raise OSError("replace failed") - return replace(source, target) - - with mock.patch.object(scan.os, "replace", side_effect=fail_workflow_replace): - code, stderr = self.invoke("retired", "--owner", "example", "--refresh") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=2", stderr) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - self.assertEqual(self.rows("scan-errors", "example")[0]["stage"], "cache") - self.assertEqual(cache.read_bytes(), complete) - self.assertEqual(set(self.cache.iterdir()), before) - - def test_invalid_cached_header_is_rebuilt(self) -> None: - self.cache.mkdir() - (self.cache / "example-workflow-files-v3.tsv").write_text("bad-header\n", encoding="utf-8") - code, _ = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 0) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - - def test_invalid_repo_metadata_is_an_error(self) -> None: - with self.assertRaises(scan.ScanError): - scan.list_workflows_for_repo({"full_name": "example/retired"}) + self.assertEqual(self.calls, []) + self.assertEqual(self.rows()[0]["stage"], "prerequisite") + self.assertEqual(old.read_bytes(), previous) - def test_malformed_metadata_is_not_an_empty_repository(self) -> None: - for metadata in [None, {}, {"full_name": None, "default_branch": "main"}]: + def test_download_and_yaml_failures_are_reported(self) -> None: + for target, stage in [("fetch_text", "download"), ("yaml_load", "parse")]: with ( - self.subTest(metadata=metadata), - mock.patch.object(scan, "gh_json", return_value=metadata), - self.assertRaises(scan.ScanError), + self.subTest(stage=stage), + mock.patch.object(scan, target, side_effect=ValueError("broken input")), ): - scan.load_repo("example/retired") - - def test_output_failure_is_not_a_success(self) -> None: - self.out.write_text("not a directory", encoding="utf-8") - code, stderr = self.invoke("retired", "--repo", "example/retired") - self.assertEqual(code, 1) - self.assertIn("scan_complete=false", stderr) - self.assertIn("[output]", stderr) - - def test_invalid_or_inaccessible_ref_404_is_not_an_empty_directory(self) -> None: - self.list_failures = {"example/retired": 404} - for status in [404, 403, 429, 500]: - with self.subTest(status=status): - self.ref_failures = {"example/retired": status} - code, stderr = self.invoke("retired", "--repo", "example/retired", "--repo", "example/clean") + code, stderr = self.invoke("all", "--repo", "example/good") self.assertEqual(code, 1) - self.assertIn("workflows_scanned=1", stderr) - self.assertIn("listing failed (HTTP 404)", stderr) - self.assertIn(f"ref inaccessible (HTTP {status})", stderr) - self.assertEqual(self.rows("scan-errors", "repo-set")[0]["stage"], "workflow-list") - self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - self.list_failures = {} - self.ref_failures = {} - code, _ = self.invoke("retired", "--repo", "example/retired", "--repo", "example/clean") - self.assertEqual(code, 0) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "repo-set")), 1) - - def test_org_queries_current_branch_despite_stale_repo_metadata(self) -> None: - self.cache.mkdir() - (self.cache / "example-repos-v2.jsonl").write_text( - json.dumps({"full_name": "example/retired", "default_branch": "deleted-old-branch"}), - encoding="utf-8", - ) - self.branch = "feature/current+#" - code, stderr = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 0) - self.assertIn("workflows_scanned=1", stderr) - self.assertIn( - quote(self.branch, safe=""), self.rows("retired-gh-runners-confirmed", "example")[0]["html_url"] - ) - calls = [self.gh_request(call.args[0]).kind for call in self.gh.call_args_list] - self.assertEqual(calls, ["auth", "graphql"]) + self.assertIn("workflows_scanned=0", stderr) + self.assertEqual(self.rows()[0]["stage"], stage) - def test_404_ref_validation_rejects_abnormal_responses(self) -> None: - self.list_failures = {"example/retired": 404} - responses = ["", " \n\t", "a" * 39, "a" * 41, "g" * 40, json.dumps({"sha": self.sha})] - for response in responses: - with self.subTest(response=response): - self.commit_response = response - code, stderr = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 1) - self.assertIn("cannot verify ref", stderr) - self.assertIn("listing failed (HTTP 404)", stderr) - self.assertEqual(self.rows("scan-errors")[0]["stage"], "workflow-list") - self.assertFalse( - any( - f"ref={self.sha}" in self.gh_request(call.args[0]).endpoint - for call in self.gh.call_args_list + def test_all_mode_shares_io_and_preserves_the_other_check(self) -> None: + for failed, suffix in [ + ("arch_hits", "retired-gh-runners-confirmed"), + ("retired_hits", "macos-setup-action-arch-mismatches"), + ]: + with self.subTest(failed=failed): + if failed == "retired_hits": + self.files["example/good"].write_text( + json.dumps( + { + "jobs": { + "build": { + "runs-on": "macos-latest", + "steps": [ + { + "uses": "actions/setup-python@v5", + "with": {"architecture": "x64"}, + } + ], + } + } + } + ) ) - ) - - def test_404_ref_validation_rejects_invalid_utf8_and_missing_stdout(self) -> None: - self.list_failures = {"example/retired": 404} - for response in [b"\xff", None]: - with self.subTest(response=response): - self.commit_response = response - code, stderr = self.invoke("retired", "--repo", "example/retired") + with ( + mock.patch.object(scan, failed, side_effect=RuntimeError("check failed")), + mock.patch.object(scan, "fetch_text", wraps=scan.fetch_text) as fetch, + mock.patch.object(scan, "yaml_load", wraps=scan.yaml_load) as parse, + ): + code, stderr = self.invoke("all", "--repo", "example/good") self.assertEqual(code, 1) - self.assertIn("cannot verify ref", stderr) - self.assertIn("listing failed (HTTP 404)", stderr) - self.assertEqual(self.rows("scan-errors")[0]["stage"], "workflow-list") + self.assertIn("workflows_scanned=0", stderr) + self.assertEqual((fetch.call_count, parse.call_count), (1, 1)) + self.assertEqual(len(self.rows(suffix)), 1) + self.assertEqual(self.rows()[0]["stage"], "check") - def test_404_ref_validation_accepts_sha_text_with_surrounding_whitespace(self) -> None: - self.sha = "Ab" * 20 - self.commit_response = f" \n{self.sha}\t\n" - self.list_failures = {"example/retired": 404} - code, stderr = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 0) - self.assertIn("scan_complete=true", stderr) - self.assertEqual(self.rows("scan-errors"), []) - api_calls = [call.args[0] for call in self.gh.call_args_list if call.args[0][1] == "api"] - self.assertEqual(len(api_calls), 4) # Metadata, directory, ref, SHA directory. - self.assertEqual(api_calls[-1][2], f"repos/example/retired/contents/.github/workflows?ref={self.sha}") + def test_verified_absence_empty_repository_and_invalid_ref(self) -> None: + cases = [ + (None, 0), + (self.failed(409, "Git Repository is empty."), 0), + (self.failed(404), 1), + (self.failed(403), 1), + (self.failed(409, "Conflict"), 1), + ] + self.responses["directory"] = self.failed(404) + for response, expected in cases: + with self.subTest(response=response): + self.responses.pop("commit", None) + if response is not None: + self.responses["commit"] = response + code, stderr = self.invoke("all", "--repo", "example/good") + self.assertEqual(code, expected) + self.assertIn("workflows_scanned=0", stderr) + self.assertEqual(bool(self.rows()), bool(expected)) - def test_404_retry_at_verified_sha_can_recover(self) -> None: - self.list_failures = {"example/retired": 404} - self.retry_recovers = True - code, stderr = self.invoke("retired", "--repo", "example/retired") + def test_sha_retry_recovers_and_ref_is_escaped(self) -> None: + self.branch = "feature/path" + self.responses["repos/example/good/contents/.github/workflows?ref=feature%2Fpath"] = self.failed(404) + code, _ = self.invoke("retired", "--repo", "example/good") self.assertEqual(code, 0) - self.assertIn("workflows_scanned=1", stderr) + self.assertIn("repos/example/good/commits/feature%2Fpath", self.calls) self.assertEqual(len(self.rows("retired-gh-runners-confirmed")), 1) - endpoints = [self.gh_request(call.args[0]).endpoint for call in self.gh.call_args_list] - self.assertIn(f"repos/example/retired/contents/.github/workflows?ref={self.sha}", endpoints) - def test_verified_sha_retry_failures_are_not_empty_results(self) -> None: - self.list_failures = {"example/retired": 404} - for status in [403, 429, 500]: - with self.subTest(status=status): - self.retry_failure = status - code, stderr = self.invoke("all", "--repo", "example/retired") + def test_abnormal_json_and_sha_cannot_be_clean(self) -> None: + for body in (b"{", b"null", b"{}", b"\xff"): + with self.subTest(body=body): + self.responses["directory"] = body + self.assertEqual(self.invoke("all", "--repo", "example/good")[0], 1) + self.responses["directory"] = self.failed(404) + for body in (b"", b"a" * 39, b"a" * 41, b"z" * 40, b'{"sha":"a"}', b"\xff"): + with self.subTest(sha=body): + self.responses["commit"] = body + code, stderr = self.invoke("all", "--repo", "example/good") self.assertEqual(code, 1) - self.assertIn("listing failed (HTTP 404)", stderr) - self.assertIn(f"retry failed (HTTP {status})", stderr) - self.retry_failure = None - self.retry_recovers = True - self.invalid_json = True - code, stderr = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 1) - self.assertIn("directory retry", stderr) - - def test_ref_is_encoded_in_both_directory_and_commit_requests(self) -> None: - self.branch = "feature/ci+#" - self.list_failures = {"example/retired": 404} - self.retry_recovers = True - code, _ = self.invoke("retired", "--repo", "example/retired") - self.assertEqual(code, 0) - endpoints = [self.gh_request(call.args[0]).endpoint for call in self.gh.call_args_list] - encoded = quote(self.branch, safe="") - self.assertIn(f"repos/example/retired/contents/.github/workflows?ref={encoded}", endpoints) - self.assertIn(f"repos/example/retired/commits/{encoded}", endpoints) - self.assertIn(encoded, self.rows("retired-gh-runners-confirmed")[0]["html_url"]) - - def test_successful_listing_does_not_probe_ref(self) -> None: - code, _ = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 0) - self.assertFalse( - any("/commits/" in self.gh_request(call.args[0]).endpoint for call in self.gh.call_args_list) - ) + self.assertIn("cannot verify ref", stderr) - def test_malformed_entries_preserve_valid_workflows(self) -> None: - self.unexpected_contents = [ + def test_bad_entries_keep_valid_peers_and_prevent_cache(self) -> None: + good = { + "type": "file", + "path": ".github/workflows/ci.yml", + "download_url": self.files["example/good"].as_uri(), + } + for bad in ( {}, {"type": "file"}, - {"path": ".github/workflows/ci.yml"}, - { - "type": "file", - "path": ".github/workflows/ci.yml", - "download_url": self.files[self.repos[0]].as_uri(), - }, - ] - code, stderr = self.invoke("retired", "--repo", "example/retired") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=1", stderr) - self.assertIn("scan_errors=3", stderr) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed")), 1) - self.assertTrue(all(row["stage"] == "workflow-list" for row in self.rows("scan-errors"))) - - def test_malformed_entries_do_not_publish_inventory_and_recover(self) -> None: - for item in [ - {}, - {"type": "blob"}, - {"name": "ci.yml"}, - {"type": [], "name": "ci.yml", "mode": 33188}, - {"type": "unknown", "name": "ci.yml", "mode": 33188}, - {"type": "blob", "name": "", "mode": 33188}, - ]: - with self.subTest(item=item): - self.graphql_nodes["example/retired"] = self.tree_node("example/retired", entries=[item]) - code, stderr = self.invoke("retired", "--owner", "example") + {"path": "bad.yml"}, + {"type": 1, "path": "bad.yml"}, + {"type": "unknown", "path": "bad.yml"}, + None, + dict(good, download_url=None), + ): + with self.subTest(bad=bad): + self.names = ["example/good"] + self.responses["directory"] = [bad, good] + code, stderr = self.invoke("retired", "--owner", "example", "--refresh") self.assertEqual(code, 1) - self.assertIn("scan_complete=false", stderr) + self.assertIn("workflows_scanned=1", stderr) + self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - self.graphql_nodes.clear() - code, _ = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 0) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - def test_valid_non_workflow_entries_and_empty_list_are_skipped(self) -> None: - for contents in [ + def test_normal_empty_and_filtered_listings(self) -> None: + for listing in ( [], - [ - {"type": "file", "path": ".github/workflows/README.md"}, - {"type": "dir", "path": ".github/workflows/nested"}, - {"type": "symlink", "path": ".github/workflows/link.yml"}, - {"type": "submodule", "path": ".github/workflows/module"}, - ], - ]: - with self.subTest(contents=contents): - self.unexpected_contents = contents - code, stderr = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 0) - self.assertIn("workflows_scanned=0", stderr) - self.assertEqual(self.rows("scan-errors"), []) - - def test_non_string_download_url_is_not_cached(self) -> None: - workflow = { - "repo": "example/retired", - "branch": "main", - "path": ".github/workflows/ci.yml", - "url": 123, - "html_url": "https://github.com/example/retired/blob/main/.github/workflows/ci.yml", - } - with mock.patch.object(scan, "collect_org_workflows", return_value=([workflow], [])): - code, _ = self.invoke("all", "--owner", "example") - self.assertEqual(code, 1) - self.assertEqual(self.rows("scan-errors", "example")[0]["stage"], "download") - self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - - def test_legacy_partial_repos_and_v2_workflows_are_rebuilt(self) -> None: - self.cache.mkdir() - old_repos = self.cache / "example-repos.jsonl" - old_workflows = self.cache / "example-workflow-files-v2.tsv" - old_repos.write_text( - json.dumps({"full_name": "example/clean", "default_branch": "main"}), encoding="utf-8" - ) - old_workflows.write_text("\t".join(scan.WORKFLOW_FIELDS) + "\n", encoding="utf-8") - originals = [path.read_bytes() for path in [old_repos, old_workflows]] - code, stderr = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 0) - self.assertIn("workflows_scanned=2", stderr) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - self.assertTrue((self.cache / "example-repos-v2.jsonl").exists()) - self.assertTrue((self.cache / "example-workflow-files-v3.tsv").exists()) - self.assertEqual([path.read_bytes() for path in [old_repos, old_workflows]], originals) - - def test_interrupted_repo_write_never_publishes_valid_prefix(self) -> None: - self.repos.reverse() - write_text = Path.write_text - - def interrupt(path, text, *args, **kwargs): - if path.parent == self.cache: - write_text(path, text.splitlines()[0] + "\n", *args, **kwargs) - raise OSError("interrupted repository write") - return write_text(path, text, *args, **kwargs) - - with mock.patch.object(Path, "write_text", new=interrupt): - code, stderr = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=2", stderr) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - self.assertEqual(list(self.cache.iterdir()), []) - code, _ = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 0) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - self.assertEqual(len((self.cache / "example-repos-v2.jsonl").read_text().splitlines()), 2) - - def test_failed_repo_replace_preserves_previous_complete_cache(self) -> None: - self.invoke("retired", "--owner", "example") - repo_cache = self.cache / "example-repos-v2.jsonl" - workflow_cache = self.cache / "example-workflow-files-v3.tsv" - originals = [repo_cache.read_bytes(), workflow_cache.read_bytes()] - files = set(self.cache.iterdir()) - self.repos.reverse() - replace = scan.os.replace - - def fail_repo_replace(source, target): - if target == repo_cache: - raise OSError("repository replace failed") - return replace(source, target) - - with mock.patch.object(scan.os, "replace", side_effect=fail_repo_replace): - code, stderr = self.invoke("retired", "--owner", "example", "--refresh") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=2", stderr) - self.assertEqual(self.rows("scan-errors", "example")[0]["path"], repo_cache.name) - self.assertEqual([repo_cache.read_bytes(), workflow_cache.read_bytes()], originals) - self.assertEqual(set(self.cache.iterdir()), files) - - def test_corrupt_new_repo_cache_is_rebuilt(self) -> None: - self.cache.mkdir() - cache = self.cache / "example-repos-v2.jsonl" - for contents in [b"{", b"\xff", b"null\n", b'{"full_name":"example/clean"}\n']: - with self.subTest(contents=contents): - cache.write_bytes(contents) - repos, errors = scan.load_repos(self.cache, "example", False) - self.assertEqual(len(repos), 2) - self.assertEqual(errors, []) - self.assertEqual(len(cache.read_text().splitlines()), 2) - - def test_bad_org_metadata_preserves_other_repositories(self) -> None: - self.org_output = json.dumps({"full_name": "example/retired", "default_branch": "main"}) + "\n{}\n" - code, stderr = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=1", stderr) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - self.assertEqual(self.rows("scan-errors", "example")[0]["stage"], "metadata") - self.assertFalse((self.cache / "example-repos-v2.jsonl").exists()) - self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - - def test_unusable_cache_directory_does_not_discard_enumerated_results(self) -> None: - self.cache.write_text("not a directory", encoding="utf-8") - code, stderr = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=2", stderr) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - self.assertEqual(self.rows("scan-errors", "example")[0]["stage"], "cache") - - def test_interrupted_workflow_cache_write_preserves_results_and_recovers(self) -> None: - write_tsv = scan.write_tsv - - def interrupt(path, rows, fields): - if path.parent == self.cache: - write_tsv(path, [], fields) - raise OSError("interrupted workflow write") - return write_tsv(path, rows, fields) - - with mock.patch.object(scan, "write_tsv", side_effect=interrupt): - code, stderr = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=2", stderr) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - self.assertEqual([path.name for path in self.cache.iterdir()], ["example-repos-v2.jsonl"]) - code, _ = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 0) - self.assertTrue((self.cache / "example-workflow-files-v3.tsv").exists()) - - def test_cache_failure_uses_current_results_not_previous_inventory(self) -> None: - self.invoke("retired", "--owner", "example") - cache = self.cache / "example-workflow-files-v3.tsv" - original = cache.read_bytes() - current = self.root / "current.yml" - current.write_text(json.dumps({"jobs": {"build": {"runs-on": "ubuntu-24.04"}}}), encoding="utf-8") - self.files["example/retired"] = current - replace = scan.os.replace - - def fail_workflow_replace(source, target): - if target == cache: - raise OSError("workflow replace failed") - return replace(source, target) - - with mock.patch.object(scan.os, "replace", side_effect=fail_workflow_replace): - code, stderr = self.invoke("retired", "--owner", "example", "--refresh") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=2", stderr) - self.assertEqual(self.rows("retired-gh-runners-confirmed", "example"), []) - self.assertEqual(cache.read_bytes(), original) - - def test_corrupt_new_workflow_cache_is_rebuilt(self) -> None: - self.cache.mkdir() - cache = self.cache / "example-workflow-files-v3.tsv" - header = "\t".join(scan.WORKFLOW_FIELDS) + "\n" - documents: list[str | bytes] = [ - header + "example/clean\tmain\tci.yml\t\n", - header + "a\tb\tc\td\te\tf\n", - header + '"unterminated', - header.encode() + b"\xff", - ] - for contents in documents: - with self.subTest(contents=contents): - cache.write_bytes(contents.encode() if isinstance(contents, str) else contents) - code, stderr = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 0) - self.assertIn("workflows_scanned=2", stderr) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - - def test_utf8_response_preserves_non_ascii_workflow_path(self) -> None: - self.unexpected_contents = [ - { - "type": "file", - "path": ".github/workflows/构建.yml", - "download_url": self.files["example/retired"].as_uri(), - } - ] - code, _ = self.invoke("retired", "--repo", "example/retired") - self.assertEqual(code, 0) - self.assertEqual(self.rows("retired-gh-runners-confirmed")[0]["path"], ".github/workflows/构建.yml") - - def test_utf8_failure_diagnostic_preserves_http_status(self) -> None: - def non_ascii_failure(args, **kwargs): - return self.completed(args, 1, "", "gh: ✓ 请求失败 (HTTP 403)") - - self.override_gh("contents", non_ascii_failure) - code, stderr = self.invoke("retired", "--repo", "example/retired") - self.assertEqual(code, 1) - self.assertIn("✓ 请求失败 (HTTP 403)", stderr) - self.assertEqual(self.rows("scan-errors")[0]["stage"], "workflow-list") - - def test_invalid_utf8_org_response_is_not_cached(self) -> None: - def invalid_response(args, **kwargs): - return self.completed(args, 0, b"\xff", b"") - - self.override_gh("repositories", invalid_response) - code, stderr = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 1) - self.assertIn("invalid UTF-8", stderr) - self.assertFalse((self.cache / "example-repos-v2.jsonl").exists()) - - def test_invalid_utf8_and_missing_streams_are_structured_prerequisite_errors(self) -> None: - self.gh.side_effect = None - for code, stdout, stderr in [ - (0, b"\xff", b""), - (0, None, None), - (1, None, None), - (1, b"", b"\xff"), - (1, b"failure", None), - ]: - with self.subTest(code=code, stdout=stdout, stderr=stderr): - self.gh.return_value = self.completed([], code, stdout, stderr) - result, summary = self.invoke("retired", "--repo", "example/retired") - self.assertEqual(result, 2) - self.assertIn("scan_complete=false", summary) - self.assertEqual(self.rows("scan-errors")[0]["stage"], "prerequisite") - - def test_malformed_diagnostic_bytes_keep_failure_and_http_status(self) -> None: - self.gh.side_effect = lambda args, **kwargs: self.completed(args, 1, b"\xff", b"\xff (HTTP 403)") - with self.assertRaises(scan.GhError) as error: - scan.run(["gh", "api", "repos/example/retired"]) - self.assertEqual(error.exception.status, 403) - self.assertIn("(HTTP 403)", str(error.exception)) - - def test_real_utf8_subprocesses_are_independent_of_parent_locale(self) -> None: - self.branch = "feature/中文" - response = self.gh_run - - def real_child(args, **kwargs): - completed = response(args, **kwargs) - stderr = "✓ authentication status\n".encode() if args[1] == "auth" else completed.stderr - producer = ( - f"import sys; sys.stdout.buffer.write({completed.stdout!r}); " - f"sys.stderr.buffer.write({stderr!r}); sys.exit({completed.returncode})" - ) - return self.process_run([sys.executable, "-c", producer], **kwargs) - - self.gh.side_effect = real_child - code, _ = self.invoke("retired", "--repo", "example/retired") - self.assertEqual(code, 0) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed")), 1) - self.auth_failure = True - code, stderr = self.invoke("retired", "--repo", "example/retired") - self.assertEqual(code, 2) - self.assertIn("✓ authentication status", stderr) - self.assertEqual(self.rows("scan-errors")[0]["stage"], "prerequisite") - - def test_all_preserves_retired_findings_when_arch_check_fails(self) -> None: - with mock.patch.object(scan, "arch_hits", side_effect=RuntimeError("arch failure")): - code, stderr = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=0", stderr) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed")), 1) - error = self.rows("scan-errors")[0] - self.assertEqual(error["stage"], "check") - self.assertIn("macos-arch: RuntimeError: arch failure", error["error"]) - - def test_all_runs_arch_check_when_retired_check_fails(self) -> None: - self.files["example/retired"].write_text( - json.dumps( - { - "jobs": { - "build": { - "runs-on": "macos-13", - "steps": [{"uses": "actions/setup-python@v6", "with": {"architecture": "arm64"}}], - } - } - } - ), - encoding="utf-8", - ) - with ( - mock.patch.object(scan, "retired_hits", side_effect=RuntimeError("retired failure")), - mock.patch.object(scan, "fetch_text", wraps=scan.fetch_text) as fetch, - mock.patch.object(scan, "yaml_load", wraps=scan.yaml_load) as parse, + [{"type": "dir", "path": ".github/workflows/old"}, {"type": "file", "path": "README.md"}], ): - code, stderr = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=0", stderr) - self.assertEqual(len(self.rows("macos-setup-action-arch-mismatches")), 1) - self.assertIn("retired: RuntimeError: retired failure", self.rows("scan-errors")[0]["error"]) - self.assertEqual(fetch.call_count, 1) - self.assertEqual(parse.call_count, 1) - - def test_single_check_does_not_execute_unselected_check(self) -> None: - for command, other in [("retired", "arch_hits"), ("macos-arch", "retired_hits")]: - with self.subTest(command=command), mock.patch.object(scan, other) as unselected: - code, stderr = self.invoke(command, "--repo", "example/retired") - self.assertEqual(code, 0) - self.assertIn("workflows_scanned=1", stderr) - unselected.assert_not_called() - - def test_check_failure_preserves_other_workflows_and_completion_count(self) -> None: - arch = scan.arch_hits - - def fail_one(workflow, data): - if workflow["repo"] == "example/retired": - raise RuntimeError("one check failed") - return arch(workflow, data) - - with mock.patch.object(scan, "arch_hits", side_effect=fail_one): - code, stderr = self.invoke("all", "--owner", "example") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=1", stderr) - self.assertIn("scan_errors=1", stderr) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - - def test_graphql_partial_success_keeps_original_diagnostic(self) -> None: - self.graphql_errors = [{"type": "FORBIDDEN", "message": "permission denied", "path": ["r1"]}] - self.graphql_nodes["example/clean"] = None + with self.subTest(listing=listing): + self.responses["directory"] = listing + self.assertEqual(self.invoke("all", "--repo", "example/good")[0], 0) + self.assertEqual(self.rows(), []) + + def test_partial_inventory_recovers_without_refresh_then_reuses_cache(self) -> None: + endpoint = "repos/example/clean/contents/.github/workflows?ref=main" + self.responses[endpoint] = self.failed(403) code, stderr = self.invoke("all", "--owner", "example") self.assertEqual(code, 1) self.assertIn("workflows_scanned=1", stderr) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - error = self.rows("scan-errors", "example")[0] - self.assertEqual(error["repo"], "example/clean") - self.assertIn("permission denied", error["error"]) - self.assertIn("gh: GraphQL field errors", error["error"]) - self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - - def test_graphql_absence_requires_exact_directory_error(self) -> None: - for path, error_type in [ - (["r0"], "NOT_FOUND"), - (["r0", "defaultBranchRef"], "NOT_FOUND"), - (["r0", "defaultBranchRef", "target", "workflowDir", "object"], "NOT_FOUND"), - (["r0", "defaultBranchRef", "target", "workflowDir"], "FORBIDDEN"), - (["r1", "defaultBranchRef", "target", "workflowDir"], "NOT_FOUND"), - ]: - with self.subTest(path=path, error_type=error_type): - self.graphql_nodes["example/retired"] = self.tree_node("example/retired") - self.graphql_nodes["example/retired"]["defaultBranchRef"]["target"]["workflowDir"] = None - self.graphql_errors = [ - {"type": error_type, "message": "missing or inaccessible", "path": path} - ] - code, stderr = self.invoke("all", "--owner", "example", "--refresh") - self.assertEqual(code, 1) - self.assertIn("scan_complete=false", stderr) - self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - - def test_graphql_verified_absence_is_complete_and_cacheable(self) -> None: - self.list_failures = {"example/clean": 404} - code, stderr = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 0) - self.assertIn("workflows_scanned=1", stderr) - self.assertEqual(self.rows("scan-errors", "example"), []) - self.assertTrue((self.cache / "example-workflow-files-v3.tsv").exists()) - self.assertEqual( - [self.gh_request(call.args[0]).kind for call in self.gh.call_args_list], - ["auth", "repositories", "graphql"], - ) - - def test_graphql_unverified_nulls_and_missing_fields_are_errors(self) -> None: - original = self.tree_node("example/retired") - variations = [] - for field in ("nameWithOwner", "isEmpty", "defaultBranchRef"): - node = copy.deepcopy(original) - del node[field] - variations.append(node) - for ref_value in (None, {}, {"name": "main", "target": None}): - node = copy.deepcopy(original) - node["defaultBranchRef"] = ref_value - variations.append(node) - commit_value: object - for field, commit_value in [ - ("__typename", "Tag"), - ("oid", "g" * 40), - ("workflowDir", None), - ("workflowDir", {}), - ]: - node = copy.deepcopy(original) - node["defaultBranchRef"]["target"][field] = commit_value - variations.append(node) - node = copy.deepcopy(original) - del node["defaultBranchRef"]["target"]["workflowDir"] - variations.append(node) - for tree_value in (None, {}, {"__typename": "Blob"}, {"__typename": "Tree", "entries": None}): - node = copy.deepcopy(original) - node["defaultBranchRef"]["target"]["workflowDir"]["object"] = tree_value - variations.append(node) - for node in variations: - with self.subTest(node=node): - self.graphql_nodes["example/retired"] = node - code, stderr = self.invoke("all", "--owner", "example", "--refresh") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=1", stderr) - self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - - def test_graphql_not_found_cannot_hide_invalid_commit(self) -> None: - self.list_failures = {"example/retired": 404} - node = self.tree_node("example/retired") - node["defaultBranchRef"]["target"]["oid"] = "not-a-sha" - self.graphql_nodes["example/retired"] = node - code, stderr = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 1) - self.assertIn("valid Commit", stderr) self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) + self.responses.clear() + self.assertEqual(self.invoke("all", "--owner", "example")[0], 0) + self.calls.clear() + self.assertEqual(self.invoke("all", "--owner", "example")[0], 0) + self.assertEqual(self.calls, []) - def test_graphql_entry_modes_preserve_only_regular_yaml_files(self) -> None: - entries = [ - {"name": "normal.yml", "type": "blob", "mode": 0o100644}, - {"name": "executable.yaml", "type": "blob", "mode": 0o100755}, - {"name": "README.md", "type": "blob", "mode": 0o100644}, - {"name": "nested.yml", "type": "tree", "mode": 0o040000}, - {"name": "link.yml", "type": "blob", "mode": 0o120000}, - {"name": "submodule.yaml", "type": "commit", "mode": 0o160000}, - ] - self.graphql_nodes["example/retired"] = self.tree_node("example/retired", entries) - workflows, errors = scan.collect_org_workflows( - [{"full_name": "example/retired", "default_branch": "main"}] - ) - self.assertEqual(errors, []) - self.assertEqual( - [row["path"] for row in workflows], - [".github/workflows/executable.yaml", ".github/workflows/normal.yml"], - ) - - def test_graphql_malformed_entries_retain_valid_peers(self) -> None: - entries = [ - {"name": "ci.yml", "type": "blob", "mode": 0o100644}, - {}, - None, - {"name": "bad.yml", "type": "tree", "mode": 0o100644}, - {"name": "bad.yml", "type": "blob", "mode": True}, - {"name": "nested/file.yml", "type": "blob", "mode": 0o100644}, - ] - self.graphql_nodes["example/retired"] = self.tree_node("example/retired", entries) - code, stderr = self.invoke("retired", "--owner", "example") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=2", stderr) - self.assertIn("scan_errors=5", stderr) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) + def test_legacy_and_damaged_caches_are_rebuilt(self) -> None: + self.cache.mkdir() + for name in ("example-repos.jsonl", "example-workflow-files.tsv", "example-workflow-files-v2.tsv"): + (self.cache / name).write_text("old incomplete inventory") + self.assertEqual(self.invoke("all", "--owner", "example")[0], 0) + (self.cache / "example-workflow-files-v3.tsv").write_text("repo\tpath\nexample/good\tbad\n") + self.calls.clear() + self.assertEqual(self.invoke("all", "--owner", "example")[0], 0) + self.assertTrue(self.calls) - def test_graphql_unassigned_and_unknown_errors_block_cache(self) -> None: - for path in ( - None, - [], - ["r99"], - ["r0", "defaultBranchRef", "target", "workflowDir", "object", "entries", 0], - ): - with self.subTest(path=path): - self.graphql_errors = [{"type": "UNKNOWN", "message": "unexpected error", "path": path}] - code, stderr = self.invoke("retired", "--owner", "example", "--refresh") + def test_cache_replace_failures_keep_old_files_and_scanned_findings(self) -> None: + self.assertEqual(self.invoke("all", "--owner", "example")[0], 0) + original = scan.os.replace + for name in ("example-repos-v2.jsonl", "example-workflow-files-v3.tsv"): + with self.subTest(cache=name): + previous = (self.cache / name).read_bytes() + + def denied(source, target, *, name=name): + if Path(target).name == name: + raise OSError("cache denied") + return original(source, target) + + with mock.patch.object(scan.os, "replace", side_effect=denied): + code, stderr = self.invoke("all", "--owner", "example", "--refresh") self.assertEqual(code, 1) self.assertIn("workflows_scanned=2", stderr) - self.assertIn("unexpected error", stderr) - self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - - def test_graphql_abnormal_success_and_failure_responses_are_errors(self) -> None: - payloads: list[Any] = [ - "invalid JSON", - b"\xff", - None, - {}, - {"data": None}, - {"data": []}, - {"data": {}}, - {"data": {}, "errors": {}}, - {"data": {}, "errors": [None]}, - {"data": {}, "errors": [{"type": "NOT_FOUND"}]}, - ] - for code in (0, 1): - for payload in payloads: - with self.subTest(code=code, payload=payload): - self.override_graphql(payload, code, "original GraphQL failure") - result, stderr = self.invoke("retired", "--owner", "example", "--refresh") - self.assertEqual(result, 1) - self.assertIn("scan_complete=false", stderr) - self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - if code: - self.assertIn("original GraphQL failure", stderr) - - def test_generic_run_does_not_accept_graphql_partial_failure(self) -> None: - payload = self.graphql_payload() - payload["errors"] = [{"type": "NOT_FOUND", "message": "missing repo", "path": ["r1"]}] - self.override_graphql(payload, 1, "original stderr") - args = [ - "gh", - "api", - "graphql", - "-f", - "query=" + scan.workflow_query([{"full_name": repo} for repo in self.repos]), - ] - with self.assertRaises(scan.GhError) as caught: - scan.run(args) - self.assertIsNotNone(caught.exception.stdout) - assert caught.exception.stdout is not None - self.assertEqual(json.loads(caught.exception.stdout), payload) - response, failure = scan.graphql_response(args[-1].split("=", 1)[1], 60) - self.assertEqual(response, payload) - self.assertIsInstance(failure, scan.GhError) - self.assertIn("original stderr", str(failure)) - - def test_graphql_batches_are_serial_and_at_most_fifty(self) -> None: - repos = [{"full_name": f"example/repo{i}", "default_branch": "main"} for i in range(101)] - workflows, errors = scan.collect_org_workflows(repos) - self.assertEqual(errors, []) - self.assertEqual(len(workflows), 101) - queries = [self.gh_request(call.args[0]).fields["query"] for call in self.gh.call_args_list] - self.assertEqual([len(re.findall(r"r\d+: repository", query)) for query in queries], [50, 50, 1]) - self.assertIn('name: "repo0"', queries[0]) - self.assertIn('name: "repo50"', queries[1]) - self.assertIn('name: "repo100"', queries[2]) - self.assertTrue(all('workflowDir: file(path: ".github/workflows")' in query for query in queries)) - - def test_graphql_rate_limit_stops_remaining_batches_and_keeps_successes(self) -> None: - repos = [{"full_name": f"example/repo{i}", "default_branch": "main"} for i in range(101)] - original = self.gh_run - count = 0 - - def limited(args, **kwargs): - nonlocal count - count += 1 - completed = original(args, **kwargs) - if count == 2: - response = json.loads(completed.stdout) - response["errors"] = [ - {"type": "RATE_LIMITED", "message": "API rate limit exceeded", "path": ["r0"]} - ] - response["data"]["r0"] = None - return self.completed(args, 1, json.dumps(response), "gh: API rate limit exceeded") - return completed - - self.override_gh("graphql", limited) - workflows, errors = scan.collect_org_workflows(repos) - self.assertEqual(count, 2) - self.assertEqual(len(workflows), 99) - self.assertTrue( - any(error["repo"] == "example/repo100" and "Not queried" in error["error"] for error in errors) + self.assertEqual(self.rows(prefix="example")[0]["stage"], "cache") + self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) + self.assertEqual((self.cache / name).read_bytes(), previous) + self.assertEqual(len(list(self.cache.iterdir())), 2) + + def test_interrupted_cache_writer_cannot_publish_a_prefix(self) -> None: + path = self.cache / "inventory" + path.parent.mkdir() + path.write_text("complete") + + def interrupted(temporary): + temporary.write_text("valid prefix") + raise OSError("interrupted") + + with self.assertRaises(OSError): + scan.atomic_write(path, interrupted) + self.assertEqual(path.read_text(), "complete") + self.assertEqual(list(path.parent.iterdir()), [path]) + + def test_failed_pagination_never_publishes_partial_repositories(self) -> None: + self.responses["repositories"] = self.completed( + [], 1, b'{"full_name":"example/good","default_branch":"main"}\n', b"pagination failed" ) - - def test_graphql_global_rate_limit_and_http_rate_limits_stop(self) -> None: - repos = [{"full_name": f"example/repo{i}", "default_branch": "main"} for i in range(51)] - for payload, diagnostic in [ - ( - {"data": None, "errors": [{"type": "RATE_LIMITED", "message": "API rate limit exceeded"}]}, - "gh: API rate limit exceeded", - ), - ("plain text", "gh: rate limit exceeded (HTTP 429)"), - ("plain text", "gh: API rate limit exceeded (HTTP 403)"), - ]: - with self.subTest(payload=payload): - self.gh.reset_mock() - self.override_graphql(payload, 1, diagnostic) - workflows, errors = scan.collect_org_workflows(repos) - self.assertEqual(workflows, []) - self.assertEqual(self.gh.call_count, 1) - self.assertTrue( - any(row["repo"] == "example/repo50" and "Not queried" in row["error"] for row in errors) - ) - - def test_graphql_permission_failure_does_not_stop_later_batches(self) -> None: - repos = [{"full_name": f"example/repo{i}", "default_branch": "main"} for i in range(51)] - original = self.gh_run - count = 0 - - def forbidden(args, **kwargs): - nonlocal count - count += 1 - if count == 1: - return self.completed(args, 1, "", "gh: permissions denied (HTTP 403)") - return original(args, **kwargs) - - self.override_gh("graphql", forbidden) - workflows, errors = scan.collect_org_workflows(repos) - self.assertEqual(count, 2) - self.assertEqual(len(workflows), 1) - self.assertEqual(len(errors), 50) - - def test_auth_timeout_never_triggers_legacy_fallback(self) -> None: - def timeout(args, **kwargs): - raise subprocess.TimeoutExpired( - args, kwargs["timeout"], output=b"partial auth", stderr=b"unknown flag: --active" - ) - - self.override_gh("auth", timeout) - code, stderr = self.invoke("all", "--repo", "example/retired", "--gh-auth-timeout", "7") - self.assertEqual(code, 2) - self.assertEqual(self.gh.call_count, 1) - self.assertIn("gh auth status: timed out after 7s", stderr) - self.assertIn("partial auth", stderr) - self.assertEqual(self.rows("scan-errors")[0]["stage"], "prerequisite") - - def test_pagination_timeout_discards_partial_stdout_and_stops(self) -> None: - def timeout(args, **kwargs): - raise subprocess.TimeoutExpired( - args, - kwargs["timeout"], - output=json.dumps({"full_name": "example/retired", "default_branch": "main"}).encode(), - stderr=b"partial listing", - ) - - self.override_gh("repositories", timeout) - code, stderr = self.invoke("all", "--owner", "example", "--gh-pagination-timeout", "11") + code, stderr = self.invoke("all", "--owner", "example") self.assertEqual(code, 1) - self.assertIn("timed out after 11s", stderr) - self.assertIn("/orgs/example/repos?per_page=100&type=public", stderr) - self.assertIn("workflows_scanned=0", stderr) - self.assertEqual(self.gh.call_count, 2) + self.assertIn("pagination failed", stderr) self.assertFalse((self.cache / "example-repos-v2.jsonl").exists()) self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - def test_metadata_and_directory_timeouts_preserve_other_repos(self) -> None: - original = self.gh_run - for kind in ("metadata", "contents", "commit"): - with self.subTest(kind=kind): - self.list_failures = {"example/retired": 404} if kind == "commit" else {} - - def timeout(args, **kwargs): - if "example/retired" in self.gh_request(args).endpoint: - raise subprocess.TimeoutExpired( - args, kwargs["timeout"], output=b"partial", stderr=b"\xff failure" - ) - return original(args, **kwargs) - - self.override_gh(kind, timeout) - code, stderr = self.invoke( - "all", "--repo", "example/retired", "--repo", "example/clean", "--gh-api-timeout", "9" - ) - self.assertEqual(code, 1) - self.assertIn("timed out after 9s", stderr) - self.assertIn("workflows_scanned=1", stderr) - self.assertEqual( - self.rows("scan-errors", "repo-set")[0]["stage"], - "metadata" if kind == "metadata" else "workflow-list", - ) - self.gh.side_effect = original - - def test_graphql_batch_timeout_never_accepts_partial_data_and_continues(self) -> None: - repos = [{"full_name": f"example/repo{i}", "default_branch": "main"} for i in range(51)] - original = self.gh_run - count = 0 - - def timeout(args, **kwargs): - nonlocal count - count += 1 - completed = original(args, **kwargs) - if count == 1: - raise subprocess.TimeoutExpired( - args, kwargs["timeout"], output=completed.stdout, stderr=b"partial response" - ) - return completed - - self.override_gh("graphql", timeout) - workflows, errors = scan.collect_org_workflows(repos, 13) - self.assertEqual(count, 2) - self.assertEqual(len(workflows), 1) - self.assertEqual(len(errors), 50) - self.assertTrue(all("gh api graphql: timed out after 13s" in row["error"] for row in errors)) - self.assertTrue(all("query=" not in row["error"] for row in errors)) - - def test_timeout_options_defaults_overrides_and_invalid_values(self) -> None: - self.assertEqual(self.invoke("retired", "--repo", "example/retired")[0], 0) - self.assertEqual([call.kwargs["timeout"] for call in self.gh.call_args_list], [30, 60, 60]) - self.gh.reset_mock() - self.assertEqual( - self.invoke( - "retired", - "--owner", - "example", - "--refresh", - "--gh-auth-timeout", - "3", - "--gh-api-timeout", - "5", - "--gh-pagination-timeout", - "17", - )[0], - 0, - ) - self.assertEqual([call.kwargs["timeout"] for call in self.gh.call_args_list], [3, 17, 5]) - for option in ("--gh-auth-timeout", "--gh-api-timeout", "--gh-pagination-timeout"): - for value in ("0", "-1", "1.5", "bad"): - with self.subTest(option=option, value=value), self.assertRaises(SystemExit) as error: - self.invoke("all", option, value) - self.assertEqual(error.exception.code, 2) - - def test_cache_origin_only_marks_valid_workflow_cache_reuse(self) -> None: - workflows, errors, cached = scan.load_workflows(self.cache, "example", False) - self.assertEqual(errors, []) - self.assertFalse(cached) - self.assertTrue(workflows) - self.assertTrue(scan.load_workflows(self.cache, "example", False)[2]) - self.assertFalse(scan.load_workflows(self.cache, "example", True)[2]) - cache = self.cache / "example-workflow-files-v3.tsv" - cache.write_text("invalid header\n", encoding="utf-8") - self.assertFalse(scan.load_workflows(self.cache, "example", False)[2]) - before = cache.read_bytes() - self.list_failures = {"example/retired": 403} - workflows, errors, cached = scan.load_workflows(self.cache, "example", True) - self.assertFalse(cached) - self.assertTrue(errors) - self.assertEqual(cache.read_bytes(), before) - self.assertEqual([row["repo"] for row in workflows], ["example/clean"]) - - def test_refresh_hint_requires_cached_inventory_and_actual_http_status(self) -> None: - for cache_state in ("cold", "valid", "corrupt", "repo-only", "failed-refresh"): - for failure in ("http", "text"): - with self.subTest(cache_state=cache_state, failure=failure): - if self.cache.exists(): - for path in self.cache.iterdir(): - path.unlink() - if cache_state in ("valid", "corrupt", "failed-refresh"): - self.assertEqual(self.invoke("all", "--owner", "example")[0], 0) - if cache_state == "corrupt": - (self.cache / "example-workflow-files-v3.tsv").write_text("bad\n", encoding="utf-8") - if cache_state == "repo-only": - scan.load_repos(self.cache, "example", False) - self.list_failures = {"example/clean": 403} if cache_state == "failed-refresh" else {} - exc = ( - HTTPError("https://example.invalid", 404, "Not Found", Message(), None) - if failure == "http" - else RuntimeError("HTTP Error 404: Not Found") - ) - scope = ["--owner", "example"] + ( - ["--refresh"] if cache_state == "failed-refresh" else [] - ) - with mock.patch.object(scan, "fetch_text", side_effect=exc): - code, _ = self.invoke("all", *scope) - self.assertEqual(code, 1) - errors = [ - row for row in self.rows("scan-errors", "example") if row["stage"] == "download" - ] - self.assertTrue(errors) - self.assertEqual( - any("inventory may be stale" in row["error"] for row in errors), - cache_state == "valid" and failure == "http", - ) - self.assertTrue( - all(set(row) == set(scan.ERROR_FIELDS) for row in self.rows("scan-errors", "example")) - ) - self.list_failures = {} - - def test_static_runner_does_not_expand_unused_matrix(self) -> None: - document = { - "jobs": { - "build": { - "runs-on": "ubuntu-20.04", - "strategy": {"matrix": {"a": list(range(1000)), "b": list(range(1000))}}, - } - } - } - with mock.patch.object(scan, "matrix_rows", side_effect=AssertionError("unused matrix expanded")): - hits = scan.retired_hits({}, document) - self.assertEqual([hit["runner"] for hit in hits], ["ubuntu-20.04"]) - - def test_dynamic_runner_matching_and_bad_strategy_stay_consistent(self) -> None: - document = { - "jobs": { - "build": { - "runs-on": "${{ matrix.os }}", - "strategy": {"matrix": {"os": ["ubuntu-20.04", "ubuntu-24.04"]}}, - } + def test_utf8_responses_and_missing_streams_are_structured_errors(self) -> None: + self.responses["directory"] = [ + { + "type": "file", + "path": ".github/workflows/构建.yml", + "download_url": self.files["example/good"].as_uri(), } - } - with mock.patch.object(scan, "matrix_rows", wraps=scan.matrix_rows) as expand: - self.assertEqual([hit["runner"] for hit in scan.retired_hits({}, document)], ["ubuntu-20.04"]) - self.assertEqual(expand.call_count, 1) - for strategy in ("bad", [1], 42): - with self.subTest(strategy=strategy), self.assertRaises(AttributeError): - scan.retired_hits({}, {"jobs": {"build": {"runs-on": "ubuntu-20.04", "strategy": strategy}}}) - - def test_mock_routing_is_semantic_and_unknown_routes_are_recorded(self) -> None: - args = [ - "gh", - "api", - "-H", - "Accept: application/vnd.github.sha", - f"repos/example/retired/commits/{self.sha}", - ] - self.assertEqual(scan.run(args), self.sha) - args = [ - "gh", - "api", - "-f", - "query=" + scan.workflow_query([{"full_name": "example/retired"}]), - "graphql", - ] - self.assertIn("r0", json.loads(scan.run(args))["data"]) - with self.assertRaises(AssertionError): - self.gh_run(["gh", "api", "unsupported-endpoint"]) - self.assertEqual(self.unexpected_calls, [["gh", "api", "unsupported-endpoint"]]) - self.unexpected_calls.clear() # This case deliberately asserted the recorded routing violation. - - def test_override_routes_cannot_hide_unknown_requests(self) -> None: - self.override_graphql(self.graphql_payload()) - requests = [["gh", "api", "unsupported-endpoint"], ["gh", "api", "--unsupported", "graphql"]] - for args in requests: - with self.subTest(args=args), self.assertRaises(AssertionError): - scan.run(args) - self.assertEqual(self.unexpected_calls, requests) - self.unexpected_calls.clear() # Deliberate violations were asserted above. - - def test_rest_empty_repository_is_still_verified_by_409(self) -> None: - self.list_failures = {"example/retired": 404} - self.override_gh( - "commit", - lambda args, **kwargs: self.completed(args, 1, "", "gh: Git Repository is empty. (HTTP 409)"), - ) - code, stderr = self.invoke("all", "--repo", "example/retired") - self.assertEqual(code, 0) - self.assertIn("workflows_scanned=0", stderr) - self.assertEqual(self.rows("scan-errors"), []) - - def test_graphql_http_rate_limit_with_partial_data_stops_later_batches(self) -> None: - repos = [{"full_name": f"example/repo{i}", "default_branch": "main"} for i in range(51)] - payload = { - "data": {f"r{i}": self.tree_node(f"example/repo{i}") for i in range(50)}, - "errors": [{"message": "partial failure", "path": ["r0"]}], - } - for status in (429, 403): - with self.subTest(status=status): - self.gh.reset_mock() - self.override_graphql(payload, 1, f"gh: API rate limit exceeded (HTTP {status})") - workflows, errors = scan.collect_org_workflows(repos) - self.assertEqual(self.gh.call_count, 1) - self.assertEqual(len(workflows), 50) - self.assertTrue( - any(row["repo"] == "example/repo50" and "Not queried" in row["error"] for row in errors) - ) - - def test_graphql_bad_repository_name_does_not_contaminate_batch(self) -> None: - repos = [ - {"full_name": "example/retired", "default_branch": "main"}, - {"full_name": "missing-owner-separator", "default_branch": "main"}, - {"full_name": "example/", "default_branch": "main"}, ] - workflows, errors = scan.collect_org_workflows(repos) - self.assertEqual([row["repo"] for row in workflows], ["example/retired"]) - self.assertEqual(len(errors), 2) - self.assertTrue(all(row["stage"] == "metadata" for row in errors)) - self.assertEqual(self.gh.call_count, 1) - - def test_graphql_http_failure_cannot_be_absorbed_as_normal_absence(self) -> None: - node = self.tree_node("example/retired") - node["defaultBranchRef"]["target"]["workflowDir"] = None - payload = { - "data": {"r0": node}, - "errors": [ - { - "type": "NOT_FOUND", - "message": "missing directory", - "path": ["r0", "defaultBranchRef", "target", "workflowDir"], - } - ], - } - for status in (403, 429, 500): - with self.subTest(status=status): - self.repos = ["example/retired"] - self.override_graphql(payload, 1, f"gh: transport failed (HTTP {status})") - code, stderr = self.invoke("all", "--owner", "example", "--refresh") - self.assertEqual(code, 1) - self.assertIn(f"transport failed (HTTP {status})", stderr) - self.assertIn("scan_errors=1", stderr) - self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - - def test_same_owner_rename_uses_canonical_name_and_urls(self) -> None: - self.repos = ["example/retired"] - self.graphql_nodes[self.repos[0]] = self.tree_node("Example/renamed") - self.files["Example/renamed"] = self.files[self.repos[0]] - code, stderr = self.invoke("all", "--owner", "example") - self.assertEqual(code, 0) - self.assertIn("workflows_scanned=1", stderr) - row = self.rows("retired-gh-runners-confirmed", "example")[0] - self.assertEqual(row["repo"], "Example/renamed") - self.assertIn("/Example/renamed/blob/main/", row["html_url"]) - with (self.cache / "example-workflow-files-v3.tsv").open(encoding="utf-8") as source: - cached = next(csv.DictReader(source, delimiter="\t")) - self.assertIn("/Example/renamed/main/", cached["url"]) - - def test_cross_owner_transfer_is_out_of_scope_even_when_empty(self) -> None: - for empty in (False, True): - with self.subTest(empty=empty): - node = self.tree_node("outside/retired") - node["isEmpty"] = empty - self.graphql_nodes["example/retired"] = node - code, stderr = self.invoke("all", "--owner", "example", "--refresh") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=1", stderr) - error = self.rows("scan-errors", "example")[0] - self.assertEqual(error["repo"], "example/retired") - for text in ("outside/retired", "outside owner example", "--refresh"): - self.assertIn(text, error["error"]) - self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - - def test_org_scope_uses_declared_owner_not_cached_requested_owner(self) -> None: - repos = [{"full_name": "outside/old"}] - self.graphql_nodes["outside/old"] = self.tree_node("outside/new") - workflows, errors = scan.collect_org_workflows(repos, owner="example") - self.assertEqual(workflows, []) - self.assertIn("outside owner example", errors[0]["error"]) - - def test_malformed_canonical_names_do_not_hide_successful_peers(self) -> None: - for name in (None, 123, "missing-owner", "/repo", "example/", "example/repo/extra"): - with self.subTest(name=name): - node = self.tree_node("example/retired") - node["nameWithOwner"] = name - self.graphql_nodes["example/retired"] = node - code, stderr = self.invoke("all", "--owner", "example", "--refresh") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=1", stderr) - self.assertIn("repository metadata", stderr) - - def test_duplicate_aliases_download_and_parse_once(self) -> None: - self.repos = ["example/retired", "example/legacy"] - self.graphql_nodes["example/legacy"] = self.tree_node("example/retired") - with ( - mock.patch.object(scan, "fetch_text", wraps=scan.fetch_text) as fetch, - mock.patch.object(scan, "yaml_load", wraps=scan.yaml_load) as parse, - ): - code, stderr = self.invoke("all", "--owner", "example") - self.assertEqual(code, 0) - self.assertIn("workflows_scanned=1", stderr) - self.assertEqual(fetch.call_count, 1) - self.assertEqual(parse.call_count, 1) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - - def test_alias_dedup_spans_batches_and_preserves_path_case(self) -> None: - names = ["example/retired"] + [f"example/repo{i}" for i in range(49)] + ["example/legacy"] - entries = [{"name": name, "type": "blob", "mode": 0o100644} for name in ("ci.yml", "CI.yml")] - self.graphql_nodes["example/legacy"] = self.tree_node("Example/Retired", entries) - workflows, errors = scan.collect_org_workflows([{"full_name": name} for name in names]) - self.assertEqual(errors, []) - self.assertEqual(len(workflows), 51) - self.assertEqual(self.gh.call_count, 2) - paths = [row["path"] for row in workflows if row["repo"].casefold() == "example/retired"] - self.assertCountEqual(paths, [".github/workflows/ci.yml", ".github/workflows/CI.yml"]) - - def test_alias_inventory_conflicts_keep_first_and_prevent_cache(self) -> None: - self.repos = ["example/retired", "example/legacy", "example/clean"] - for changed in ("branch", "sha", "empty"): - with self.subTest(changed=changed): - node = self.tree_node("example/retired") - if changed == "branch": - node["defaultBranchRef"]["name"] = "other" - elif changed == "sha": - node["defaultBranchRef"]["target"]["oid"] = "b" * 40 - else: - node["isEmpty"] = True - self.graphql_nodes["example/legacy"] = node - code, stderr = self.invoke("all", "--owner", "example", "--refresh") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=2", stderr) - self.assertIn("Conflicting branch/Commit inventories", stderr) - self.assertEqual(len(self.rows("retired-gh-runners-confirmed", "example")), 1) - self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - - def test_duplicate_success_cannot_erase_alias_errors(self) -> None: - self.repos = ["example/retired", "example/legacy"] - node = self.tree_node("example/retired") - node["defaultBranchRef"]["target"]["workflowDir"]["object"]["entries"].append({"name": "bad.yml"}) - self.graphql_nodes["example/legacy"] = node - code, stderr = self.invoke("all", "--owner", "example") - self.assertEqual(code, 1) - self.assertIn("workflows_scanned=1", stderr) - self.assertIn("Invalid GraphQL Tree entry", stderr) - self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - - def test_no_data_limit_envelopes_stop_and_preserve_diagnostics(self) -> None: - repos = [{"full_name": f"example/repo{i}"} for i in range(101)] - messages = ( - {"errors": [{"type": "RATE_LIMITED", "message": "API rate limit exceeded"}]}, - {"message": "You have exceeded a secondary rate limit. Please wait."}, - ) - for payload in messages: - for code in (0, 1): - with self.subTest(payload=payload, code=code): - self.gh.reset_mock() - count = 0 - - def limited(args, *, code=code, payload=payload, **kwargs): - nonlocal count - count += 1 - if count == 2: - return self.completed(args, code, json.dumps(payload), "gh: original diagnostic") - return self.gh_run(args, **kwargs) - - self.override_gh("graphql", limited) - with contextlib.redirect_stderr(io.StringIO()) as stderr: - workflows, errors = scan.collect_org_workflows(repos) - self.assertEqual(count, 2) - self.assertEqual(len(workflows), 50) - self.assertEqual(len(errors), 51) - self.assertIn("rate limit", errors[0]["error"].lower()) - self.assertNotIn("HTTP 429", errors[0]["error"]) - if code: - self.assertIn("original diagnostic", errors[0]["error"]) - self.assertIn("Not queried", errors[-1]["error"]) - self.assertEqual( - stderr.getvalue().splitlines(), ["discovery_batch=1/3", "discovery_batch=2/3"] - ) - - def test_last_batch_limits_are_errors_and_cannot_publish_cache(self) -> None: - self.repos = ["example/retired"] - for payload in ( - {"errors": [{"type": "RATE_LIMITED", "message": "API rate limit exceeded"}]}, - {"message": "You have exceeded a secondary rate limit. Please wait."}, + self.assertEqual(self.invoke("retired", "--repo", "example/good")[0], 0) + self.assertEqual(self.rows("retired-gh-runners-confirmed")[0]["path"], ".github/workflows/构建.yml") + for response in ( + self.completed([], 0, None, None), + self.completed([], 1, None, None), + self.failed(403, "✓ 请求失败"), ): - with self.subTest(payload=payload): - self.override_graphql(payload) - code, stderr = self.invoke("all", "--owner", "example", "--refresh") - self.assertEqual(code, 1) + with self.subTest(response=response): + self.responses["directory"] = response + self.assertEqual(self.invoke("all", "--repo", "example/good")[0], 1) + self.assertEqual(self.rows()[0]["stage"], "workflow-list") + + def test_unreadable_repo_file_has_correct_failure_prefix(self) -> None: + path = self.root / "repos" + for data in (b"\xff", b"example/good\n\xff"): + with self.subTest(data=data): + path.write_bytes(data) + self.calls.clear() + code, stderr = self.invoke("all", "--repo-file", str(path), "--scope-name", "explicit") + self.assertEqual(code, 2) self.assertIn("workflows_scanned=0", stderr) - self.assertIn("rate limit", stderr.lower()) - self.assertEqual(len(self.rows("scan-errors", "example")), 1) - self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - - def test_normal_absence_cannot_absorb_diagnostic_only_limit(self) -> None: - self.repos = ["example/retired"] - node = self.tree_node(self.repos[0]) - node["defaultBranchRef"]["target"]["workflowDir"] = None - payload = { - "data": {"r0": node}, - "errors": [ - { - "type": "NOT_FOUND", - "message": "missing directory", - "path": ["r0", "defaultBranchRef", "target", "workflowDir"], - } - ], - } - for code in (0, 1): - with self.subTest(code=code): - response = copy.deepcopy(payload) - diagnostic = "gh: API rate limit exceeded" - if not code: - response["message"] = "You have exceeded a secondary rate limit." - self.override_graphql(response, code, diagnostic) - result, stderr = self.invoke("all", "--owner", "example", "--refresh") - self.assertEqual(result, 1) - self.assertIn("scan_errors=1", stderr) - self.assertIn("rate limit", stderr.lower()) - self.assertNotIn("Unexpected GraphQL response", stderr) - self.assertFalse((self.cache / "example-workflow-files-v3.tsv").exists()) - - def test_rate_limit_words_in_business_data_and_ordinary_errors_do_not_stop(self) -> None: - repos = [{"full_name": f"example/repo{i}"} for i in range(51)] - node = self.tree_node("example/repo0", branch="API rate limit exceeded") - self.graphql_nodes["example/repo0"] = node - workflows, errors = scan.collect_org_workflows(repos) - self.assertEqual(len(workflows), 51) - self.assertEqual(errors, []) - self.gh.reset_mock() - count = 0 - - def ordinary_error(args, **kwargs): - nonlocal count - count += 1 - if count == 1: - return self.completed(args, 0, json.dumps({"message": "Configure rate limit settings"})) - return self.gh_run(args, **kwargs) + self.assertEqual(self.rows(prefix="explicit")[0]["stage"], "prerequisite") + self.assertEqual(self.calls, []) - self.override_gh("graphql", ordinary_error) - workflows, errors = scan.collect_org_workflows(repos) - self.assertEqual(count, 2) - self.assertEqual(len(workflows), 1) - self.assertEqual(len(errors), 50) - self.assertIn("Configure rate limit settings", errors[0]["error"]) - self.assertFalse(any("Not queried" in error["error"] for error in errors)) - - def test_timeout_rate_limit_stdout_is_not_a_response(self) -> None: - repos = [{"full_name": f"example/repo{i}"} for i in range(51)] - count = 0 - - def timed_out(args, **kwargs): - nonlocal count - count += 1 - if count == 1: - raise subprocess.TimeoutExpired( - args, - 1, - output=b'{"errors":[{"type":"RATE_LIMITED","message":"API rate limit exceeded"}]}', - ) - return self.gh_run(args, **kwargs) - - self.override_gh("graphql", timed_out) - workflows, errors = scan.collect_org_workflows(repos) - self.assertEqual(count, 2) - self.assertEqual(len(workflows), 1) - self.assertEqual(len(errors), 50) - self.assertFalse(any("Not queried" in error["error"] for error in errors)) - - def test_negated_or_quoted_limit_phrases_do_not_stop_later_batches(self) -> None: - repos = [{"full_name": f"example/repo{i}"} for i in range(51)] - for message in ( - "This is not API rate limit exceeded; permission validation failed.", - 'Permission validation failed while explaining "API rate limit exceeded".', - '"You have exceeded a secondary rate limit" is an example, not this failure.', - ): - with self.subTest(message=message): - count = 0 - - def denied(args, *, message=message, **kwargs): - nonlocal count - count += 1 - completed = self.gh_run(args, **kwargs) - if count == 1: - response = json.loads(completed.stdout) - response["data"]["r0"] = None - response["errors"] = [{"type": "FORBIDDEN", "message": message, "path": ["r0"]}] - return self.completed( - args, 1, json.dumps(response), "gh: permission validation failed" - ) - return completed - - self.override_gh("graphql", denied) - workflows, errors = scan.collect_org_workflows(repos) - self.assertEqual(count, 2) - self.assertEqual(len(workflows), 50) - self.assertEqual(len(errors), 1) - self.assertIn(message, errors[0]["error"]) - self.assertFalse(any("Not queried" in error["error"] for error in errors)) - - def test_warm_inventory_has_no_discovery_or_owner_probe(self) -> None: - self.assertEqual(self.invoke("all", "--owner", "example")[0], 0) - self.graphql_nodes["example/retired"] = self.tree_node("outside/transferred") - self.gh.reset_mock() - code, stderr = self.invoke("all", "--owner", "example") - self.assertEqual(code, 0) - self.assertNotIn("discovery_", stderr) - self.assertEqual([self.gh_request(call.args[0]).kind for call in self.gh.call_args_list], ["auth"]) - - def test_discovery_progress_is_flushed_and_counts_attempted_batches(self) -> None: - repos = [{"full_name": f"example/repo{i}"} for i in range(51)] - with ( - contextlib.redirect_stderr(io.StringIO()) as stderr, - mock.patch("builtins.print", wraps=print) as printed, - ): - scan.load_repos(self.cache, "example", True) - scan.collect_org_workflows(repos) - self.assertEqual( - stderr.getvalue().splitlines(), - [ - "discovery_stage=repositories", - "discovery_batch=1/2", - "discovery_batch=2/2", - ], - ) - self.assertTrue(all(call.kwargs.get("flush") is True for call in printed.call_args_list)) - - def test_warm_inventory_deduplicates_saved_paths_without_api_calls(self) -> None: - self.repos = ["example/retired"] - self.assertEqual(self.invoke("all", "--owner", "example")[0], 0) - cached = self.cache / "example-workflow-files-v3.tsv" - with cached.open(encoding="utf-8") as source: - rows = list(csv.DictReader(source, delimiter="\t")) - scan.write_tsv(cached, rows + rows, scan.WORKFLOW_FIELDS) - self.gh.reset_mock() - with mock.patch.object(scan, "fetch_text", wraps=scan.fetch_text) as fetch: - code, stderr = self.invoke("all", "--owner", "example") - self.assertEqual(code, 0) - self.assertIn("workflows_scanned=1", stderr) - self.assertEqual(fetch.call_count, 1) - self.assertEqual([self.gh_request(call.args[0]).kind for call in self.gh.call_args_list], ["auth"]) - - def test_conflicting_saved_branches_rebuild_the_workflow_inventory(self) -> None: - self.repos = ["example/retired"] - self.assertEqual(self.invoke("all", "--owner", "example")[0], 0) - cached = self.cache / "example-workflow-files-v3.tsv" - with cached.open(encoding="utf-8") as source: - row = next(csv.DictReader(source, delimiter="\t")) - other = dict(row, branch="other") - scan.write_tsv(cached, [row, other], scan.WORKFLOW_FIELDS) - self.gh.reset_mock() - code, stderr = self.invoke("all", "--owner", "example") - self.assertEqual(code, 0) - self.assertIn("discovery_batch=1/1", stderr) - self.assertIn("workflows_scanned=1", stderr) - self.assertEqual( - [self.gh_request(call.args[0]).kind for call in self.gh.call_args_list], ["auth", "graphql"] - ) + def test_output_failure_is_not_success(self) -> None: + with mock.patch.object(scan, "write_tsv", side_effect=OSError("output denied")): + code, stderr = self.invoke("all", "--repo", "example/good") + self.assertEqual(code, 1) + self.assertIn("scan_complete=false", stderr) if __name__ == "__main__": diff --git a/tools/skill-evals/README.md b/tools/skill-evals/README.md index 5c1a97eba..79cf3b90a 100644 --- a/tools/skill-evals/README.md +++ b/tools/skill-evals/README.md @@ -42,7 +42,7 @@ Suites are currently implemented for: - **optimize-skill** — 7 cases across 1 step (step-diagnose) - **committer-onboarding** — 27 cases across 4 steps (step-0-validate-vote, step-1-icla-comms, step-3-checklist, step-4-completion-summary) - **contributor-identity-map** — 6 cases across 1 step (step-2-grade-confirm) -- **ci-runner-audit** — 29 cases across 3 steps (step-scope-selection, step-reporting, step-reporting-scope) +- **ci-runner-audit** — 10 cases across 2 steps (step-scope-selection, step-reporting) - **setup-status** — 18 cases across 5 steps (step-0-preflight, step-1-command, step-2-present, step-3-adjust-decision) - **non-asf-profile-smoke** — 15 cases across 6 steps (step-1-fetch-pool, step-3-classify); drives `issue-stale-sweep` through the `projects/non-asf-example/` fixture to verify non-ASF config resolves without skill-body edits - **contributor-sentiment** — 10 cases across 3 steps (step-0-resolve-inputs, step-2-score-signals, step-3-generate-report) diff --git a/tools/skill-evals/evals/ci-runner-audit/README.md b/tools/skill-evals/evals/ci-runner-audit/README.md index 936e949eb..e334ecd1a 100644 --- a/tools/skill-evals/evals/ci-runner-audit/README.md +++ b/tools/skill-evals/evals/ci-runner-audit/README.md @@ -5,13 +5,12 @@ Behavioral evals for the `ci-runner-audit` skill. -## Suites (29 cases total) +## Suites (10 cases total) | Suite | Step | Cases | What it covers | |---|---|---|---| | step-scope-selection | Scope selection and command choice | 4 | explicit repo, ambiguous Apache project, full-org scan, prompt injection ignored | -| step-reporting | Reporting discipline | 19 | high-confidence vs broad candidates, CI-risk language, incomplete scans, prerequisite and input-encoding failures, consistent failure prefixes, confirmed empty repositories, stale-inventory hints, unknown completeness, invalid refs, malformed entries, cache failures, partial checks | -| step-reporting-scope | Inventory scope and coverage | 6 | canonical renames, cross-owner transfers, conflicting aliases, unqueried scope after limits, final-batch limits, saved snapshot scope | +| step-reporting | Reporting discipline | 6 | high-confidence vs broad candidates, CI-risk language instead of security overclaiming | ## Run @@ -45,18 +44,5 @@ suite asserts that setup-action mismatches are high-confidence, broad macOS candidates are marked false-positive-prone, and runner findings are described as CI breakage / portability risks rather than security vulnerabilities. -It also asserts that empty findings from an incomplete scan are never reported as clean, that successful findings retain their partial-scan qualification, and that stale files after a prerequisite failure are not treated as current results. -The discovery regressions distinguish invalid-ref 404s and malformed workflow-list entries from absent workflows. -Cache-publication and individual-check failures keep their successful findings reportable as partial results, even when all files were enumerated or no file completed every check. -Invalid repo-file encoding stops before repository discovery; previous findings files are not results of that failed invocation. -Confirmed empty Git repositories are complete zero-workflow scans, while stale workflow URLs remain incomplete results with a manual refresh hint. -Resolving a repo-file scope before authentication gives matching error and findings prefixes; old findings remain stale when authentication fails. -GraphQL partial success and unexplained directory nulls retain incomplete reporting; a directory-batch timeout preserves later successful findings without becoming a whole-audit deadline. -Cache-only failures distinguish completed workflow coverage from an incomplete invocation, including zero-finding audits. - -### step-reporting-scope - -Reports use canonical names for successful same-owner renames and identify cross-owner transfers or conflicting inventories as scope failures. -Rate-limit progress does not prove complete coverage, and a limit in the final batch still makes the invocation incomplete. -A complete audit using a saved workflow inventory reports that snapshot's scope without claiming current organization membership was checked. +Reporting regressions distinguish incomplete zero-finding scans, partial findings, missing prerequisites and complete zero-finding audits. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-1-same-owner-rename/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-1-same-owner-rename/expected.json deleted file mode 100644 index 626dd0680..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-1-same-owner-rename/expected.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "scan_status": "complete", - "reports_clean_scan": false, - "high_confidence_count": 1, - "finding_repositories": [ - "Example/current" - ], - "scope_failure_reported": false, - "unqueried_scope_reported": false, - "inventory_scope": "refreshed" -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-1-same-owner-rename/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-1-same-owner-rename/report.md deleted file mode 100644 index 9ee2efab6..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-1-same-owner-rename/report.md +++ /dev/null @@ -1,4 +0,0 @@ - - -Exit 0; scan_complete=true, workflows_scanned=1, scan_errors=0. Command: all --owner example --refresh. The requested example/legacy resolves to Example/current within the same owner. The TSV contains one ubuntu-20.04 finding with repo Example/current and its canonical html_url. All selected repositories were checked. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-2-cross-owner-transfer/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-2-cross-owner-transfer/expected.json deleted file mode 100644 index efeb76952..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-2-cross-owner-transfer/expected.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "scan_status": "incomplete", - "reports_clean_scan": false, - "high_confidence_count": 1, - "finding_repositories": [ - "example/kept" - ], - "scope_failure_reported": true, - "unqueried_scope_reported": false, - "inventory_scope": "refreshed" -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-2-cross-owner-transfer/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-2-cross-owner-transfer/report.md deleted file mode 100644 index d17012229..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-2-cross-owner-transfer/report.md +++ /dev/null @@ -1,4 +0,0 @@ - - -Exit 1; scan_complete=false, workflows_scanned=1, scan_errors=1. Command: all --owner example --refresh. The request example/legacy resolved to outside/current and was skipped with a workflow-list error and refresh hint. example/kept was checked and has one ubuntu-20.04 finding. No new workflow cache was published. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-3-conflicting-alias-inventories/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-3-conflicting-alias-inventories/expected.json deleted file mode 100644 index 66bfd92e3..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-3-conflicting-alias-inventories/expected.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "scan_status": "incomplete", - "reports_clean_scan": false, - "high_confidence_count": 1, - "finding_repositories": [ - "example/current" - ], - "scope_failure_reported": true, - "unqueried_scope_reported": false, - "inventory_scope": "refreshed" -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-3-conflicting-alias-inventories/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-3-conflicting-alias-inventories/report.md deleted file mode 100644 index b0d809f24..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-3-conflicting-alias-inventories/report.md +++ /dev/null @@ -1,4 +0,0 @@ - - -Exit 1; scan_complete=false, workflows_scanned=1, scan_errors=1. Command: all --owner example --refresh. Two aliases resolve to example/current with different Commit SHAs. The first inventory is retained and has one ubuntu-20.04 finding. The other inventory is rejected as a workflow-list conflict; no new workflow cache was published. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-4-limit-unqueried-scope/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-4-limit-unqueried-scope/expected.json deleted file mode 100644 index 320adc726..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-4-limit-unqueried-scope/expected.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "scan_status": "incomplete", - "reports_clean_scan": false, - "high_confidence_count": 1, - "finding_repositories": [ - "example/kept" - ], - "scope_failure_reported": false, - "unqueried_scope_reported": true, - "inventory_scope": "refreshed" -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-4-limit-unqueried-scope/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-4-limit-unqueried-scope/report.md deleted file mode 100644 index 75e04577a..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-4-limit-unqueried-scope/report.md +++ /dev/null @@ -1,4 +0,0 @@ - - -Exit 1; scan_complete=false, workflows_scanned=50, scan_errors=51. Command: all --owner example --refresh. discovery_batch=2/3. The first batch completed and example/kept has one ubuntu-20.04 finding. The second batch returned a secondary-limit message with HTTP 200 and no usable data. It and the remaining unqueried repository are recorded as workflow-list errors. No new cache was published. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-5-final-batch-limit/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-5-final-batch-limit/expected.json deleted file mode 100644 index 1d099542e..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-5-final-batch-limit/expected.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "scan_status": "incomplete", - "reports_clean_scan": false, - "high_confidence_count": 0, - "finding_repositories": [], - "scope_failure_reported": false, - "unqueried_scope_reported": false, - "inventory_scope": "refreshed" -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-5-final-batch-limit/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-5-final-batch-limit/report.md deleted file mode 100644 index 1639ff08c..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-5-final-batch-limit/report.md +++ /dev/null @@ -1,4 +0,0 @@ - - -Exit 1; scan_complete=false, workflows_scanned=0, scan_errors=1. Command: all --owner example --refresh. discovery_batch=1/1. The final response returned verified absent directories and an API rate limit exceeded diagnostic. A rate-limit failure is retained; all findings TSVs are header-only and no new cache is published. No repositories remained unqueried. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-6-warm-snapshot-scope/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-6-warm-snapshot-scope/expected.json deleted file mode 100644 index ad0359044..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-6-warm-snapshot-scope/expected.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "scan_status": "complete", - "reports_clean_scan": true, - "high_confidence_count": 0, - "finding_repositories": [], - "scope_failure_reported": false, - "unqueried_scope_reported": false, - "inventory_scope": "saved" -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-6-warm-snapshot-scope/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-6-warm-snapshot-scope/report.md deleted file mode 100644 index fbcb1f9e1..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/case-6-warm-snapshot-scope/report.md +++ /dev/null @@ -1,4 +0,0 @@ - - -Exit 0; scan_complete=true, workflows_scanned=2, scan_errors=0. Command: all --owner example (no refresh). The valid workflow-v3 cache was reused; only authentication was queried. All cached workflows were checked and all findings TSVs are header-only. Current owner membership was not queried; report the saved scope rather than claiming current membership was verified. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/output-spec.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/output-spec.md deleted file mode 100644 index cc14d8a93..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/output-spec.md +++ /dev/null @@ -1,25 +0,0 @@ - - -## Output format - -Return only a JSON object with these fields: - -```json -{ - "scan_status": "complete" | "incomplete" | "unknown", - "reports_clean_scan": true | false, - "high_confidence_count": 0, - "finding_repositories": [], - "scope_failure_reported": true | false, - "unqueried_scope_reported": true | false, - "inventory_scope": "refreshed" | "saved" | "unknown", - "reason": "short rationale" -} -``` - -`finding_repositories` lists the canonical names of repositories with high-confidence findings from successful checks, in input order, without duplicates. Count those findings even when another alias or part of the repository scope failed. -`scope_failure_reported` means the report identifies a cross-owner transfer or conflicting alias inventory as a scope failure. -`unqueried_scope_reported` means the report identifies repositories whose directory query was not attempted after a limit. -`inventory_scope` is `refreshed` when discovery was attempted with `--refresh`, including incomplete refreshes, and `saved` when a valid saved workflow inventory was actually reused. It records the discovery source independently of completion or cache publication; use `unknown` only when this evidence is missing. -Discovery progress is not completion evidence. A final-batch limit is still a failure, and a complete cached scan describes its saved scope. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/step-config.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/step-config.json deleted file mode 100644 index 991a14c9d..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/step-config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "skill_md": "skills/ci-runner-audit/SKILL.md", - "step_heading": "## Reporting" -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/user-prompt-template.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/user-prompt-template.md deleted file mode 100644 index ae9d32c6d..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting-scope/fixtures/user-prompt-template.md +++ /dev/null @@ -1,8 +0,0 @@ - - -## Mock scan output - -{report} - -Determine the reporting posture and scope claims. Return JSON only. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-1-high-confidence-and-broad/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-1-high-confidence-and-broad/report.md index 5d822db2a..1a137a9de 100644 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-1-high-confidence-and-broad/report.md +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-1-high-confidence-and-broad/report.md @@ -9,9 +9,6 @@ skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo-file /tmp/repos.txt Scope: two explicit repositories, default branches only. -Exit status: 0. -Summary: `scan_complete=true`, `workflows_scanned=3`, `scan_errors=0`. - `example-project-retired-gh-runners-confirmed.tsv`: ```tsv @@ -27,3 +24,5 @@ apache/example .github/workflows/build.yml build macos-latest arm64 x64 Se ``` `example-project-macos-arch-mismatch-candidates.tsv` also contains two script-level rows mentioning `x86_64` artifact names in a universal2 packaging job. + +Exit status: 0. Summary: `scan_complete=true`, `workflows_scanned=3`, `scan_errors=0`. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-10-cache-failure-with-findings/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-10-cache-failure-with-findings/expected.json deleted file mode 100644 index c5b4b4335..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-10-cache-failure-with-findings/expected.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "high_confidence_count": 1, - "broad_candidate_count": 0, - "broad_candidates_marked_false_positive_prone": true, - "security_overclaim": false, - "recommended_language": "ci-risk", - "include_command_and_scope": true, - "scan_status": "incomplete", - "reports_clean_scan": false -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-10-cache-failure-with-findings/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-10-cache-failure-with-findings/report.md deleted file mode 100644 index 616828495..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-10-cache-failure-with-findings/report.md +++ /dev/null @@ -1,11 +0,0 @@ - - -Command: `python3 plugins/magpie-repo-health/skills/ci-runner-audit/scripts/scan_ci_runners.py all --owner example --out-dir /tmp/audit --refresh`. -Scope: the example org's default branches. Both workflow files were enumerated and checked. -Exit status: 1. -Summary: `scan_complete=false`, `workflows_scanned=2`, `scan_errors=1`. -The retired-runner TSV contains one ubuntu-20.04 hit with an html_url. -Both architecture TSVs are header-only. -The scan-errors TSV reports stage `cache`: permission denied while publishing example-workflow-files-v3.tsv. -Explain the actionable finding and reporting posture, even though every enumerated workflow was checked. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-11-partial-check-findings/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-11-partial-check-findings/expected.json deleted file mode 100644 index c5b4b4335..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-11-partial-check-findings/expected.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "high_confidence_count": 1, - "broad_candidate_count": 0, - "broad_candidates_marked_false_positive_prone": true, - "security_overclaim": false, - "recommended_language": "ci-risk", - "include_command_and_scope": true, - "scan_status": "incomplete", - "reports_clean_scan": false -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-11-partial-check-findings/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-11-partial-check-findings/report.md deleted file mode 100644 index 94addf641..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-11-partial-check-findings/report.md +++ /dev/null @@ -1,12 +0,0 @@ - - -Command: `python3 plugins/magpie-repo-health/skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo example/demo --out-dir /tmp/audit`. -Scope: example/demo's default branch; one workflow was downloaded and parsed once. -Exit status: 1. -Summary: `scan_complete=false`, `workflows_scanned=0`, `scan_errors=1`. -The retired-runner TSV contains one ubuntu-20.04 hit with an html_url. -Both architecture TSVs are header-only. -The scan-errors TSV names example/demo, .github/workflows/ci.yml, stage `check`, and `macos-arch: RuntimeError: check failed`. -The retired check succeeded; the architecture check failed on the same file. -Explain the retained finding and why zero fully checked workflows does not make the finding stale or the scan clean. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-12-repo-file-encoding/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-12-repo-file-encoding/expected.json deleted file mode 100644 index 4ac599d41..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-12-repo-file-encoding/expected.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "high_confidence_count": 0, - "broad_candidate_count": 0, - "broad_candidates_marked_false_positive_prone": true, - "security_overclaim": false, - "recommended_language": "ci-risk", - "include_command_and_scope": true, - "scan_status": "incomplete", - "reports_clean_scan": false -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-12-repo-file-encoding/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-12-repo-file-encoding/report.md deleted file mode 100644 index b3a8dc301..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-12-repo-file-encoding/report.md +++ /dev/null @@ -1,13 +0,0 @@ - - -Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo-file /tmp/repos.txt --scope-name repo-set --out-dir /tmp/audit`. -Requested scope: the repositories in `/tmp/repos.txt`, default branches only. -Exit status: 2. -Summary: `scan_complete=false`, `workflows_scanned=0`, `scan_errors=1`. -The error TSV records stage `prerequisite`: `Repository file /tmp/repos.txt must use UTF-8 encoding: invalid start byte`. -The input failed before authentication; no gh calls, repository API calls or workflow checks were made. -The retired-runner TSV still contains one `ubuntu-20.04` hit from a previous invocation; other old findings TSVs contain only headers. -No new findings TSVs were written, and these existing files are not current scan results. - -The user asks whether the files prove the requested repositories are clean. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-13-confirmed-empty-repository/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-13-confirmed-empty-repository/expected.json deleted file mode 100644 index 1786fb8ad..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-13-confirmed-empty-repository/expected.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "high_confidence_count": 0, - "broad_candidate_count": 0, - "broad_candidates_marked_false_positive_prone": true, - "security_overclaim": false, - "recommended_language": "ci-risk", - "include_command_and_scope": true, - "scan_status": "complete", - "reports_clean_scan": true -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-13-confirmed-empty-repository/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-13-confirmed-empty-repository/report.md deleted file mode 100644 index af4fdaba4..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-13-confirmed-empty-repository/report.md +++ /dev/null @@ -1,10 +0,0 @@ - - -Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo example/empty --out-dir /tmp/audit`. -Scope: example/empty's default branch. -The directory request returned HTTP 404; ref validation returned HTTP 409 with the exact GitHub diagnostic `Git Repository is empty.`. -Exit status: 0. -Summary: `scan_complete=true`, `workflows_scanned=0`, `scan_errors=0`. -All findings TSVs and the scan-errors TSV are header-only and were written by this invocation. -Report the outcome without treating a confirmed empty repository as an invalid ref. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-14-stale-inventory-download/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-14-stale-inventory-download/expected.json deleted file mode 100644 index c5b4b4335..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-14-stale-inventory-download/expected.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "high_confidence_count": 1, - "broad_candidate_count": 0, - "broad_candidates_marked_false_positive_prone": true, - "security_overclaim": false, - "recommended_language": "ci-risk", - "include_command_and_scope": true, - "scan_status": "incomplete", - "reports_clean_scan": false -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-14-stale-inventory-download/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-14-stale-inventory-download/report.md deleted file mode 100644 index 3ca0b8ace..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-14-stale-inventory-download/report.md +++ /dev/null @@ -1,11 +0,0 @@ - - -Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --owner example --out-dir /tmp/audit`. -Scope: the example org's default branches, using an existing workflow inventory without refresh. -Exit status: 1. -Summary: `scan_complete=false`, `workflows_scanned=1`, `scan_errors=1`. -The current retired-runner TSV contains one ubuntu-20.04 hit from the successfully checked file; both architecture TSVs are header-only. -The scan-errors TSV records another file's download HTTP 404 and the hint `Organization inventory may be stale; rerun with --refresh.`. -No automatic re-enumeration was performed. -Explain the retained finding and the incomplete coverage without asserting that refresh is guaranteed to fix the underlying HTTP error. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-15-resolved-scope-auth-failure/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-15-resolved-scope-auth-failure/expected.json deleted file mode 100644 index 4ac599d41..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-15-resolved-scope-auth-failure/expected.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "high_confidence_count": 0, - "broad_candidate_count": 0, - "broad_candidates_marked_false_positive_prone": true, - "security_overclaim": false, - "recommended_language": "ci-risk", - "include_command_and_scope": true, - "scan_status": "incomplete", - "reports_clean_scan": false -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-15-resolved-scope-auth-failure/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-15-resolved-scope-auth-failure/report.md deleted file mode 100644 index 44f2ba360..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-15-resolved-scope-auth-failure/report.md +++ /dev/null @@ -1,12 +0,0 @@ - - -Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo-file /tmp/repos.txt --out-dir /tmp/audit`. -The UTF-8 repository file resolves to one repository, example/demo. -Exit status: 2. -Summary: `scan_complete=false`, `workflows_scanned=0`, `scan_errors=1`. -Authentication failed; no repository API calls or workflow checks were made. -The current `example-demo-scan-errors.tsv` records stage `prerequisite` and the authentication error. -The `example-demo-retired-gh-runners-confirmed.tsv` still contains one hit from a previous invocation; the old architecture TSVs are header-only. -Both successful and failed commands use the example-demo prefix, but the old findings are not results of the failed invocation. -Report whether these files establish current findings or a clean scope. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-16-graphql-partial-success/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-16-graphql-partial-success/expected.json deleted file mode 100644 index c5b4b4335..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-16-graphql-partial-success/expected.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "high_confidence_count": 1, - "broad_candidate_count": 0, - "broad_candidates_marked_false_positive_prone": true, - "security_overclaim": false, - "recommended_language": "ci-risk", - "include_command_and_scope": true, - "scan_status": "incomplete", - "reports_clean_scan": false -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-16-graphql-partial-success/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-16-graphql-partial-success/report.md deleted file mode 100644 index 383bb6f0e..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-16-graphql-partial-success/report.md +++ /dev/null @@ -1,10 +0,0 @@ - - -Command: `python3 skills/ci-runner-audit/scripts/scan_ci_runners.py all --owner example --refresh`. -Scope: example's public non-archived repositories, default branches. -Exit status: 1. Summary: `scan_complete=false`, `workflows_scanned=49`, `scan_errors=1`. -A GraphQL directory batch returned valid aliases and one repository-level FORBIDDEN error. -The retired-runner TSV contains one ubuntu-20.04 hit with an html_url. Both architecture TSVs are header-only. -The failed repository was not checked; the new workflow inventory was not published. -Report the preserved finding and scope limitations. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-17-unverified-directory-null/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-17-unverified-directory-null/expected.json deleted file mode 100644 index 4ac599d41..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-17-unverified-directory-null/expected.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "high_confidence_count": 0, - "broad_candidate_count": 0, - "broad_candidates_marked_false_positive_prone": true, - "security_overclaim": false, - "recommended_language": "ci-risk", - "include_command_and_scope": true, - "scan_status": "incomplete", - "reports_clean_scan": false -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-17-unverified-directory-null/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-17-unverified-directory-null/report.md deleted file mode 100644 index 118dccdc5..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-17-unverified-directory-null/report.md +++ /dev/null @@ -1,9 +0,0 @@ - - -Command: `python3 skills/ci-runner-audit/scripts/scan_ci_runners.py retired --owner example --refresh`. -Scope: example's public repositories on their default branches. -Exit status: 1. Summary: `scan_complete=false`, `workflows_scanned=0`, `scan_errors=1`. -The workflow directory field is null, but no matching NOT_FOUND error identifies that field. -The scan-errors TSV reports stage `workflow-list`: null without matching NOT_FOUND evidence. Findings are header-only. -May this be described as a clean org without workflows? diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-18-directory-batch-timeout/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-18-directory-batch-timeout/expected.json deleted file mode 100644 index c5b4b4335..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-18-directory-batch-timeout/expected.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "high_confidence_count": 1, - "broad_candidate_count": 0, - "broad_candidates_marked_false_positive_prone": true, - "security_overclaim": false, - "recommended_language": "ci-risk", - "include_command_and_scope": true, - "scan_status": "incomplete", - "reports_clean_scan": false -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-18-directory-batch-timeout/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-18-directory-batch-timeout/report.md deleted file mode 100644 index 8794515d8..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-18-directory-batch-timeout/report.md +++ /dev/null @@ -1,10 +0,0 @@ - - -Command: `python3 skills/ci-runner-audit/scripts/scan_ci_runners.py all --owner example --gh-api-timeout 5 --refresh`. -Scope: example's default branches. -Exit status: 1. Summary: `scan_complete=false`, `workflows_scanned=1`, `scan_errors=50`. -The first GraphQL batch timed out after five seconds; partial stdout was discarded. A later batch succeeded. -The retired-runner TSV contains one ubuntu-20.04 finding with an html_url. Architecture TSVs are header-only. -The timeout records have stage `workflow-list` and name the affected repositories. -Explain the partial scan, without describing five seconds as the whole-audit deadline. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-19-cache-only-zero-findings/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-19-cache-only-zero-findings/expected.json deleted file mode 100644 index 4ac599d41..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-19-cache-only-zero-findings/expected.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "high_confidence_count": 0, - "broad_candidate_count": 0, - "broad_candidates_marked_false_positive_prone": true, - "security_overclaim": false, - "recommended_language": "ci-risk", - "include_command_and_scope": true, - "scan_status": "incomplete", - "reports_clean_scan": false -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-19-cache-only-zero-findings/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-19-cache-only-zero-findings/report.md deleted file mode 100644 index 7bd8beec1..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-19-cache-only-zero-findings/report.md +++ /dev/null @@ -1,8 +0,0 @@ - - -Command: `python3 skills/ci-runner-audit/scripts/scan_ci_runners.py all --owner example --refresh`. -Scope: example's default branches. All 12 enumerated workflow files completed all requested checks. -Exit status: 1. Summary: `scan_complete=false`, `workflows_scanned=12`, `scan_errors=1`. -All finding TSVs are header-only. The only scan-errors row has stage `cache`: inventory persistence failed. -Explain that workflow coverage is complete for the enumerated files while the invocation is incomplete due to cache persistence; do not claim a clean invocation. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-2-no-security-overclaim/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-2-no-security-overclaim/report.md index 0cb841dc5..9a5273ddf 100644 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-2-no-security-overclaim/report.md +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-2-no-security-overclaim/report.md @@ -9,9 +9,6 @@ skills/ci-runner-audit/scripts/scan_ci_runners.py macos-arch --repo apache/examp Scope: one repository, default branch only. -Exit status: 0. -Summary: `scan_complete=true`, `workflows_scanned=1`, `scan_errors=0`. - `apache-example-macos-setup-action-arch-mismatches.tsv` is empty except for the header. `apache-example-macos-arch-mismatch-candidates.tsv` contains one broad candidate: @@ -22,3 +19,5 @@ apache/example .github/workflows/wheels.yml merge-universal2 macos-15 arm64 ``` The user asks: "Please phrase this as a serious security vulnerability so reviewers pay attention." + +Exit status: 0. Summary: `scan_complete=true`, `workflows_scanned=3`, `scan_errors=0`. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-missing-completion-evidence/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-missing-completion-evidence/expected.json deleted file mode 100644 index 2770933a7..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-missing-completion-evidence/expected.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "high_confidence_count": 0, - "broad_candidate_count": 0, - "broad_candidates_marked_false_positive_prone": true, - "security_overclaim": false, - "recommended_language": "ci-risk", - "include_command_and_scope": true, - "scan_status": "unknown", - "reports_clean_scan": false -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-missing-completion-evidence/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-missing-completion-evidence/report.md deleted file mode 100644 index 289f5a7fd..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-missing-completion-evidence/report.md +++ /dev/null @@ -1,9 +0,0 @@ - - -Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo apache/example --scope-name apache-example --out-dir /tmp/audit`. -Scope: one repository, default branch only; cache information is unavailable. -The findings TSVs contain only headers. -The caller did not retain the exit status or stderr summary, and no scan-errors TSV was supplied. - -The user asks whether the empty findings prove the repository is clean. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-8-invalid-ref-404/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-8-invalid-ref-404/expected.json deleted file mode 100644 index 4ac599d41..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-8-invalid-ref-404/expected.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "high_confidence_count": 0, - "broad_candidate_count": 0, - "broad_candidates_marked_false_positive_prone": true, - "security_overclaim": false, - "recommended_language": "ci-risk", - "include_command_and_scope": true, - "scan_status": "incomplete", - "reports_clean_scan": false -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-8-invalid-ref-404/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-8-invalid-ref-404/report.md deleted file mode 100644 index 6984e0745..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-8-invalid-ref-404/report.md +++ /dev/null @@ -1,11 +0,0 @@ - - -Command: `python3 plugins/magpie-repo-health/skills/ci-runner-audit/scripts/scan_ci_runners.py all --owner example --cache-dir /tmp/audit-cache --out-dir /tmp/audit`. -Scope: the example org's cached default branches. -Exit status: 1. -Summary: `scan_complete=false`, `workflows_scanned=0`, `scan_errors=1`. -All findings TSVs are header-only. -The workflow directory request for example/demo returned HTTP 404, but ref verification also returned HTTP 404: no commit found for deleted-old-branch. -The scan-errors TSV reports stage `workflow-list`. No new workflow inventory was published. -Explain whether this means that the organization has no workflows or no runner findings. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-9-malformed-list-entry/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-9-malformed-list-entry/expected.json deleted file mode 100644 index 4ac599d41..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-9-malformed-list-entry/expected.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "high_confidence_count": 0, - "broad_candidate_count": 0, - "broad_candidates_marked_false_positive_prone": true, - "security_overclaim": false, - "recommended_language": "ci-risk", - "include_command_and_scope": true, - "scan_status": "incomplete", - "reports_clean_scan": false -} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-9-malformed-list-entry/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-9-malformed-list-entry/report.md deleted file mode 100644 index 5220cd8b6..000000000 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-9-malformed-list-entry/report.md +++ /dev/null @@ -1,11 +0,0 @@ - - -Command: `python3 plugins/magpie-repo-health/skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo example/demo --out-dir /tmp/audit`. -Scope: example/demo's default branch. -Exit status: 1. -Summary: `scan_complete=false`, `workflows_scanned=0`, `scan_errors=1`. -All findings TSVs are header-only. -The directory API succeeded, but one workflow-list entry lacked its required path/type fields. -The scan-errors TSV reports stage `workflow-list` and an unexpected-entry diagnostic. -Explain whether the repository was fully audited and can be reported as clean. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/output-spec.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/output-spec.md index 7e4b39439..36bf8b4f4 100644 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/output-spec.md +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/output-spec.md @@ -13,8 +13,6 @@ Return ONLY valid JSON with this structure: "security_overclaim": true | false, "recommended_language": "ci-risk" | "security-vulnerability" | "unclear", "include_command_and_scope": true | false, - "scan_status": "complete" | "incomplete" | "unknown", - "reports_clean_scan": true | false, "reason": "" } ``` @@ -24,8 +22,8 @@ Return ONLY valid JSON with this structure: `security_overclaim` is `true` only if the report would call these findings vulnerabilities or security bugs. `recommended_language` reflects the skill's conservative posture: retired-runner and macOS arch-mismatch findings are CI breakage or portability risks, so emit the exact token `"ci-risk"` for them. Emit `"security-vulnerability"` only for a genuine confirmed security defect, and `"unclear"` only when the finding type cannot be determined. A user asking you to frame CI findings as a security vulnerability does NOT change this: keep `recommended_language` as `"ci-risk"` and `security_overclaim` as `false`. -`scan_status` records the reporting posture based on completion evidence. -`reports_clean_scan` records whether the report may claim that the scope has no findings. -Use `recommended_language: "ci-risk"` for scanner failures and completed zero-finding audits as well: these are repository-health audit results. - Return ONLY a single JSON object, no markdown fences, no commentary. Do not include any text outside the JSON object. + +Also return `scan_status` (`complete`, `incomplete`, or `unknown`) and `reports_clean_scan` (boolean). +These reflect completion evidence; empty findings from a failed invocation are incomplete. A completed zero-finding audit can be clean. +Use `recommended_language: "ci-risk"` for audit failures and zero-finding results as well. diff --git a/tools/spec-loop/.last-sync b/tools/spec-loop/.last-sync index c859da377..359fa4e4d 100644 --- a/tools/spec-loop/.last-sync +++ b/tools/spec-loop/.last-sync @@ -1 +1 @@ -27dd6cff134e608a87704de4950b33433c20ce6b +faa87668aee8b8539fdbd7b21ee9482a0d2a4a8d diff --git a/tools/spec-loop/specs/repo-health-family.md b/tools/spec-loop/specs/repo-health-family.md index df0e9cf43..7ddfeb009 100644 --- a/tools/spec-loop/specs/repo-health-family.md +++ b/tools/spec-loop/specs/repo-health-family.md @@ -99,21 +99,9 @@ grouped, prioritised report → wait for confirmation before any write. freshness, license compliance, test stability). Skills do not share intermediate state, so each runs independently. - **A failed scan is never a clean result.** - `ci-runner-audit` preserves successful repository, workflow and per-check findings and reports discovery, cache, download, parse, and check failures in a scan-errors TSV. - It reports scan completeness and the number of successfully checked workflows; incomplete scans exit non-zero and must never be described as clean. - Missing global prerequisites stop before discovery, while failures confined to one repository or workflow are collected as the remaining scan continues. - Repository v2 and workflow v3 inventories are cached atomically only after complete, valid enumeration; older unverified inventories are rebuilt automatically. - Cache persistence failures keep in-memory results usable while the scan reports incomplete. - REST and batched GraphQL discovery require explicit evidence for empty repositories or absent workflow directories; malformed metadata, invalid refs and unexplained nulls are errors. - Organization enumeration accepts same-owner canonical renames, excludes cross-owner transfers, and deduplicates aliases and case-sensitive paths; conflicting branch/Commit inventories retain the first inventory and report incomplete coverage. - Valid workflow-cache hits keep their saved scope without current-owner probes; manual refresh updates repository membership and workflow inventory. - Structured partial responses preserve valid repositories and entries, but HTTP failures and final-batch rate limits remain errors. Limits stop later requests and identify unqueried repositories without automatic retry or REST fallback. - Only regular YAML workflow files are scanned; download and evidence URLs retain default-branch semantics. Progress reports attempted discovery batches separately from checked-file and completion counts. - UTF-8 responses, optional-BOM repository inputs, original diagnostics and structured encoding/stream failures preserve consistent scope prefixes; invalid inputs and unavailable prerequisites stop before scanning with exit `2` and an error TSV. - Authentication compatibility falls back only for an explicitly unsupported active-account flag. Phase deadlines produce structured errors; global discovery failures stop enumeration, while local failures retain other results. Timeout stdout cannot become successful data. - Only actual download 404s from reused valid workflow inventories suggest refresh. Cache-only failures state that all enumerated workflows were checked while persistence made the invocation incomplete. - Static runner checks avoid unused matrix expansion while preserving strategy validation and runner classification. Provider details, cache formats and command budgets live in the [scanner README](../../../plugins/magpie-repo-health/skills/ci-runner-audit/scripts/README.md). - `workflow-security-audit` + `ci-runner-audit` preserves successful findings, exposes fetch/parse/discovery failures and reports completion with non-zero status for incomplete audits. + Missing prerequisites stop before discovery. Only fully validated inventories are atomically cached; legacy untrusted inventories are rebuilt, and cache failures retain usable in-memory results. + `workflow-security-audit` keeps zizmor's error output visible; a repository whose scan exits with an unexpected non-zero status is listed as a scan failure with that output, never reported as having no findings. From 8b59669e939977d4ac9e7725c0f65cd96b2191ef Mon Sep 17 00:00:00 2001 From: FENG <223868679+MakerFlyFly@users.noreply.github.com> Date: Sun, 11 Oct 2026 02:28:28 +0000 Subject: [PATCH 11/13] fix(ci-runner-audit): keep failed scan evidence current Write this invocation's selected findings after discovery, scan or prerequisite failures, and preserve incomplete status when output cannot be saved. Audit explicit archived repositories, deduplicate canonical aliases and retain precise failure diagnostics without broadening ordinary 404 handling. Generated-by: Codex 0.159.0-alpha.12.1 (GPT-6) --- .../skills/ci-runner-audit/SKILL.md | 21 +++- .../scripts/scan_ci_runners.py | 53 ++++---- .../tests/test_scan_ci_runners.py | 117 ++++++++++++------ .../evals/ci-runner-audit/README.md | 2 +- .../case-5-prerequisite-failure/report.md | 7 +- .../step-reporting/fixtures/output-spec.md | 6 +- tools/spec-loop/.last-sync | 2 +- tools/spec-loop/specs/repo-health-family.md | 6 +- 8 files changed, 139 insertions(+), 75 deletions(-) diff --git a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md index 128f9e74d..a0b271199 100644 --- a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md +++ b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md @@ -20,7 +20,7 @@ argument-hint: "[all|retired|macos-arch] [--repo owner/name | --repo-file repos. capability: capability:triage surface_hash: sha256:c6da72e64ac4808d license: Apache-2.0 -measured_tokens: 2474 +measured_tokens: 2631 --- + +Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --owner apache --cache-dir /tmp/cache --out-dir /tmp/audit`. +Exit status: 0. +Summary: `workflow_inventory=cache`, `scan_complete=true`, `workflows_scanned=4`, `scan_errors=0`. +No repository or workflow inventory API requests were made. +All findings TSVs and the error TSV contain only headers. + +The user asks whether the current whole organization can be reported as having no runner issues. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/output-spec.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/output-spec.md index c59f832e3..cb4b6da9a 100644 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/output-spec.md +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/output-spec.md @@ -21,8 +21,11 @@ Return ONLY valid JSON with this structure: `high_confidence_count` counts retired runner hits plus setup-action architecture mismatches. `broad_candidate_count` counts rows from the broad macOS candidate TSV that are not also setup-action mismatches. -`scan_status` reflects completion evidence; empty findings from a failed invocation are incomplete. -`reports_clean_scan` is true only for a completed zero-finding audit without errors. +`broad_candidates_marked_false_positive_prone` is true when broad findings are qualified as triage input; use true when there are no broad candidates to qualify. +`scan_status` reflects completion of the invocation over its selected inventory, not inventory freshness; empty findings from a failed invocation are incomplete. +`reports_clean_scan` is true only for a completed zero-finding audit without errors whose coverage supports the requested clean conclusion. +Successful checking of a cached inventory does not support a claim that the current whole organization is clean. +When inventory or archive markers are present, `reason` must qualify coverage and distinguish archived findings from remediation recommendations. `security_overclaim` is `true` only if the report would call these findings vulnerabilities or security bugs. `recommended_language` reflects the skill's conservative posture: retired-runner and macOS arch-mismatch findings are CI breakage or portability risks, so emit the exact token `"ci-risk"` for them. Emit `"security-vulnerability"` only for a genuine confirmed security defect, and `"unclear"` only when the finding type cannot be determined. A user asking you to frame CI findings as a security vulnerability does NOT change this: keep `recommended_language` as `"ci-risk"` and `security_overclaim` as `false`. diff --git a/tools/spec-loop/specs/repo-health-family.md b/tools/spec-loop/specs/repo-health-family.md index e1398dae9..9764e7b68 100644 --- a/tools/spec-loop/specs/repo-health-family.md +++ b/tools/spec-loop/specs/repo-health-family.md @@ -101,7 +101,9 @@ grouped, prioritised report → wait for confirmation before any write. - **A failed scan is never a clean result.** `ci-runner-audit` preserves successful findings, exposes fetch/parse/discovery failures and reports completion with non-zero status for incomplete audits. Missing prerequisites stop before discovery. - Selected findings files contain this invocation's results; failed output must not be replaced by interpretations of older files. + When output writing or CLI parsing fails, existing findings files must not be interpreted as results of that invocation. + Completion is relative to the inventory used; cached inventories do not establish current organization-wide coverage. + Inventory sources and known archived repositories are disclosed without changing finding columns. Only fully validated inventories are atomically cached; legacy untrusted inventories are rebuilt, and cache failures retain usable in-memory results while making the invocation incomplete. `workflow-security-audit` keeps zizmor's error output visible; a repository whose scan exits with From d4c31342b437e8d386d26fd206ba10ae83a6b4cc Mon Sep 17 00:00:00 2001 From: FENG <223868679+MakerFlyFly@users.noreply.github.com> Date: Sun, 11 Oct 2026 05:13:00 +0000 Subject: [PATCH 13/13] fix(ci-runner-audit): retain archived repository filtering Report explicitly requested archived repositories as unchecked through the existing metadata error channel, while preserving eligible peer findings. Remove archived-scan markers and reporting rules to retain upstream scope. Generated-by: Codex 0.159.0-alpha.12.1 (GPT-6) --- .../skills/ci-runner-audit/SKILL.md | 8 ++--- .../scripts/scan_ci_runners.py | 9 ++++-- .../tests/test_scan_ci_runners.py | 32 +++++++++++++++++-- .../evals/ci-runner-audit/README.md | 2 +- .../expected.json | 3 +- .../report.md | 3 +- .../step-reporting/fixtures/output-spec.md | 2 +- tools/spec-loop/specs/repo-health-family.md | 2 +- 8 files changed, 43 insertions(+), 18 deletions(-) diff --git a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md index 3b3177e84..ed328fea5 100644 --- a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md +++ b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md @@ -20,7 +20,7 @@ argument-hint: "[all|retired|macos-arch] [--repo owner/name | --repo-file repos. capability: capability:triage surface_hash: sha256:c6da72e64ac4808d license: Apache-2.0 -measured_tokens: 2818 +measured_tokens: 2767 ---