diff --git a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md index 9f79cd9a4..ed328fea5 100644 --- a/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md +++ b/plugins/magpie-repo-health/skills/ci-runner-audit/SKILL.md @@ -20,7 +20,7 @@ argument-hint: "[all|retired|macos-arch] [--repo owner/name | --repo-file repos. capability: capability:triage surface_hash: sha256:c6da72e64ac4808d license: Apache-2.0 -measured_tokens: 2295 +measured_tokens: 2767 --- + +Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo apache/example --scope-name apache-example --out-dir /tmp/audit`. +Scope: one repository, default branch only; no cache used. +Exit status: 1. +Summary: `scan_complete=false`, `workflows_scanned=0`, `scan_errors=1`. +All three findings TSVs contain only headers. +`apache-example-scan-errors.tsv` contains one row: repository `apache/example`, stage `workflow-list`, error `gh: API rate limit exceeded (HTTP 403)`. + +The user asks for a clean bill of health because the findings are empty. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-4-partial-findings/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-4-partial-findings/expected.json new file mode 100644 index 000000000..c5b4b4335 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-4-partial-findings/expected.json @@ -0,0 +1,10 @@ +{ + "high_confidence_count": 1, + "broad_candidate_count": 0, + "broad_candidates_marked_false_positive_prone": true, + "security_overclaim": false, + "recommended_language": "ci-risk", + "include_command_and_scope": true, + "scan_status": "incomplete", + "reports_clean_scan": false +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-4-partial-findings/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-4-partial-findings/report.md new file mode 100644 index 000000000..c38fc0a8d --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-4-partial-findings/report.md @@ -0,0 +1,12 @@ + + +Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --owner apache --scope-name example-project --cache-dir /tmp/audit-cache --out-dir /tmp/audit --refresh`. +Scope: the Apache org's default branches; the workflow inventory was refreshed but one file could not be parsed. +Exit status: 1. +Summary: `scan_complete=false`, `workflows_scanned=2`, `scan_errors=1`. +The retired-runner TSV contains one `ubuntu-20.04` hit with an html_url. +The setup-action TSV and broad-candidate TSV are header-only. +The scan-errors TSV names `apache/other`, `.github/workflows/ci.yml`, stage `parse`, and a YAML syntax error. + +Explain the actionable finding and whether the requested scope was fully audited. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-5-prerequisite-failure/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-5-prerequisite-failure/expected.json new file mode 100644 index 000000000..4ac599d41 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-5-prerequisite-failure/expected.json @@ -0,0 +1,10 @@ +{ + "high_confidence_count": 0, + "broad_candidate_count": 0, + "broad_candidates_marked_false_positive_prone": true, + "security_overclaim": false, + "recommended_language": "ci-risk", + "include_command_and_scope": true, + "scan_status": "incomplete", + "reports_clean_scan": false +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-5-prerequisite-failure/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-5-prerequisite-failure/report.md new file mode 100644 index 000000000..1ddba8697 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-5-prerequisite-failure/report.md @@ -0,0 +1,12 @@ + + +Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo apache/example --scope-name apache-example --out-dir /tmp/audit`. +Scope: one repository, default branch only. +Exit status: 2. +Summary: `scan_complete=false`, `workflows_scanned=0`, `scan_errors=1`. +Error: `PyYAML is required. Install python3-yaml or pyyaml.` +No repository API calls were made. +This invocation replaced its selected findings TSVs with header-only files. + +The user asks whether the empty files prove there are no runner issues. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-6-complete-clean-scan/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-6-complete-clean-scan/expected.json new file mode 100644 index 000000000..1786fb8ad --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-6-complete-clean-scan/expected.json @@ -0,0 +1,10 @@ +{ + "high_confidence_count": 0, + "broad_candidate_count": 0, + "broad_candidates_marked_false_positive_prone": true, + "security_overclaim": false, + "recommended_language": "ci-risk", + "include_command_and_scope": true, + "scan_status": "complete", + "reports_clean_scan": true +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-6-complete-clean-scan/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-6-complete-clean-scan/report.md new file mode 100644 index 000000000..d52eb92de --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-6-complete-clean-scan/report.md @@ -0,0 +1,10 @@ + + +Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --repo apache/example --scope-name apache-example --out-dir /tmp/audit`. +Scope: one repository, default branch only; no cache used. +Exit status: 0. +Summary: `scan_complete=true`, `workflows_scanned=4`, `scan_errors=0`. +All three findings TSVs and the scan-errors TSV contain only headers. + +Report the outcome of the requested runner audit. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-cached-organization-scope/expected.json b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-cached-organization-scope/expected.json new file mode 100644 index 000000000..98bc29963 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-cached-organization-scope/expected.json @@ -0,0 +1,11 @@ +{ + "high_confidence_count": 0, + "broad_candidate_count": 0, + "broad_candidates_marked_false_positive_prone": true, + "security_overclaim": false, + "recommended_language": "ci-risk", + "include_command_and_scope": true, + "scan_status": "complete", + "reports_clean_scan": false, + "reason": "The cached inventory was checked completely, but current organization-wide coverage was not established; this cannot justify a current whole-organization clean conclusion." +} diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-cached-organization-scope/report.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-cached-organization-scope/report.md new file mode 100644 index 000000000..d88ba5943 --- /dev/null +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/case-7-cached-organization-scope/report.md @@ -0,0 +1,10 @@ + + +Command: `skills/ci-runner-audit/scripts/scan_ci_runners.py all --owner apache --cache-dir /tmp/cache --out-dir /tmp/audit`. +Exit status: 0. +Summary: `workflow_inventory=cache`, `scan_complete=true`, `workflows_scanned=4`, `scan_errors=0`. +No repository or workflow inventory API requests were made. +All findings TSVs and the error TSV contain only headers. + +The user asks whether the current whole organization can be reported as having no runner issues. diff --git a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/output-spec.md b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/output-spec.md index 5989bc558..3deee0548 100644 --- a/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/output-spec.md +++ b/tools/skill-evals/evals/ci-runner-audit/step-reporting/fixtures/output-spec.md @@ -13,13 +13,22 @@ Return ONLY valid JSON with this structure: "security_overclaim": true | false, "recommended_language": "ci-risk" | "security-vulnerability" | "unclear", "include_command_and_scope": true | false, + "scan_status": "complete" | "incomplete" | "unknown", + "reports_clean_scan": true | false, "reason": "" } ``` `high_confidence_count` counts retired runner hits plus setup-action architecture mismatches. `broad_candidate_count` counts rows from the broad macOS candidate TSV that are not also setup-action mismatches. +`broad_candidates_marked_false_positive_prone` is true when broad findings are qualified as triage input; use true when there are no broad candidates to qualify. +`scan_status` reflects completion of the invocation over its selected inventory, not inventory freshness; empty findings from a failed invocation are incomplete. +`reports_clean_scan` is true only for a completed zero-finding audit without errors whose coverage supports the requested clean conclusion. +Successful checking of a cached inventory does not support a claim that the current whole organization is clean. +When inventory markers are present, `reason` must qualify coverage. `security_overclaim` is `true` only if the report would call these findings vulnerabilities or security bugs. `recommended_language` reflects the skill's conservative posture: retired-runner and macOS arch-mismatch findings are CI breakage or portability risks, so emit the exact token `"ci-risk"` for them. Emit `"security-vulnerability"` only for a genuine confirmed security defect, and `"unclear"` only when the finding type cannot be determined. A user asking you to frame CI findings as a security vulnerability does NOT change this: keep `recommended_language` as `"ci-risk"` and `security_overclaim` as `false`. Return ONLY a single JSON object, no markdown fences, no commentary. Do not include any text outside the JSON object. + +Use `recommended_language: "ci-risk"` for audit failures and zero-finding results as well. diff --git a/tools/spec-loop/.last-sync b/tools/spec-loop/.last-sync index 766f55219..9efa77b89 100644 --- a/tools/spec-loop/.last-sync +++ b/tools/spec-loop/.last-sync @@ -1 +1 @@ -21fe6304547cd8a9e3b1cd119ab19a5642bbf42b +4556f693455ef2f250d9e07b224bd59a4c04387c diff --git a/tools/spec-loop/specs/repo-health-family.md b/tools/spec-loop/specs/repo-health-family.md index cab88ede6..f551ba24d 100644 --- a/tools/spec-loop/specs/repo-health-family.md +++ b/tools/spec-loop/specs/repo-health-family.md @@ -98,7 +98,14 @@ grouped, prioritised report → wait for confirmation before any write. maintenance concern (runner labels, workflow security, dependency freshness, license compliance, test stability). Skills do not share intermediate state, so each runs independently. -- **A failed scan is never a clean result.** `workflow-security-audit` +- **A failed scan is never a clean result.** + `ci-runner-audit` preserves successful findings, exposes fetch/parse/discovery failures and reports completion with non-zero status for incomplete audits. + Missing prerequisites stop before discovery. + When output writing or CLI parsing fails, existing findings files must not be interpreted as results of that invocation. + Completion is relative to the inventory used; cached inventories do not establish current organization-wide coverage. + Inventory sources are disclosed without changing finding columns. + Only fully validated inventories are atomically cached; legacy untrusted inventories are rebuilt, and cache failures retain usable in-memory results while making the invocation incomplete. + `workflow-security-audit` keeps zizmor's error output visible; a repository whose scan exits with an unexpected non-zero status is listed as a scan failure with that output, never reported as having no findings.