From 5a0e5e59b70ff76a4239396eae5d322359407026 Mon Sep 17 00:00:00 2001 From: Remi Antczak <140408771+remigiusz-antczak@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:59:54 +0100 Subject: [PATCH] Wave 10: CI jobs a diff fires, repo-settings audit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delivery (refs #1559): - ci_cost_advisor.py ROOT --diff BASE lists every pull_request workflow job the committed diff BASE...HEAD fires, with its run: lines, so a fix lane knows which CI jobs to run locally (or name "left to CI") before it pushes to a repo it does not own. It applies the cost model's types, branches, paths and paths-ignore rules; branches filters are matched against the branch BASE names, never against --default-branch, and a sha or tag BASE lists each branch-gated job tagged "(branches: unchecked)" instead of dropping it. A workflow the parser cannot read but that names pull_request is listed as UNPARSED, and one line names the workflows that start only on push, merge_group or workflow_run. Jobs that build images or start containers (including services: and container:) are tagged [runtime]. Read-only and offline. - A segment-leading **/ in a paths filter now also matches no directory, as GitHub's filter cheat sheet specifies, so **/Dockerfile matches a root Dockerfile. The fix is in the glob matcher shared with the cost model. - scripts/test_ci_cost_advisor.py no longer leaves a temp dir behind on every run; NoTempLeak re-runs the file in a child with a private TMPDIR. Security (repository settings): - agentic-delivery/scripts/repo_settings_audit.py: a read-only audit of the GitHub settings a diff review cannot see (default-branch protection and rulesets together, the Actions token default, review approval and allowed actions, collaborator counts, write deploy keys, insecure webhooks, unprotected environments, secret scanning and push protection). A 403 or 404, a reply missing a field the check reads, an unreadable list item and a list of 1000 or more are "not checked" rows, never clean; a 404 "Branch not protected" is a finding only when the reply shows admin for the token. Counts and booleans only: no login, email, key title, hook URL or API message text is printed. Routed from agentic-delivery/SKILL.md and references/tool-hub.md (legitify and OpenSSF Scorecard named as the OSS options); sources and fetch dates in docs/standards-index.md. - scripts/test_skill_index_split.py: the script-lookup budget grows by 300 bytes for the new index row (measured 22162). size-budget-raise: .claude/skills/deep-code-review/references/parallel-audit.md 39795→40669 one routed bullet: run the CI jobs a fix diff fires, or name them left to CI size-budget-raise: .claude/skills/agentic-delivery/SKILL.md 7202→7457 one route row for repo_settings_audit.py size-budget-raise: .claude/skills/deep-code-review/references/tool-hub.md 14455→15368 one Repo settings section holding the repo_settings_audit.py row and its OSS options Tests: 20/20 affected suites pass, run one at a time (19 on the tip before the budget bump, then test_skill_index_split 7/7 after it; repo_settings_audit 55/55, ci_cost_advisor 80 with 1 skip); test-ci-gates.sh left to CI. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/INDEX.md | 1 + .../skills/agentic-delivery/INDEX-scripts.md | 1 + .claude/skills/agentic-delivery/INDEX.md | 2 +- .claude/skills/agentic-delivery/SKILL.md | 1 + .../scripts/repo_settings_audit.py | 328 ++++++++++ .claude/skills/deep-code-review/INDEX.md | 4 +- .../references/parallel-audit.md | 10 + .../deep-code-review/references/tool-hub.md | 6 + .../scripts/ci_cost_advisor.py | 173 +++++- SHA256SUMS | 15 +- changelog.d/ci-jobs-for-diff.md | 8 + changelog.d/repo-settings-audit.md | 6 + docs/INDEX.md | 2 +- docs/standards-index.md | 23 + scripts/ci-tests.d/repo-settings-audit.sh | 2 + scripts/size-budgets.tsv | 12 +- scripts/test_ci_cost_advisor.py | 319 +++++++++- scripts/test_repo_settings_audit.py | 575 ++++++++++++++++++ scripts/test_skill_index_split.py | 2 +- 19 files changed, 1450 insertions(+), 40 deletions(-) create mode 100644 .claude/skills/agentic-delivery/scripts/repo_settings_audit.py create mode 100644 changelog.d/ci-jobs-for-diff.md create mode 100644 changelog.d/repo-settings-audit.md create mode 100644 scripts/ci-tests.d/repo-settings-audit.sh create mode 100644 scripts/test_repo_settings_audit.py diff --git a/.claude/skills/INDEX.md b/.claude/skills/INDEX.md index bbeaf776d..a88c3b0cf 100644 --- a/.claude/skills/INDEX.md +++ b/.claude/skills/INDEX.md @@ -86,6 +86,7 @@ Run `python3 /scripts/` (`bash` for `.sh`). `name: purpose — argu - queue_guard.py: single priority queue: print the next issue to pull, flag P0… — --issues issues.json --priorities… - reap_own.sh: kill your own dev servers older than MAX_AGE_S whose cwd is… - refix_gate.py: re-touching a recently-fixed file requires a class-level… — --base REF --head REF [--window-hours… +- repo_settings_audit.py: read-only audit of the GitHub settings that decide who can… — OWNER/NAME [OWNER/NAME ...]… - review_digest.py: a returning reviewer's change digest, not a full re-read. — --since SHA [--head REF] [--paths PATH… - review_gate.py: pre-merge gate: every PR needs an independent deep-code-review… — check --head SHA [--repo R] - review_status.py: post a review verdict as commit status perun/review on an… — --sha <40-hex> --red N [--repo… diff --git a/.claude/skills/agentic-delivery/INDEX-scripts.md b/.claude/skills/agentic-delivery/INDEX-scripts.md index f5d05313e..b198341f7 100644 --- a/.claude/skills/agentic-delivery/INDEX-scripts.md +++ b/.claude/skills/agentic-delivery/INDEX-scripts.md @@ -83,6 +83,7 @@ Generated by `scripts/skill_index.py` (do not hand-edit). The script half of `IN | `scripts/refix_gate.py` | re-touching a recently-fixed file requires a class-level artifact. | refix_gate.py --base REF --head REF [--window-hours 72] [--class-glob GLOB ...] refix_gate.py --selftest | | `scripts/repeat_poll_guard.py` | PreToolUse(Bash) hook: nudge when the same command runs more than 3 times in one session. | — | | `scripts/reply_cap.py` | owner rule: chat replies max 3 lines, 500 chars. | — | +| `scripts/repo_settings_audit.py` | repo_settings_audit.py: read-only audit of the GitHub settings that decide who can change a reposit… | repo_settings_audit.py OWNER/NAME [OWNER/NAME ...] [--repos-file FILE] [--json] --repos-file holds one OWNER/NAME per l… | | `scripts/restore_ledger.py` | classify every branch after a VCS history-rewrite incident. | — | | `scripts/review_digest.py` | a returning reviewer's change digest, not a full re-read. | review_digest.py --since SHA [--head REF] [--paths PATH ...] [--max-lines N] [--allow-diverged] review_digest.py --self… | | `scripts/review_gate.py` | pre-merge gate: every PR needs an independent deep-code-review DIFF pass plus an evil-twin check. | review_gate.py check --head SHA [--repo R] | diff --git a/.claude/skills/agentic-delivery/INDEX.md b/.claude/skills/agentic-delivery/INDEX.md index c94f4c03b..801f10afd 100644 --- a/.claude/skills/agentic-delivery/INDEX.md +++ b/.claude/skills/agentic-delivery/INDEX.md @@ -38,4 +38,4 @@ Generated by `scripts/skill_index.py` (do not hand-edit; rerun it after changing ## Scripts -108 scripts. Read `INDEX-scripts.md` when you need a script (a trigger and CLI per script). `../INDEX.md`, `### agentic-delivery`, has only each one-line purpose and arguments. +109 scripts. Read `INDEX-scripts.md` when you need a script (a trigger and CLI per script). `../INDEX.md`, `### agentic-delivery`, has only each one-line purpose and arguments. diff --git a/.claude/skills/agentic-delivery/SKILL.md b/.claude/skills/agentic-delivery/SKILL.md index 9c3ac6964..e35440c64 100644 --- a/.claude/skills/agentic-delivery/SKILL.md +++ b/.claude/skills/agentic-delivery/SKILL.md @@ -24,6 +24,7 @@ Gated-delivery **pattern**, not a runtime or standing swarm. Does not replace `d | Verify or relay a result; slow pipeline | `verification-handback.md`; `pipeline-optimizer.md` | | Long session, compaction, new session, memory or docs bloat | `references/focus-kit.md`: update `.claude/handoff.md` (`scripts/focus_kit.py handoff set`) at each gate, before a context-heavy step, before handback | | New machine or fleet | `operating-discipline.md` | +| Audit a repo's GitHub settings (branch protection, Actions token, deploy keys, webhooks) | run `python3 scripts/repo_settings_audit.py OWNER/NAME... [--repos-file FILE] [--json]`: read-only, a 403 or 404 is `not checked`; admin token for the full set | | PR state; hand off; changelog | `scripts/pr_state.py `; `scripts/land_ready.py` (open PRs ready to land, or why not); `scripts/review_status.py --sha S --red N` (post `perun/review`); `scripts/premise_check.py ...`; `scripts/ci-gates.sh fragments` | | Cost cut, lane cap; "enforced" claim, CI cost | `cost-quality-guardrails.md`; `host-enforcement.md` | | Fan-out size, host probe | `fanout-host-sizing.md`, `fast-agentic-delivery.md` | diff --git a/.claude/skills/agentic-delivery/scripts/repo_settings_audit.py b/.claude/skills/agentic-delivery/scripts/repo_settings_audit.py new file mode 100644 index 000000000..c60c72f52 --- /dev/null +++ b/.claude/skills/agentic-delivery/scripts/repo_settings_audit.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +"""repo_settings_audit.py: read-only audit of the GitHub settings that decide who can change a repository and what its CI may do. + +WHY: a clean code review says nothing about a default branch anyone can force-push, a workflow token that can write, +a deploy key that can push, or a webhook that skips TLS verification. Those live in repository settings, not in the +diff, and they decide whether a review gate is worth anything. This lists them per repository, one row each: repo, +check, severity, one-line fix. It reuses `pipeline_hub.gh_get` (bare `gh api` GET, with `gh_rest.fallback` for the +macOS-sandbox TLS failure) and adds no analysis engine. + +Checks, per repository: + default branch classic branch protection and rulesets (`/rules/branches/{branch}`) together: no protection, + force pushes allowed, fewer than 1 required approving review, no required status checks, admins + exempt (classic `enforce_admins`) + Actions default workflow token has write, workflows can approve pull request reviews, allowed actions = all + people collaborator counts: admins, outside collaborators + deploy keys keys with write access (count) + webhooks TLS verification off (`insecure_ssl`), plain http delivery (counts) + environments environments with no protection rule (count) + security secret scanning and push protection, from `security_and_analysis` (a private or internal + organization repository needs GitHub Secret Protection for either; the fix text says so) + +A call the token cannot make (HTTP 403 or 404) is a `not checked (needs admin)` row, never a clean result. So is a +reply that lacks a field the check reads, and a list of 1000 items or more (see Limits). A list holding one item the +audit cannot read is `not checked` as a whole, since a count of it would be wrong. One reading is a finding on its +own: a 404 `Branch not protected` on a branch no ruleset covers means the branch has no protection, but only when +the repository reply shows admin for the token (`permissions.admin` true). The branch-protection REST page does not +say what a token without admin gets for a protected branch, so with any other token (admin false, or the field +missing) that 404 is `not checked`. A weakness inside the protection (force pushes, reviews, checks) is reported +only when classic protection and rulesets were both readable, since either can fill what the other lacks. Not read: +ruleset bypass lists (a `not checked` row says so whenever rulesets apply). + +USAGE: repo_settings_audit.py OWNER/NAME [OWNER/NAME ...] [--repos-file FILE] [--json] +`--repos-file` holds one OWNER/NAME per line (blank lines and `#` comments ignored). A repository twice, in any case, +is audited once. `--json` prints `{"rows": [{repo, check, severity, fix}], "summary": {repos, findings, not_checked}}`. +Severity is high, medium, low, info (a count with nothing to fix) or not-checked. + +Exit codes: 0 the audit ran (findings do not change it), 2 usage: a bad slug, no repository, an unreadable +--repos-file, or `gh` not on PATH. +Side effects: `gh api` GET calls only (about a dozen per repository); nothing is written or changed. +Limits: counts and booleans only. No login, email, key title or hook URL is printed, and no API message text, only +fixed phrases. A list is read 100 items a page for up to 10 pages; one that fills all ten (1000 items or more) is +`not checked`, not a truncated count. An archived repository is audited like any other. The full set needs an admin +token (admin on the repository, plus an org owner or security manager for `security_and_analysis`); with less, the +rows that need it say so. +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import sys +import urllib.parse + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import pipeline_profile as pp # noqa: E402 +from pipeline_hub import gh_get # noqa: E402 + +SLUG = re.compile(r"[A-Za-z0-9][A-Za-z0-9-]*/(?!\.{1,2}$)[A-Za-z0-9_.-]+") # no `..` or `.`: the slug goes into a URL path +SEVERITY = ("high", "medium", "low", "info", "not-checked") +FINDING = SEVERITY[:3] +MAX_PAGES = 10 # ponytail: 1000 items per list; a list that fills every page is `not checked`, raise if one ever matters +NEEDS_ADMIN = ("needs admin", "re-run with a token that has admin access to the repository") +NOT_FOUND = ("not found or no access", "check the OWNER/NAME slug and that the token can see the repository") +NOT_REPORTED = ("not reported by the API", "check it in the repository settings, or re-run with a token that has admin access to the repository") +UNEXPECTED, TOO_MANY = "unexpected reply", "too many items" # error texts `why` turns into fixed `not checked` rows + + +def dig(d, *keys): + """d[k1][k2]... or None as soon as a step is missing or not a dict (a surprising reply never raises).""" + for k in keys: + if not isinstance(d, dict): + return None + d = d.get(k) + return d + + +def has(d, need=()): + """True when d is an object carrying every key path in `need` with a non-null value (a missing field is never + read as a clean one).""" + return isinstance(d, dict) and all(dig(d, *p) is not None for p in need) + + +def read(get, path, kind, params=None, need=()): + """(reply, None) or (None, error): one GET through `get`. A reply that is not a `kind` (dict or list), a dict + missing a key path of `need`, a non-JSON reply and an API failure (OSError from `gh_get`) are all errors, so a + caller never has to type-check.""" + try: + d = get(path, params) if params else get(path) + if not isinstance(d, kind) or (kind is dict and not has(d, need)): + raise ValueError(UNEXPECTED) + return d, None + except (OSError, ValueError) as e: + return None, e + + +def listing(get, path, params=None, key=None, need=()): + """(items, None) or (None, error): every item of a list endpoint, 100 per page, at most MAX_PAGES pages. `key` + names the array inside an object reply (the environments list); without it the reply is the array. Every item + must be an object carrying the key paths in `need`, and a list that fills all MAX_PAGES pages is an error too (it + may be longer than what was read): a count of a list the audit could not read in full would be wrong.""" + items = [] + for page in range(1, MAX_PAGES + 1): + d, e = read(get, path, dict if key else list, dict(params or {}, per_page=100, page=page)) + chunk = d.get(key) if key and d is not None else d + if e or not isinstance(chunk, list) or not all(has(i, need) for i in chunk): + return None, e or ValueError(UNEXPECTED) + items += chunk + if len(chunk) < 100: + return items, None + return None, ValueError(TOO_MANY) + + +def why(e, miss=NEEDS_ADMIN): + """(label, fix) for a failed read, from the HTTP status only. 403 or 404 is `miss`; a rate limit and a plan limit + are told apart by their message (matched, never printed); an odd reply and a list too long to read in full have + their own labels; anything else is the status or `API error`.""" + if str(e) == UNEXPECTED: + return NOT_REPORTED + if str(e) == TOO_MANY: + return "%d or more items" % (MAX_PAGES * 100), "review this list in the repository settings" + m = re.search(r"\(HTTP (\d+)\)$", str(e)) + s, msg = int(m.group(1)) if m else None, str(e).lower() + if s in (403, 429) and "rate limit" in msg: + return "rate limited", "wait for the limit to reset, then re-run" + if s == 403 and "upgrade to github" in msg: + return "not available on this plan", "upgrade the plan or make the repository public, then re-run" + if s in (403, 404): + return miss + return ("HTTP %d" % s if s else "API error"), "re-run; if it persists check `gh auth status` and the network" + + +def nc(check, reason): + """The row for a check that could not run: `: not checked (