diff --git a/plugins/magpie-release-management/skills/keys-sync/scripts/check_key.py b/plugins/magpie-release-management/skills/keys-sync/scripts/check_key.py index c5dcb5520..3ba9e44f5 100755 --- a/plugins/magpie-release-management/skills/keys-sync/scripts/check_key.py +++ b/plugins/magpie-release-management/skills/keys-sync/scripts/check_key.py @@ -28,7 +28,13 @@ Prints the Step 1 JSON: algorithm / size floor (RSA and DSA >= 2048 bits, EdDSA and ECDSA on an accepted curve; secp256k1 is refused), the DSA advisory, and the advisory for an expiry within 90 days. A key that has -already expired is a blocker, like a sub-floor key. +already expired is a blocker, like a sub-floor key. Every usable signing +subkey must meet the same floor, and at least one usable signing key or +subkey must meet it. Retired signing subkeys do not prevent key rotation. +GPG's expired validity is authoritative even on the expiry date. ``--today`` +overrides only calendar-date checks and advisories, not GPG's clock or +computed validity. Historical replay needs a colon listing captured at +the intended instant; an earlier date cannot reactivate an expired key. KEYS-directory mode (Step 2):: @@ -59,6 +65,7 @@ ECDSA_CURVES = {"nistp256", "nistp384", "nistp521", "brainpoolp256r1", "brainpoolp384r1", "brainpoolp512r1"} FLOOR_BITS = 2048 UNUSABLE_VALIDITY = { + "e": "key is expired (gpg marked it expired); extend its expiry or generate a new key", "r": "key is revoked; generate a new key", "i": "key is invalid (gpg could not validate it); fix or replace the key", "d": "key is disabled", @@ -78,8 +85,22 @@ def _date(epoch: str) -> str | None: return dt.datetime.fromtimestamp(int(epoch), dt.UTC).date().isoformat() +def _key_fields(f: list[str]) -> dict[str, Any]: + """Fields shared by primary keys and subkeys in a colon listing.""" + return { + "validity": f[1], + "algo_id": f[3], + "bits": int(f[2]) if f[2].isdigit() else None, + "keyid": f[4], + "created": _date(f[5]), + "expiry": _date(f[6]), + "capabilities": f[11], + "curve": f[16].lower(), + } + + def parse_colons(text: str) -> list[dict[str, Any]]: - """Primary keys from ``gpg --with-colons --fixed-list-mode`` output.""" + """Primary keys and their subkeys from ``gpg --with-colons --fixed-list-mode``.""" keys: list[dict[str, Any]] = [] current: dict[str, Any] | None = None want_fpr = False @@ -87,16 +108,7 @@ def parse_colons(text: str) -> list[dict[str, Any]]: f = line.split(":") + [""] * 20 kind = f[0] if kind == "pub": - current = { - "validity": f[1], - "algo_id": f[3], - "bits": int(f[2]) if f[2].isdigit() else None, - "created": _date(f[5]), - "expiry": _date(f[6]), - "curve": f[16].lower(), - "fingerprint": None, - "uids": [], - } + current = {**_key_fields(f), "fingerprint": None, "uids": [], "subkeys": []} keys.append(current) want_fpr = True elif kind == "fpr" and current is not None and want_fpr: @@ -104,38 +116,20 @@ def parse_colons(text: str) -> list[dict[str, Any]]: want_fpr = False elif kind == "sub": want_fpr = False + if current is not None: + current["subkeys"].append(_key_fields(f)) elif kind == "uid" and current is not None and f[1] not in ("r", "e"): current["uids"].append(f[9]) return keys -def assess(key: dict[str, Any] | None, fingerprint: str, today: dt.date) -> dict[str, Any]: - out: dict[str, Any] = { - "verdict": "blocked", - "key_found": key is not None, - "fingerprint": fingerprint, - "uid": None, - "algorithm": None, - "bit_length": None, - "created": None, - "expiry": None, - "strength_check": None, - "strength_note": None, - } - if key is None: - return out +def _check(key: dict[str, Any], today: dt.date) -> tuple[bool, list[str]]: + """Apply the existing strength, validity and expiry rules to one key.""" algorithm = ALGORITHMS.get(key["algo_id"]) - out.update( - uid=key["uids"][0] if key["uids"] else None, - algorithm=algorithm, - created=key["created"], - expiry=key["expiry"], - ) notes: list[str] = [] passed = False if algorithm in ("RSA", "DSA"): bits = key["bits"] - out["bit_length"] = bits passed = bits is not None and bits >= FLOOR_BITS if not passed: notes.append( @@ -154,9 +148,11 @@ def assess(key: dict[str, Any] | None, fingerprint: str, today: dt.date) -> dict notes.append(f"ECDSA curve {key['curve'] or 'unknown'!r} is not P-256 or stronger") else: notes.append(f"public-key algorithm id {key['algo_id']} is not accepted for release signing") - # gpg's validity field: a revoked, invalid or disabled key never signs a release, + # gpg's validity field: an expired, revoked, invalid or disabled key never signs a release, # whatever its strength or expiry date says. unusable = UNUSABLE_VALIDITY.get(key.get("validity", "")) + if not unusable and "D" in key["capabilities"]: + unusable = UNUSABLE_VALIDITY["d"] if unusable: passed = False notes.append(unusable) @@ -165,10 +161,61 @@ def assess(key: dict[str, Any] | None, fingerprint: str, today: dt.date) -> dict if days < 0: passed = False notes.append(f"key expired on {key['expiry']}; extend its expiry or generate a new key") - elif days <= EXPIRY_ADVISORY_DAYS: + elif days <= EXPIRY_ADVISORY_DAYS and key["validity"] != "e": notes.append( f"key expires on {key['expiry']}, within {EXPIRY_ADVISORY_DAYS} days; consider extending it" ) + return passed, notes + + +def assess(key: dict[str, Any] | None, fingerprint: str, today: dt.date) -> dict[str, Any]: + out: dict[str, Any] = { + "verdict": "blocked", + "key_found": key is not None, + "fingerprint": fingerprint, + "uid": None, + "algorithm": None, + "bit_length": None, + "created": None, + "expiry": None, + "strength_check": None, + "strength_note": None, + } + if key is None: + return out + algorithm = ALGORITHMS.get(key["algo_id"]) + out.update( + uid=key["uids"][0] if key["uids"] else None, + algorithm=algorithm, + bit_length=key["bits"] if algorithm in ("RSA", "DSA") else None, + created=key["created"], + expiry=key["expiry"], + ) + passed, notes = _check(key, today) + # Uppercase capabilities on a primary describe the entire key group. + signer = passed and "s" in key["capabilities"] + retired: list[str] = [] + for sub in key["subkeys"]: + if "s" not in sub["capabilities"]: + continue + sub_passed, sub_notes = _check(sub, today) + sub_notes = [f"signing subkey {sub['keyid']}: {note}" for note in sub_notes] + if sub_passed: + signer = True + notes.extend(sub_notes) + elif ( + sub["validity"] in UNUSABLE_VALIDITY + or "D" in sub["capabilities"] + or (sub["expiry"] and dt.date.fromisoformat(sub["expiry"]) < today) + ): + retired.extend(sub_notes) + else: + passed = False + notes.extend(sub_notes) + if not signer: + passed = False + notes.extend(retired) + notes.append("no usable signing key or subkey meets the ASF floor; add a signing subkey that does") out["strength_check"] = "pass" if passed else "fail" out["strength_note"] = "; ".join(notes) if notes else None out["verdict"] = "proceed" if passed else "blocked" @@ -218,7 +265,10 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--key-file", help="fetched public key (armoured or binary); may be empty") parser.add_argument("--fingerprint") - parser.add_argument("--today", help="YYYY-MM-DD; defaults to the current UTC date") + parser.add_argument( + "--today", + help="YYYY-MM-DD for date checks; defaults to the current UTC date; GPG validity is unchanged", + ) parser.add_argument("--keys-url", help="keys_file_url, for the KEYS-directory mode") parser.add_argument("--colons-file", help=argparse.SUPPRESS) # tests: pre-captured gpg listing args = parser.parse_args(argv) diff --git a/plugins/magpie-release-management/skills/keys-sync/tests/test_check_key.py b/plugins/magpie-release-management/skills/keys-sync/tests/test_check_key.py index 2293a4a92..8b27c8d02 100644 --- a/plugins/magpie-release-management/skills/keys-sync/tests/test_check_key.py +++ b/plugins/magpie-release-management/skills/keys-sync/tests/test_check_key.py @@ -36,8 +36,11 @@ CREATED = "1647302400" # 2022-03-15 -def colons(algo: str, bits: str, curve: str = "", expires: str = "", fpr: str = FPR) -> str: - pub = ["pub", "-", bits, algo, "KEYID", CREATED, expires] + [""] * 9 + [curve, "", "0", ""] +def colons( + algo: str, bits: str, curve: str = "", expires: str = "", fpr: str = FPR, caps: str = "scESC" +) -> str: + pub = ["pub", "-", bits, algo, "KEYID", CREATED, expires] + [""] * 4 + [caps] + [""] * 4 + pub += [curve, "", "0", ""] return "\n".join( [ "tru::1:1791072876:0:3:1:5", @@ -50,6 +53,21 @@ def colons(algo: str, bits: str, curve: str = "", expires: str = "", fpr: str = ) +def subkey( + algo: str = "1", + bits: str = "4096", + curve: str = "", + expires: str = "", + *, + validity: str = "-", + caps: str = "s", + keyid: str = "SIGNSUB", + fpr: str = "B" * 40, +) -> str: + sub = ["sub", validity, bits, algo, keyid, CREATED, expires] + [""] * 4 + [caps] + [""] * 4 + return ":".join([*sub, curve, ""]) + f"\nfpr:::::::::{fpr}:" + + def check(text: str, fpr: str = FPR) -> dict: match = next((k for k in check_key.parse_colons(text) if k["fingerprint"] == fpr), None) return check_key.assess(match, fpr, TODAY) @@ -268,6 +286,61 @@ def test_cli_rejects_a_bad_date(self) -> None: self.assertEqual(code, 2) self.assertIn("error", out) + def test_cli_blocks_a_signing_subkey_without_changing_the_json_contract(self) -> None: + listing = colons("1", "4096", caps="cSC") + "\n" + subkey(bits="1024") + code, out = self.run_cli( + "--fingerprint", FPR, "--colons-file", self.colons_file(listing), "--today", TODAY.isoformat() + ) + self.assertEqual(code, 0) + self.assertEqual((out["verdict"], out["strength_check"]), ("blocked", "fail")) + self.assertEqual((out["algorithm"], out["bit_length"]), ("RSA", 4096)) + self.assertEqual( + set(out), + { + "verdict", + "key_found", + "fingerprint", + "uid", + "algorithm", + "bit_length", + "created", + "expiry", + "strength_check", + "strength_note", + }, + ) + + def test_cli_blocks_an_only_signer_already_expired_today(self) -> None: + listing = ( + colons("1", "4096", caps="cSC") + + "\n" + + subkey("22", "255", curve="ed25519", expires=epoch(TODAY.isoformat()), validity="e") + ) + code, out = self.run_cli( + "--fingerprint", FPR, "--colons-file", self.colons_file(listing), "--today", TODAY.isoformat() + ) + self.assertEqual(code, 0) + self.assertEqual((out["verdict"], out["strength_check"]), ("blocked", "fail")) + self.assertEqual((out["algorithm"], out["bit_length"], out["expiry"]), ("RSA", 4096, None)) + self.assertIn("no usable signing key or subkey", out["strength_note"]) + self.assertNotIn("within 90 days", out["strength_note"]) + + def test_cli_admits_a_replacement_after_a_weak_subkey_expires_today(self) -> None: + listing = "\n".join( + [ + colons("1", "4096", caps="cSC"), + subkey(bits="1024", expires=epoch(TODAY.isoformat()), validity="e"), + subkey("22", "255", curve="ed25519", keyid="CURRENT"), + ] + ) + code, out = self.run_cli( + "--fingerprint", FPR, "--colons-file", self.colons_file(listing), "--today", TODAY.isoformat() + ) + self.assertEqual(code, 0) + self.assertEqual((out["verdict"], out["strength_check"]), ("proceed", "pass")) + self.assertEqual((out["algorithm"], out["bit_length"], out["expiry"]), ("RSA", 4096, None)) + self.assertIsNone(out["strength_note"]) + class ValidityTest(unittest.TestCase): """gpg's validity field: revoked, invalid and disabled keys never pass.""" @@ -292,6 +365,214 @@ def test_ordinary_validity_still_proceeds(self) -> None: with self.subTest(validity=validity): self.assertEqual(check(self.listing(validity))["verdict"], "proceed") + def test_disabled_capability_blocks_a_primary_with_a_strong_signing_subkey(self) -> None: + out = check(colons("1", "4096", caps="scSCD") + "\n" + subkey()) + self.assertEqual(out["verdict"], "blocked") + self.assertIn("disabled", out["strength_note"]) + + def test_expired_today_primary_cannot_be_rescued_by_a_strong_subkey(self) -> None: + listing = colons("1", "4096", expires=epoch(TODAY.isoformat()), caps="scESC") + listing = listing.replace("pub:-:", "pub:e:", 1) + "\n" + subkey() + out = check(listing) + self.assertEqual((out["verdict"], out["strength_check"]), ("blocked", "fail")) + self.assertEqual(out["expiry"], TODAY.isoformat()) + self.assertIn("expired", out["strength_note"]) + self.assertNotIn("within 90 days", out["strength_note"]) + + +class SigningSubkeyTest(unittest.TestCase): + """All usable signers meet the floor; retired subkeys do not prevent rotation.""" + + def listing(self, *subkeys: str, caps: str = "cSC") -> str: + return "\n".join([colons("1", "4096", caps=caps), *subkeys]) + + def test_weak_signing_subkey_blocks(self) -> None: + out = check(self.listing(subkey(bits="1024"))) + self.assertEqual((out["verdict"], out["strength_check"]), ("blocked", "fail")) + self.assertIn("signing subkey SIGNSUB: RSA 1024-bit key is below", out["strength_note"]) + self.assertEqual((out["algorithm"], out["bit_length"]), ("RSA", 4096)) + + def test_strong_signing_subkey_proceeds(self) -> None: + out = check(self.listing(subkey("22", "255", curve="ed25519"))) + self.assertEqual(out["verdict"], "proceed") + self.assertIsNone(out["strength_note"]) + + def test_certify_only_key_without_signing_subkey_blocks(self) -> None: + out = check(self.listing(caps="cC")) + self.assertEqual(out["verdict"], "blocked") + self.assertIn("no usable signing key or subkey", out["strength_note"]) + + def test_uppercase_aggregate_signing_capability_is_not_a_signer(self) -> None: + out = check(self.listing(subkey(caps="eS"))) + self.assertEqual(out["verdict"], "blocked") + self.assertIn("no usable signing key or subkey", out["strength_note"]) + + def test_only_expired_signing_subkey_blocks_and_explains_why(self) -> None: + out = check(self.listing(subkey(expires=epoch("2026-06-18")))) + self.assertEqual(out["verdict"], "blocked") + self.assertIn("signing subkey SIGNSUB: key expired on 2026-06-18", out["strength_note"]) + self.assertIn("no usable signing key or subkey", out["strength_note"]) + + def test_only_revoked_invalid_or_disabled_signing_subkey_blocks(self) -> None: + for validity, caps, word in ( + ("r", "s", "revoked"), + ("i", "s", "invalid"), + ("d", "s", "disabled"), + ("-", "sD", "disabled"), + ): + with self.subTest(validity=validity, caps=caps): + out = check(self.listing(subkey(validity=validity, caps=caps))) + self.assertEqual(out["verdict"], "blocked") + self.assertIn(word, out["strength_note"]) + self.assertIn("no usable signing key or subkey", out["strength_note"]) + + def test_retired_weak_subkeys_beside_a_strong_one_proceed(self) -> None: + for retired in ( + subkey(bits="1024", expires=epoch("2020-01-01")), + subkey(bits="1024", validity="e", expires=epoch("2020-01-01")), + subkey(bits="1024", validity="r"), + subkey(bits="1024", validity="i"), + subkey(bits="1024", validity="d"), + subkey(bits="1024", caps="sD"), + ): + with self.subTest(retired=retired): + out = check(self.listing(retired, subkey(keyid="CURRENT"))) + self.assertEqual(out["verdict"], "proceed") + self.assertIsNone(out["strength_note"]) + + def test_active_weak_signing_subkey_blocks_a_strong_signing_subkey(self) -> None: + out = check(self.listing(subkey(), subkey(bits="1024", keyid="WEAK"))) + self.assertEqual(out["verdict"], "blocked") + self.assertIn("signing subkey WEAK", out["strength_note"]) + + def test_active_weak_signing_subkey_blocks_a_signing_primary(self) -> None: + out = check(self.listing(subkey(bits="1024"), caps="scESC")) + self.assertEqual(out["verdict"], "blocked") + + def test_retired_signing_subkey_does_not_block_a_signing_primary(self) -> None: + out = check(self.listing(subkey(bits="1024", expires=epoch("2020-01-01")), caps="scESC")) + self.assertEqual(out["verdict"], "proceed") + self.assertIsNone(out["strength_note"]) + + def test_non_signing_subkeys_do_not_participate_in_the_floor(self) -> None: + for caps in ("e", "a"): + with self.subTest(caps=caps): + out = check(self.listing(subkey(bits="1024", caps=caps), caps="scESC")) + self.assertEqual(out["verdict"], "proceed") + self.assertIsNone(out["strength_note"]) + + def test_signing_subkey_strength_boundaries(self) -> None: + for algo, bits, curve, verdict in ( + ("1", "2048", "", "proceed"), + ("1", "2047", "", "blocked"), + ("22", "255", "ed25519", "proceed"), + ("22", "448", "ed448", "proceed"), + ("19", "256", "nistp256", "proceed"), + ("19", "256", "secp256k1", "blocked"), + ("22", "255", "unknown", "blocked"), + ("18", "255", "cv25519", "blocked"), + ): + with self.subTest(algo=algo, bits=bits, curve=curve): + self.assertEqual(check(self.listing(subkey(algo, bits, curve)))["verdict"], verdict) + + def test_dsa_signing_subkey_keeps_the_advisory(self) -> None: + out = check(self.listing(subkey("17", "2048"))) + self.assertEqual(out["verdict"], "proceed") + self.assertIn("signing subkey SIGNSUB: DSA keys are discouraged", out["strength_note"]) + + def test_signing_subkey_expiry_within_90_days_is_advisory(self) -> None: + out = check(self.listing(subkey(expires=epoch("2026-08-01")))) + self.assertEqual(out["verdict"], "proceed") + self.assertIn("signing subkey SIGNSUB: key expires on 2026-08-01", out["strength_note"]) + + def test_signing_subkey_expiring_today_is_advisory(self) -> None: + out = check(self.listing(subkey(expires=epoch(TODAY.isoformat())))) + self.assertEqual(out["verdict"], "proceed") + self.assertIn("within 90 days", out["strength_note"]) + + def test_already_expired_today_weak_subkey_with_replacement_proceeds(self) -> None: + out = check( + self.listing( + subkey(bits="1024", expires=epoch(TODAY.isoformat()), validity="e"), + subkey("22", "255", curve="ed25519", keyid="CURRENT"), + ) + ) + self.assertEqual(out["verdict"], "proceed") + self.assertIsNone(out["strength_note"]) + + def test_already_expired_today_only_signer_blocks_without_advisory(self) -> None: + out = check( + self.listing(subkey("22", "255", curve="ed25519", expires=epoch(TODAY.isoformat()), validity="e")) + ) + self.assertEqual((out["verdict"], out["strength_check"]), ("blocked", "fail")) + self.assertIn("signing subkey SIGNSUB:", out["strength_note"]) + self.assertIn("expired", out["strength_note"]) + self.assertIn("no usable signing key or subkey", out["strength_note"]) + self.assertNotIn("within 90 days", out["strength_note"]) + + def test_earlier_today_cannot_clear_gpg_expired_status(self) -> None: + listing = self.listing( + subkey("22", "255", curve="ed25519", expires=epoch(TODAY.isoformat()), validity="e") + ) + key = check_key.parse_colons(listing)[0] + out = check_key.assess(key, FPR, TODAY - dt.timedelta(days=1)) + self.assertEqual(out["verdict"], "blocked") + self.assertIn("expired", out["strength_note"]) + self.assertNotIn("within 90 days", out["strength_note"]) + + def test_already_expired_today_subkey_does_not_block_a_signing_primary(self) -> None: + out = check( + self.listing(subkey(bits="1024", expires=epoch(TODAY.isoformat()), validity="e"), caps="scESC") + ) + self.assertEqual(out["verdict"], "proceed") + self.assertIsNone(out["strength_note"]) + + def test_far_future_signing_subkey_expiry_has_no_advisory(self) -> None: + out = check(self.listing(subkey(expires=epoch("2028-01-01")))) + self.assertEqual(out["verdict"], "proceed") + self.assertIsNone(out["strength_note"]) + + def test_unusable_primary_cannot_be_rescued_by_a_strong_subkey(self) -> None: + for validity in ("r", "i", "d"): + with self.subTest(validity=validity): + listing = self.listing(subkey()).replace("pub:-:", f"pub:{validity}:", 1) + self.assertEqual(check(listing)["verdict"], "blocked") + self.assertEqual( + check(colons("1", "4096", expires=epoch("2020-01-01"), caps="cSC") + "\n" + subkey())["verdict"], + "blocked", + ) + + +class SubkeyParsingTest(unittest.TestCase): + def test_subkey_fingerprint_never_replaces_the_primary_fingerprint(self) -> None: + listing = colons("1", "4096", caps="cSC") + "\n" + subkey(fpr="C" * 40) + keys = check_key.parse_colons(listing) + self.assertEqual(keys[0]["fingerprint"], FPR) + out = check(listing, "C" * 40) + self.assertFalse(out["key_found"]) + self.assertEqual(out["verdict"], "blocked") + + def test_subkeys_stay_with_their_own_primary(self) -> None: + second = "D" * 40 + listing = "\n".join( + [ + colons("1", "4096", caps="cSC"), + subkey(bits="1024", keyid="FIRST"), + colons("1", "4096", fpr=second, caps="cSC"), + subkey(keyid="SECOND"), + ] + ) + keys = check_key.parse_colons(listing) + self.assertEqual(keys[0]["subkeys"][-1]["keyid"], "FIRST") + self.assertEqual(keys[1]["subkeys"][-1]["keyid"], "SECOND") + self.assertEqual(check(listing)["verdict"], "blocked") + self.assertEqual(check(listing, second)["verdict"], "proceed") + + def test_orphan_subkey_is_not_attached_to_the_next_primary(self) -> None: + listing = subkey(bits="1024") + "\n" + colons("1", "4096") + self.assertEqual(check(listing)["verdict"], "proceed") + self.assertEqual(len(check_key.parse_colons(listing)[0]["subkeys"]), 1) + if __name__ == "__main__": unittest.main() diff --git a/tools/skill-evals/README.md b/tools/skill-evals/README.md index 9fbc76bc6..8e989b1dd 100644 --- a/tools/skill-evals/README.md +++ b/tools/skill-evals/README.md @@ -72,7 +72,7 @@ Suites are currently implemented for: - **release-announce-draft** — 9 cases across 3 suites (step-0-preflight, step-2-announce-draft, step-3-site-bump) - **release-archive-sweep** — 10 cases across 3 suites (step-0-preflight, step-1-load-listing, step-2-emit-commands) - **release-audit-report** — 9 cases across 3 suites (step-0-preflight, step-1-gather-record, step-2-assemble-record) -- **release-keys-sync** — 9 cases across 3 suites (step-0-preflight, step-1-key-fetch, step-2-svn-commands) +- **release-keys-sync** — 14 cases across 3 suites (step-0-preflight, step-1-key-fetch, step-2-svn-commands) - **release-prepare** — 15 cases across 4 suites (step-0-preflight, step-1-plan, step-14-post, step-2-prep) - **release-promote** — 9 cases across 2 suites (step-0-preflight, step-2-emit-commands) - **release-rc-cut** — 16 cases across 4 suites (step-0-preflight, step-2-tag-build-sign, step-2b-reproducibility, step-3-staging) diff --git a/tools/skill-evals/evals/release-keys-sync/README.md b/tools/skill-evals/evals/release-keys-sync/README.md index 9f7508d62..06ce23ab0 100644 --- a/tools/skill-evals/evals/release-keys-sync/README.md +++ b/tools/skill-evals/evals/release-keys-sync/README.md @@ -5,12 +5,12 @@ Behavioral evals for the `release-keys-sync` skill. -## Suites (9 cases total) +## Suites (14 cases total) | Suite | Step | Cases | What it covers | |---|---|---|---| | step-0-preflight | Step 0 (pre-flight check) | 3 | clean pass (proceed), fingerprint already in KEYS (noop), key rolled with different UID (blocked) | -| step-1-key-fetch | Step 1 (fetch and validate key) | 3 | RSA 4096-bit key found and passes strength check, key not found on keyserver, weak RSA 1024-bit key fails floor | +| step-1-key-fetch | Step 1 (fetch and validate key) | 8 | strong and missing keys, weak primary or signing subkey, no usable signer, rotation, already-expired same-day subkeys with and without a replacement | | step-2-svn-commands | Step 2 (draft KEYS block and command sequence) | 3 | standard ASF svnpubsub commands, Ed25519 key on non-ASF GitHub-hosted KEYS, RSA key expiring within 90 days triggers advisory | ## Run diff --git a/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/assertions.json b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/assertions.json index b9d677c6f..2583eeda6 100644 --- a/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/assertions.json +++ b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/assertions.json @@ -3,5 +3,21 @@ "type": "contains_all", "field": "strength_note", "substrings": ["1024", "2048"] + }, + "has_no_signer_note": { + "type": "contains_all", + "field": "strength_note", + "substrings": ["no usable signing key or subkey", "add a signing subkey"] + }, + "has_expired_subkey_note": { + "type": "contains_all", + "field": "strength_note", + "substrings": ["signing subkey", "expired"] + }, + "has_no_expiry_advisory": { + "type": "contains", + "field": "strength_note", + "substring": "within 90 days", + "negate": true } } diff --git a/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-4-weak-signing-subkey/expected.json b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-4-weak-signing-subkey/expected.json new file mode 100644 index 000000000..82165dc3d --- /dev/null +++ b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-4-weak-signing-subkey/expected.json @@ -0,0 +1,13 @@ +{ + "verdict": "blocked", + "key_found": true, + "fingerprint": "A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD", + "uid": "Example Release Manager ", + "algorithm": "RSA", + "bit_length": 4096, + "created": "2022-03-15", + "expiry": null, + "strength_check": "fail", + "has_weak_key_note": true, + "has_no_signer_note": true +} diff --git a/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-4-weak-signing-subkey/report.md b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-4-weak-signing-subkey/report.md new file mode 100644 index 000000000..59ae4089b --- /dev/null +++ b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-4-weak-signing-subkey/report.md @@ -0,0 +1,25 @@ + + +Pre-flight: PASS (verdict: proceed) +Fetching fingerprint A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD from keys.openpgp.org + +The keyserver returned an RSA 4096-bit certify-only primary key and one active RSA 1024-bit signing subkey, SIGNSUB. +Today's date: 2026-06-19. + +Output of `python3 /scripts/check_key.py --key-file fetched-key.asc --fingerprint A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD`: + +```json +{ + "verdict": "blocked", + "key_found": true, + "fingerprint": "A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD", + "uid": "Example Release Manager ", + "algorithm": "RSA", + "bit_length": 4096, + "created": "2022-03-15", + "expiry": null, + "strength_check": "fail", + "strength_note": "signing subkey SIGNSUB: RSA 1024-bit key is below the ASF floor of 2048 bits; generate a new key that meets the floor; no usable signing key or subkey meets the ASF floor; add a signing subkey that does" +} +``` diff --git a/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-5-no-signing-key/expected.json b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-5-no-signing-key/expected.json new file mode 100644 index 000000000..fc19ffde8 --- /dev/null +++ b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-5-no-signing-key/expected.json @@ -0,0 +1,12 @@ +{ + "verdict": "blocked", + "key_found": true, + "fingerprint": "A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD", + "uid": "Example Release Manager ", + "algorithm": "RSA", + "bit_length": 4096, + "created": "2022-03-15", + "expiry": null, + "strength_check": "fail", + "has_no_signer_note": true +} diff --git a/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-5-no-signing-key/report.md b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-5-no-signing-key/report.md new file mode 100644 index 000000000..ebbd9a985 --- /dev/null +++ b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-5-no-signing-key/report.md @@ -0,0 +1,25 @@ + + +Pre-flight: PASS (verdict: proceed) +Fetching fingerprint A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD from keys.openpgp.org + +The keyserver returned an RSA 4096-bit certify-only primary key with capabilities `cC` and no signing subkey. +Today's date: 2026-06-19. + +Output of `python3 /scripts/check_key.py --key-file fetched-key.asc --fingerprint A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD`: + +```json +{ + "verdict": "blocked", + "key_found": true, + "fingerprint": "A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD", + "uid": "Example Release Manager ", + "algorithm": "RSA", + "bit_length": 4096, + "created": "2022-03-15", + "expiry": null, + "strength_check": "fail", + "strength_note": "no usable signing key or subkey meets the ASF floor; add a signing subkey that does" +} +``` diff --git a/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-6-retired-weak-subkey/expected.json b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-6-retired-weak-subkey/expected.json new file mode 100644 index 000000000..c413f11cb --- /dev/null +++ b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-6-retired-weak-subkey/expected.json @@ -0,0 +1,12 @@ +{ + "verdict": "proceed", + "key_found": true, + "fingerprint": "A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD", + "uid": "Example Release Manager ", + "algorithm": "RSA", + "bit_length": 4096, + "created": "2022-03-15", + "expiry": null, + "strength_check": "pass", + "strength_note": null +} diff --git a/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-6-retired-weak-subkey/report.md b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-6-retired-weak-subkey/report.md new file mode 100644 index 000000000..ee06cc2a1 --- /dev/null +++ b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-6-retired-weak-subkey/report.md @@ -0,0 +1,25 @@ + + +Pre-flight: PASS (verdict: proceed) +Fetching fingerprint A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD from keys.openpgp.org + +The keyserver returned an RSA 4096-bit certify-only primary key, an RSA 1024-bit signing subkey that expired on 2020-01-01, and a current Ed25519 signing subkey. +Today's date: 2026-06-19. + +Output of `python3 /scripts/check_key.py --key-file fetched-key.asc --fingerprint A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD`: + +```json +{ + "verdict": "proceed", + "key_found": true, + "fingerprint": "A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD", + "uid": "Example Release Manager ", + "algorithm": "RSA", + "bit_length": 4096, + "created": "2022-03-15", + "expiry": null, + "strength_check": "pass", + "strength_note": null +} +``` diff --git a/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-7-expired-today-with-replacement/expected.json b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-7-expired-today-with-replacement/expected.json new file mode 100644 index 000000000..c413f11cb --- /dev/null +++ b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-7-expired-today-with-replacement/expected.json @@ -0,0 +1,12 @@ +{ + "verdict": "proceed", + "key_found": true, + "fingerprint": "A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD", + "uid": "Example Release Manager ", + "algorithm": "RSA", + "bit_length": 4096, + "created": "2022-03-15", + "expiry": null, + "strength_check": "pass", + "strength_note": null +} diff --git a/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-7-expired-today-with-replacement/report.md b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-7-expired-today-with-replacement/report.md new file mode 100644 index 000000000..8bccdf38d --- /dev/null +++ b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-7-expired-today-with-replacement/report.md @@ -0,0 +1,26 @@ + + +Pre-flight: PASS (verdict: proceed) +Fetching fingerprint A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD from keys.openpgp.org + +The RSA 4096-bit certify-only primary has an RSA 1024-bit signing subkey that expired today at 06:00 UTC and an unexpired Ed25519 replacement. +GPG computed the listing at 12:00 UTC on 2026-06-19 and marked the weak signing subkey with validity `e`. +The helper's `--today` is 2026-06-19; the calendar date must not make the already-expired weak subkey active again. + +Output of `python3 /scripts/check_key.py --key-file fetched-key.asc --fingerprint A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD --today 2026-06-19`: + +```json +{ + "verdict": "proceed", + "key_found": true, + "fingerprint": "A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD", + "uid": "Example Release Manager ", + "algorithm": "RSA", + "bit_length": 4096, + "created": "2022-03-15", + "expiry": null, + "strength_check": "pass", + "strength_note": null +} +``` diff --git a/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-8-only-expired-today-signer/expected.json b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-8-only-expired-today-signer/expected.json new file mode 100644 index 000000000..7c8abface --- /dev/null +++ b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-8-only-expired-today-signer/expected.json @@ -0,0 +1,14 @@ +{ + "verdict": "blocked", + "key_found": true, + "fingerprint": "A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD", + "uid": "Example Release Manager ", + "algorithm": "RSA", + "bit_length": 4096, + "created": "2022-03-15", + "expiry": null, + "strength_check": "fail", + "has_no_signer_note": true, + "has_expired_subkey_note": true, + "has_no_expiry_advisory": true +} diff --git a/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-8-only-expired-today-signer/report.md b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-8-only-expired-today-signer/report.md new file mode 100644 index 000000000..ee67c8d58 --- /dev/null +++ b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/case-8-only-expired-today-signer/report.md @@ -0,0 +1,26 @@ + + +Pre-flight: PASS (verdict: proceed) +Fetching fingerprint A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD from keys.openpgp.org + +The RSA 4096-bit certify-only primary has only one Ed25519 signing subkey, SIGNSUB, which expired today at 06:00 UTC. +GPG computed the listing at 12:00 UTC on 2026-06-19 and marked that subkey with validity `e`. +The helper's `--today` is 2026-06-19; there is no usable signer and no non-blocking expiry advisory. + +Output of `python3 /scripts/check_key.py --key-file fetched-key.asc --fingerprint A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD --today 2026-06-19`: + +```json +{ + "verdict": "blocked", + "key_found": true, + "fingerprint": "A1B2C3D4E5F6A1B2C3D4E5F6A1B2C3D4E5F6ABCD", + "uid": "Example Release Manager ", + "algorithm": "RSA", + "bit_length": 4096, + "created": "2022-03-15", + "expiry": null, + "strength_check": "fail", + "strength_note": "signing subkey SIGNSUB: key is expired (gpg marked it expired); extend its expiry or generate a new key; no usable signing key or subkey meets the ASF floor; add a signing subkey that does" +} +``` diff --git a/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/output-spec.md b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/output-spec.md index 6fa5aeb4f..25660bb4d 100644 --- a/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/output-spec.md +++ b/tools/skill-evals/evals/release-keys-sync/step-1-key-fetch/fixtures/output-spec.md @@ -22,11 +22,17 @@ The model must return ONLY valid JSON matching this schema: Grading rules: - `verdict` must be `"proceed"` only when `key_found` is `true` AND `strength_check` is `"pass"`. -- `verdict` must be `"blocked"` when `key_found` is `false` (key not on keyserver) or when `strength_check` is `"fail"` (key below ASF floor). +- `verdict` must be `"blocked"` when `key_found` is `false` (key not on keyserver) or when `strength_check` is `"fail"`. - `key_found` must be `true` or `false` based on whether the keyserver returned a key. - `strength_check` must be `null` when `key_found` is `false`. -- `strength_check` must be `"pass"` for RSA ≥ 2048 bits, EdDSA (Ed25519), and ECDSA (P-256+). -- `strength_check` must be `"fail"` for RSA < 2048 bits and DSA < 2048 bits. +- Return the script's JSON unchanged; never rejudge a key from its primary algorithm or size. +- `algorithm`, `bit_length`, `created` and `expiry` describe the primary key; `strength_check` describes the complete signing-key assessment. +- `strength_check` is `"pass"` only when the primary and all usable signing subkeys pass their strength checks, and at least one usable signing key or subkey meets the floor. +- `strength_check` is `"fail"` for a sub-floor primary, any usable sub-floor signing subkey, an unusable primary, or no usable compliant signer. +- Expired, revoked, invalid or disabled signing subkeys cannot provide a signer, but do not block a compliant replacement. +- GPG validity `e` means already expired, including on the expiry date; a matching or earlier `--today` cannot restore usability. +- `--today` changes only calendar-date checks and advisories, not GPG's computed validity or clock; historical replay requires a listing captured at the intended instant. +- A same-day key not marked expired can have an expiry advisory; a key marked `e` must not also receive a non-blocking expiry advisory. - `bit_length` must be `null` for curve-based algorithms (EdDSA, ECDSA). -- `strength_note` must be non-null when `strength_check` is `"fail"`, and must explain why the key fails the ASF floor. +- `strength_note` must be non-null when `strength_check` is `"fail"`, and must explain the failure, including a subkey identifier for subkey-specific reasons. - No extra keys are permitted in the response. diff --git a/tools/spec-loop/specs/release-management-lifecycle.md b/tools/spec-loop/specs/release-management-lifecycle.md index 7519f93e3..bfa936a39 100644 --- a/tools/spec-loop/specs/release-management-lifecycle.md +++ b/tools/spec-loop/specs/release-management-lifecycle.md @@ -153,6 +153,14 @@ code lands. - **Drafts, never sends.** `[VOTE]` (step 7) and `[ANNOUNCE]` (step 11) email bodies are drafted to the maintainer's outbox; no skill calls a send. +- **Usable signing keys meet the floor.** `release-keys-sync` validates + the primary and every usable signing subkey, and requires at least one + usable compliant signer. An active weak signing subkey blocks even + alongside a strong signer; this is the skill's conservative admission + policy, not a separate ASF mandate for mixed key groups. Expired, + revoked, invalid or disabled subkeys do not prevent rotation to a + compliant signer. The JSON metadata still describes the primary; + `strength_check` and `strength_note` describe the complete assessment. - **Conservative tally.** `release-vote-tally` classifies +1/0/-1 binding vs non-binding against the PMC roster and refuses to count ambiguous votes, flagging `AMBIGUOUS, needs RM call` rather than guessing.