perf(ci): cut GITHUB_TOKEN calls in pr-verify and pace timer lanes on burn rate

.changes/pr-pipeline/perf-github-token-cuts-30e1480a.md

@@ -0,0 +1,19 @@
+---
+bump: minor
+---
+
+### Added
+
+- `limits.api` github-token `pace` (default 0.2, in rate-limits.json; set it in pr-pipeline.yaml,
+  0 turns pacing off): the lost-run reconciler
+  and pr-unblock-dispatch skip every other scheduled tick (the even run numbers) while the
+  GITHUB_TOKEN pool's remaining at its reset, projected from the window's burn rate so far, is
+  under that share of the header's limit, so the timer lanes slow to half cadence before the
+  breaker stops every lane. Any other event, or a failed read, runs as before.
+
+### Changed
+
+- pr-verify runs its read-only resolver before its trigger gate, so a run the resolver skips (a
+  head already judged or moved past, a draft, a run red for anything an approval does not lift,
+  an author it does not judge) no longer spends the gate's reads. A judged run makes the same
+  reads as before.

.github/actions/log-api-budget/log-api-budget.test.mjs

@@ -785,7 +785,8 @@ test("the action writes the content-creation notice after the token lines", () =
 // content-creating writes.
 test("each heavy lane job names its API_BUDGET_TOKEN and ends with the log-api-budget step", () => {
   const jobs = {
-    "intake-triage.yml": { "born-triaged": "github-token", "apply-verdict": "lanes-app", "refine-select": "github-token", "apply-refinement": "lanes-app" },
+    // apply-verdict's resolve-config reads with the job token, recorded under github-token.
+    "intake-triage.yml": { "born-triaged": "github-token", "apply-verdict": ["lanes-app", "lanes-app github-token"], "refine-select": "github-token", "apply-refinement": "lanes-app" },
     "work-resolve-issue.yml": { admit: "github-token", "pick-slot": "github-token", "report-statuses": "github-token" },
     "pr-verify.yml": { "check-intent": "github-token", "check-intent-subscription": "github-token", "check-intent-fallback": "github-token" },
     // label-unblocked's ACTIONS_TOKEN client records under github-token, so
@@ -814,6 +815,29 @@ test("each heavy lane job names its API_BUDGET_TOKEN and ends with the log-api-b
   }
 });
 
+// pr-review-main's record job has no checkout of the action, and its only
+// calls are one check-run POST per verdict file, so it counts them itself and
+// ends with the notice in api_budget_emit_calls' shape.
+test("pr-review-main's record job ends with its own api-calls notice in the emitter's shape", () =>
+  withDir((dir) => {
+    const text = readFileSync(new URL("../../workflows/pr-review-main.yml", import.meta.url), "utf8");
+    const block = /\n  record:\n([\s\S]*?)(?=\n  [a-z0-9-]+:\n|$)/.exec(text)?.[1] ?? "";
+    const last = block.split(/\n {6}- name: /).at(-1);
+    assert.match(last, /^Log the API calls at the end\n {8}if: always\(\)\n/);
+    const run = /\n {8}run: \|\n((?: {10}[^\n]*\n?)+)/.exec(last)?.[1].replace(/^ {10}/gm, "");
+    assert.ok(run, "the step has a run block");
+    const lib = new URL("api-budget.bash", import.meta.url).pathname;
+    const emitted = (count) => {
+      if (count !== undefined) writeFileSync(join(dir, "pr-review-main-calls"), `${count}\n`);
+      // As Actions runs `shell: bash`: -e ends the step on any failing command.
+      return execFileSync("bash", ["--noprofile", "--norc", "-eo", "pipefail", "-c", run], { encoding: "utf8", env: { PATH: process.env.PATH, RUNNER_TEMP: dir } });
+    };
+    const shape = execFileSync("bash", ["-c", `. "${lib}"; api_budget_emit_calls github-token`], { encoding: "utf8", env: { PATH: process.env.PATH, RUNNER_TEMP: dir } });
+    assert.equal(emitted(), shape, "no count file is 0, as the emitter writes with no reading");
+    assert.equal(emitted(2), shape.replace("github-token=0", "github-token=2"));
+    assert.equal(emitted("2; rm"), shape, "anything but a count is 0");
+  }));
+
 // Jobs that hold the job token but call GitHub only through actions the
 // counter cannot see still end by logging it, so each run carries its
 // api-calls line, 0 when no step of the job's own called GitHub.

.github/actions/log-api-budget/pace.mjs

@@ -0,0 +1,77 @@
+// The burn-rate pacer for timer-driven lanes. A budget trip (the breaker
+// share of the header's limit left, rate-limits.json) turns the kill switch
+// on and stops every lane; this slows the timer-driven ones first. On a
+// schedule tick it reads the GITHUB_TOKEN pool once, projects the remaining
+// at the window's reset from the burn rate the window has shown so far, and
+// writes paced=true when that falls under `pace` of the header's limit
+// (limits.api github-token pace: pr-pipeline.yaml, which each paced workflow
+// hands over as RESOLVED_CONFIG, over rate-limits.json; 0 is off). Only an
+// even-numbered tick is paced, so a paced lane runs at half its cadence
+// rather than stopping: the lost-run reconciler, which re-runs the runs a
+// breaker trip stops, keeps working while the pool recovers. A paced tick
+// skips its work and ends green. Any other event, a reading that cannot
+// project, a run number that is no number, or a failed read is
+// paced=false: the breaker stays the backstop.
+//
+//   node pace.mjs   EVENT_NAME, RUN_NUMBER, GITHUB_TOKEN, GITHUB_REPOSITORY
+//                   and GITHUB_OUTPUT; RESOLVED_CONFIG optional
+
+import { appendFileSync } from "node:fs";
+import process from "node:process";
+import { pathToFileURL } from "node:url";
+
+import { apiLimits, countRequests } from "./rate-guard.mjs";
+
+// GitHub's primary rate-limit window.
+const WINDOW_SECONDS = 3600;
+// Before this much of the window has passed, the burn rate is too short to project.
+const MIN_ELAPSED_SECONDS = 60;
+
+const round = (value) => Math.round(value * 100) / 100;
+
+// { paced, projected } for one reading at `now` (epoch seconds): projected is
+// the share of the header's limit left at the reset, at the window's burn
+// rate so far, clamped to 0..1.
+export function paceDecision({ limit, remaining, reset } = {}, share, now) {
+  const left = reset - now;
+  if (![limit, remaining, reset].every(Number.isFinite) || limit <= 0 || remaining < 0 || remaining > limit || left < 0 || !(share > 0)) {
+    return { paced: false, projected: null };
+  }
+  const elapsed = WINDOW_SECONDS - Math.min(left, WINDOW_SECONDS);
+  const used = limit - remaining;
+  const atReset = elapsed < MIN_ELAPSED_SECONDS ? remaining : remaining - (used * left) / elapsed;
+  const projected = Math.max(0, atReset) / limit;
+  return { paced: projected < share, projected: round(projected) };
+}
+
+export async function main(env = process.env, { fetchImpl = fetch, now = () => Date.now() / 1000 } = {}) {
+  let paced = false;
+  if (env.EVENT_NAME === "schedule") {
+    try {
+      const share = apiLimits({ resolvedPath: env.RESOLVED_CONFIG })["github-token"].pace;
+      // Counted for the job-end api-calls notice like any guarded call.
+      const response = await countRequests(fetchImpl)(`${env.GITHUB_API_URL ?? "https://api.github.com"}/repos/${env.GITHUB_REPOSITORY}`, {
+        headers: { Accept: "application/vnd.github+json", Authorization: `Bearer ${env.GITHUB_TOKEN}`, "X-GitHub-Api-Version": "2022-11-28" },
+      });
+      const header = (name) => (response.headers.has(name) ? Number(response.headers.get(name)) : Number.NaN);
+      const reading = { limit: header("x-ratelimit-limit"), remaining: header("x-ratelimit-remaining"), reset: header("x-ratelimit-reset") };
+      const decision = response.ok ? paceDecision(reading, share, now()) : { paced: false, projected: null };
+      const even = /^[0-9]+$/.test(env.RUN_NUMBER ?? "") && Number(env.RUN_NUMBER) % 2 === 0;
+      paced = decision.paced && even;
+      const why = paced ? "; this tick skips its work" : decision.paced ? "; an odd tick runs at half cadence" : "";
+      if (decision.projected !== null) {
+        console.log(`::notice title=api-pace::github-token projected=${decision.projected} pace=${share} paced=${paced}${why}`);
+      }
+    } catch (error) {
+      console.log(`::warning::the pacer could not read the GITHUB_TOKEN pool (${error.message}); the tick runs`);
+      paced = false;
+    }
+  }
+  appendFileSync(env.GITHUB_OUTPUT, `paced=${paced}\n`);
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  main().catch((error) => {
+    console.log(`::warning::the pacer failed (${error.message}); the tick runs`);
+  });
+}

.github/actions/log-api-budget/pace.test.mjs

@@ -0,0 +1,178 @@
+// The burn-rate pacer's rules (pace.mjs): a timer-driven run is paced when
+// the GITHUB_TOKEN pool's remaining at its reset, projected at the burn rate
+// the window has shown so far, falls under `pace` of the header's limit, the
+// share the breaker is also stated in (pr-pipeline.schema.json api-pool).
+// Expected values are worked by hand from limit, remaining and reset.
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { test } from "node:test";
+
+import { main, paceDecision } from "./pace.mjs";
+import { apiLimits } from "./rate-guard.mjs";
+import { tempDir } from "../../../lib/test-tmp.mjs";
+
+const NOW = 1_800_000_000;
+
+test("the pace share is a config value: rate-limits.json's github-token default, under limits.api", () => {
+  assert.equal(apiLimits({ resolvedPath: "" })["github-token"].pace, 0.2);
+  const dir = tempDir("pace-config-");
+  const resolved = join(dir, "resolved.json");
+  writeFileSync(resolved, JSON.stringify({ limits: { api: { "github-token": { pace: 0.35 } } } }));
+  assert.equal(apiLimits({ resolvedPath: resolved })["github-token"].pace, 0.35);
+  // The schema lets limits.api set it, in the same range as the breaker.
+  const schema = JSON.parse(readFileSync(new URL("../../../docs/conventions/pr-pipeline/pr-pipeline.schema.json", import.meta.url), "utf8"));
+  assert.deepEqual(
+    { ...schema.$defs["api-pool"].properties.pace, description: undefined },
+    { ...schema.$defs["api-pool"].properties.breaker, description: undefined },
+  );
+});
+
+test("a run is paced when the projected remaining at reset falls under the share", () => {
+  // 30 minutes in: 2,000 of 5,000 used, so 2,000 more by the reset leaves 1,000, 20%.
+  const half = { limit: 5000, remaining: 3000, reset: NOW + 1800 };
+  assert.deepEqual(paceDecision(half, 0.2, NOW), { paced: false, projected: 0.2 });
+  // 2,100 used: 2,100 more leaves 800, 16%.
+  assert.deepEqual(paceDecision({ ...half, remaining: 2900 }, 0.2, NOW), { paced: true, projected: 0.16 });
+  // The evidence window's reading: used 4,289, remaining 711, about 4 minutes to reset.
+  assert.equal(paceDecision({ limit: 5000, remaining: 711, reset: NOW + 252 }, 0.2, NOW).paced, true);
+  // A slow burn early in the window: 100 used in 10 minutes projects 500 more, 4,400 left.
+  assert.deepEqual(paceDecision({ limit: 5000, remaining: 4900, reset: NOW + 3000 }, 0.2, NOW), { paced: false, projected: 0.88 });
+  // A projection under zero is clamped to it.
+  assert.deepEqual(paceDecision({ limit: 1000, remaining: 100, reset: NOW + 1800 }, 0.2, NOW), { paced: true, projected: 0 });
+});
+
+test("the first minute projects nothing, so only the remaining itself can pace it", () => {
+  assert.deepEqual(paceDecision({ limit: 5000, remaining: 4000, reset: NOW + 3590 }, 0.2, NOW), { paced: false, projected: 0.8 });
+  assert.deepEqual(paceDecision({ limit: 5000, remaining: 900, reset: NOW + 3590 }, 0.2, NOW), { paced: true, projected: 0.18 });
+});
+
+test("a reading that cannot project, or a share of 0, never paces", () => {
+  for (const reading of [{}, { limit: 0, remaining: 0, reset: NOW + 60 }, { limit: 5000, remaining: "na", reset: NOW + 60 }, { limit: 5000, remaining: 100, reset: NOW - 10 }, { limit: 5000, remaining: 6000, reset: NOW + 60 }]) {
+    assert.equal(paceDecision(reading, 0.2, NOW).paced, false, JSON.stringify(reading));
+  }
+  assert.equal(paceDecision({ limit: 5000, remaining: 10, reset: NOW + 1800 }, 0, NOW).paced, false);
+});
+
+// main's one read: a 200 answer's x-ratelimit headers.
+const answer = (headers, status = 200) => async () => new Response("{}", { status, headers });
+const run = async (env, fetchImpl) => {
+  const dir = tempDir("pace-");
+  const output = join(dir, "output");
+  writeFileSync(output, "");
+  const lines = [];
+  const log = console.log;
+  console.log = (line) => lines.push(line);
+  try {
+    await main({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", GITHUB_OUTPUT: output, RUNNER_TEMP: dir, EVENT_NAME: "schedule", RUN_NUMBER: "42", ...env }, { fetchImpl, now: () => NOW });
+  } finally {
+    console.log = log;
+  }
+  return { output: readFileSync(output, "utf8"), lines };
+};
+
+test("on a schedule tick main reads the pool once and writes paced, with a notice", async () => {
+  let calls = 0;
+  const fetchImpl = async (...args) => {
+    calls += 1;
+    return answer({ "x-ratelimit-limit": "5000", "x-ratelimit-remaining": "2900", "x-ratelimit-used": "2100", "x-ratelimit-reset": String(NOW + 1800), "x-ratelimit-resource": "core" })(...args);
+  };
+  const got = await run({}, fetchImpl);
+  assert.equal(calls, 1);
+  assert.equal(got.output, "paced=true\n");
+  assert.deepEqual(got.lines, ["::notice title=api-pace::github-token projected=0.16 pace=0.2 paced=true; this tick skips its work"]);
+});
+
+// Pacing halves a lane's cadence rather than stopping it: a paced
+// projection still runs the odd-numbered ticks, so the lost-run reconciler
+// keeps re-running lost runs while the pool recovers.
+test("a paced projection skips only the even-numbered ticks", async () => {
+  const low = answer({ "x-ratelimit-limit": "5000", "x-ratelimit-remaining": "2900", "x-ratelimit-reset": String(NOW + 1800) });
+  assert.equal((await run({ RUN_NUMBER: "42" }, low)).output, "paced=true\n");
+  const odd = await run({ RUN_NUMBER: "43" }, low);
+  assert.equal(odd.output, "paced=false\n");
+  assert.deepEqual(odd.lines, ["::notice title=api-pace::github-token projected=0.16 pace=0.2 paced=false; an odd tick runs at half cadence"]);
+  // A run number that is no number runs the tick.
+  for (const number of ["", "4x", "-2"]) {
+    assert.equal((await run({ RUN_NUMBER: number }, low)).output, "paced=false\n", JSON.stringify(number));
+  }
+});
+
+test("any other event, an unreadable answer or a failed read is never paced", async () => {
+  let calls = 0;
+  const counting = async () => {
+    calls += 1;
+    return new Response("{}", { status: 200 });
+  };
+  assert.equal((await run({ EVENT_NAME: "workflow_dispatch" }, counting)).output, "paced=false\n");
+  assert.equal((await run({ EVENT_NAME: "push" }, counting)).output, "paced=false\n");
+  assert.equal(calls, 0, "no read off a schedule tick");
+  assert.equal((await run({}, counting)).output, "paced=false\n", "no headers");
+  assert.equal((await run({}, answer({}, 502))).output, "paced=false\n");
+  const throws = async () => {
+    throw new Error("down");
+  };
+  const failed = await run({}, throws);
+  assert.equal(failed.output, "paced=false\n");
+  assert.match(failed.lines.join("\n"), /::warning::/);
+});
+
+// The step that hands the base config's limits.api to the pacer, in
+// resolve-config's output shape, which apiLimits reads.
+const READ_STEP = "Read the pace override from the base config";
+const READ_RUN = `yq -o=json '{"limits": {"api": (.limits.api // {})}}' .base/docs/conventions/pr-pipeline.yaml >"$RUNNER_TEMP/pace-config.json"`;
+
+// The documented off switch, limits.api github-token pace: 0 in
+// pr-pipeline.yaml, through that step to main: a pool burning toward the
+// breaker on an even tick runs, with no pace notice.
+test("pace: 0 in the base config turns the pacer off", async () => {
+  const dir = tempDir("pace-off-");
+  mkdirSync(join(dir, ".base/docs/conventions"), { recursive: true });
+  const read = (yaml) => {
+    writeFileSync(join(dir, ".base/docs/conventions/pr-pipeline.yaml"), yaml);
+    const result = spawnSync("bash", ["--noprofile", "--norc", "-eo", "pipefail", "-c", READ_RUN], { cwd: dir, env: { PATH: process.env.PATH, RUNNER_TEMP: dir }, encoding: "utf8" });
+    assert.equal(result.status, 0, result.stderr);
+    return join(dir, "pace-config.json");
+  };
+  const low = answer({ "x-ratelimit-limit": "5000", "x-ratelimit-remaining": "2900", "x-ratelimit-reset": String(NOW + 1800) });
+  const off = await run({ RESOLVED_CONFIG: read("limits:\n  api:\n    github-token:\n      pace: 0\n") }, low);
+  assert.equal(off.output, "paced=false\n");
+  assert.deepEqual(off.lines, []);
+  // With no api entry the rate-limits.json default paces the same reading.
+  assert.equal((await run({ RESOLVED_CONFIG: read("limits:\n  wip-cap: 12\n") }, low)).output, "paced=true\n");
+  assert.equal(apiLimits({ resolvedPath: read("lanes: {}\n") })["github-token"].pace, 0.2, "no limits section");
+});
+
+// The timer-driven lanes that read the pacer: each reads it once on the
+// event, never fails on it, and gates every step that spends the pool.
+test("each paced timer lane reads the pacer before its work and gates every spending step on it", () => {
+  const lanes = {
+    "maintenance-reconcile-lost-runs.yml": { job: "reconcile", gated: ["Find lost runs", "Re-run lost runs"] },
+    "pr-unblock-dispatch.yml": { job: "dispatch", gated: ["Install resolve-config's dependencies", "Restore the dispatch record", "Dispatch pr-unblock for conflicting, review-tier Dependabot, failing lanes App and Markdown-changing PRs"] },
+  };
+  for (const [file, { job, gated }] of Object.entries(lanes)) {
+    const text = readFileSync(new URL(`../../workflows/${file}`, import.meta.url), "utf8");
+    const block = new RegExp(`\\n  ${job}:\\n([\\s\\S]*?)(?=\\n  [a-z0-9-]+:\\n|$)`).exec(text)?.[1] ?? "";
+    // Less the blank and comment lines that lead into the next step.
+    const steps = block.split(/\n(?= {6}- name: )/).slice(1).map((s) => ({ name: /^ {6}- name: (.*)$/m.exec(s)[1], s: s.replace(/(\n *(#[^\n]*)?)+$/, "") }));
+    const at = (name) => steps.findIndex((step) => step.name === name);
+    const pace = steps[at("Pace the tick on the GITHUB_TOKEN burn rate")]?.s ?? "";
+    // The base config's limits.api reaches the pacer, so its pace (0 for off)
+    // overrides rate-limits.json.
+    assert.ok(at(READ_STEP) >= 0 && at(READ_STEP) < at("Pace the tick on the GITHUB_TOKEN burn rate"), `${file}: the override is read before the pacer`);
+    assert.equal(steps[at(READ_STEP)].s.replace(/^[\s\S]*?\n {8}run: \|\n {10}/, ""), READ_RUN, file);
+    assert.match(steps[at(READ_STEP)].s, /\n {8}continue-on-error: true\n/, file);
+    assert.equal(/\n {8}if: (.*)\n/.exec(steps[at(READ_STEP)].s)?.[1], /\n {8}if: (.*)\n/.exec(pace)?.[1], `${file}: the read runs when the pacer does`);
+    assert.match(pace, /\n {10}RESOLVED_CONFIG: \$\{\{ runner\.temp \}\}\/pace-config\.json\n/, file);
+    assert.match(pace, /\n {8}id: pace\n/, file);
+    assert.match(pace, /\n {8}continue-on-error: true\n/, file);
+    assert.match(pace, /\n {10}EVENT_NAME: \$\{\{ github\.event_name \}\}\n/, file);
+    assert.match(pace, /\n {10}RUN_NUMBER: \$\{\{ github\.run_number \}\}\n/, file);
+    assert.match(pace, /\n {8}run: node \.base\/\.github\/actions\/log-api-budget\/pace\.mjs\n?$/, file);
+    for (const name of gated) {
+      assert.ok(at(name) > at("Pace the tick on the GITHUB_TOKEN burn rate"), `${file}: ${name} follows the pacer`);
+      assert.match(steps[at(name)].s, /steps\.pace\.outputs\.paced != 'true'/, `${file}: ${name}`);
+    }
+  }
+});

.github/actions/log-api-budget/rate-guard.mjs

@@ -53,7 +53,8 @@ const DEFAULTS = new URL("./rate-limits.json", import.meta.url);
 // belong to content-creation alone.
 const SHARE = (value) => typeof value === "number" && value >= 0 && value <= 1;
 const COUNT = (value) => Number.isInteger(value) && value >= 1;
-const FIELDS = { plan: (value) => SHARE(value) && value > 0, "writer-pause": SHARE, breaker: SHARE, "read-floor": SHARE, "limit-cap": COUNT };
+// pace, optional, is pace.mjs's share for timer-driven lanes.
+const FIELDS = { plan: (value) => SHARE(value) && value > 0, "writer-pause": SHARE, breaker: SHARE, "read-floor": SHARE, "limit-cap": COUNT, pace: SHARE };
 const CONTENT_FIELDS = { ...FIELDS, "per-minute": COUNT, "per-hour": COUNT };
 const REQUIRED = ["plan", "writer-pause", "breaker", "read-floor"];