From 990f2f6315fece75c01010a158a5588b551fca3d Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:05:42 -0400 Subject: [PATCH 1/8] perf(pr-pipeline): re-run ci-status once per head after the review lanes finish rerun-ci-status re-ran the whole ci-status job for every completed review run, two per head, though ci-status fails while the other lane's newest run has not finished. It now reads the other lane's newest run first and stops while that run is still going: its completion starts the job again, and the per-head concurrency group runs the two lanes' jobs one after the other. A head with no other-lane run re-runs on this lane alone. A ci-status that started after both lanes finished, or a re-run already queued, is left alone. The job list is read once; each 60 s poll reads only the run. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/pr-feedback-dispatch.yml | 113 ++++++++++----- docs/conventions/pr-pipeline/README.md | 9 +- scripts/ci-rerun-ci-status.test.sh | 158 ++++++++++++++++----- 3 files changed, 211 insertions(+), 69 deletions(-) diff --git a/.github/workflows/pr-feedback-dispatch.yml b/.github/workflows/pr-feedback-dispatch.yml index 7b110db041..fbe5e833f3 100644 --- a/.github/workflows/pr-feedback-dispatch.yml +++ b/.github/workflows/pr-feedback-dispatch.yml @@ -25,8 +25,8 @@ name: pr-feedback-dispatch # run either way. # # A completed review run on any same-repository PR also starts the -# rerun-ci-status job, which re-runs the head's ci-status so it reads the -# finished review (see that job). +# rerun-ci-status job, which re-runs the head's ci-status once the head's +# last review lane has finished, so it reads both reviews (see that job). # # Every completed review run at a head starts this file, so a run leaves a # record for the next one at that head in the Actions cache @@ -262,17 +262,26 @@ jobs: tokens: github-token ${{ github.token }} # ci-status's `Require both review lanes on this head` step reads the - # review runs when it runs, and a review usually finishes after it. So - # when either review caller completes on a pull request of this - # repository, this job re-runs the ci-status job of the newest - # pr-require-checks run on that head, which reads both lanes and the live - # labels again. Every PR's review starts it, not only a lanes App PR's. - # A run GitHub skipped (a draft) reviewed nothing, and a fork's review - # never runs, so neither starts it. A head the PR has moved past, or a - # closed PR, is left alone. A ci-status job not started yet reads the - # reviews when it starts, so it is left alone too; one in progress is - # polled every 60 s for at most 10 minutes, then re-run. No checkout, no - # model: the token can only read runs and the PR and re-run a job. + # newest pull_request run of each review lane on the head when it runs, and + # a review usually finishes after it. So when either review caller + # completes on a pull request of this repository, this job re-runs the + # ci-status job of the newest pr-require-checks run on that head, which + # reads both lanes and the live labels again. Every PR's review starts it, + # not only a lanes App PR's. A run GitHub skipped (a draft) reviewed + # nothing, and a fork's review never runs, so neither starts it. + # + # One re-run per head, after the last lane: ci-status fails while the + # other lane's newest run has not finished, so a re-run before then can + # only be red, and that lane's own completion starts this job again. The + # workflow's per-head concurrency group runs the two lanes' runs of this + # job one after the other, so the later one finds the earlier lane + # finished. A head with no run of the other lane (one lane only) re-runs + # on this lane alone. A head the PR has moved past, or a closed PR, is left + # alone. A ci-status job not started yet, or one that started after both + # lanes finished, reads them anyway, so it is left alone too; one that + # started earlier and is still in progress waits for its run, polled every + # 60 s for at most 10 minutes, then is re-run. No checkout, no model: the + # token can only read runs and the PR and re-run a job. # Pinned by scripts/ci-rerun-ci-status.test.sh, which runs this body. rerun-ci-status: if: >- @@ -286,7 +295,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 permissions: - actions: write # list pr-require-checks runs and jobs; re-run the ci-status job + actions: write # list review and pr-require-checks runs and jobs; re-run the ci-status job pull-requests: read # the PR's state and live head steps: - name: Re-run ci-status on the reviewed head @@ -294,6 +303,8 @@ jobs: GH_TOKEN: ${{ github.token }} HEAD_SHA: ${{ github.event.workflow_run.head_sha }} PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} + LANE_PATH: ${{ github.event.workflow_run.path }} + LANE_DONE_AT: ${{ github.event.workflow_run.updated_at }} POLL_SECONDS: '60' POLL_LIMIT_SECONDS: '600' run: | @@ -305,6 +316,19 @@ jobs: echo "::notice::the review run names no pull request; nothing to re-run." exit 0 fi + utc='^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$' + if [[ ! $LANE_DONE_AT =~ $utc ]]; then + echo "::error::the review run's finish time '$LANE_DONE_AT' is not a UTC timestamp." + exit 1 + fi + case $LANE_PATH in + .github/workflows/pr-review-hosted.yml) other='pr-review-security-hosted.yml' ;; + .github/workflows/pr-review-security-hosted.yml) other='pr-review-hosted.yml' ;; + *) + echo "::error::'$LANE_PATH' is not a review lane." + exit 1 + ;; + esac read_api() { local attempt body for attempt in 1 2 3; do @@ -316,28 +340,37 @@ jobs: sleep $((attempt * 5)) done } - pr=$(read_api "#$PR_NUMBER" "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER") || exit 1 - if [ "$(jq -r '"\(.state) \(.head.sha)"' <<<"$pr")" != "open $HEAD_SHA" ]; then - echo "#$PR_NUMBER is closed or no longer at $HEAD_SHA; nothing to re-run." - exit 0 - fi - waited=0 - while :; do - runs=$(read_api "the pr-require-checks runs on $HEAD_SHA" \ - "repos/$GITHUB_REPOSITORY/actions/workflows/pr-require-checks.yml/runs?head_sha=$HEAD_SHA&event=pull_request&per_page=100") || exit 1 - run=$(jq -c --arg sha "$HEAD_SHA" '[.workflow_runs[] - | select(.path == ".github/workflows/pr-require-checks.yml" and .head_sha == $sha and .event == "pull_request")] - | max_by(.id) // empty' <<<"$runs") - if [ -z "$run" ]; then - echo "No pr-require-checks run on $HEAD_SHA yet; its ci-status reads the reviews when it runs." + # ci-status's own choice of run: the newest pull_request run of the file on the head. + newest() { + jq -c --arg path ".github/workflows/$1" --arg sha "$HEAD_SHA" '[.workflow_runs[] + | select(.path == $path and .head_sha == $sha and .event == "pull_request")] + | max_by(.id) // empty' + } + runs=$(read_api "the $other runs on $HEAD_SHA" \ + "repos/$GITHUB_REPOSITORY/actions/workflows/$other/runs?head_sha=$HEAD_SHA&event=pull_request&per_page=100") || exit 1 + lane=$(newest "$other" <<<"$runs") + done_at=$LANE_DONE_AT + if [ -n "$lane" ]; then + if [ "$(jq -r .status <<<"$lane")" != completed ]; then + echo "$other has not finished on $HEAD_SHA; its completion re-runs ci-status." exit 0 fi - id=$(jq -r .id <<<"$run") - jobs=$(read_api "the jobs of run $id" "repos/$GITHUB_REPOSITORY/actions/runs/$id/jobs?filter=latest&per_page=100") || exit 1 - job=$(jq -c '[.jobs[] | select(.name == "ci-status")] | max_by(.id) // empty' <<<"$jobs") - if [ "$(jq -r .status <<<"$run")" = completed ]; then - break + other_at=$(jq -r '.updated_at // ""' <<<"$lane") + if [[ $other_at > $done_at ]]; then + done_at=$other_at fi + fi + runs=$(read_api "the pr-require-checks runs on $HEAD_SHA" \ + "repos/$GITHUB_REPOSITORY/actions/workflows/pr-require-checks.yml/runs?head_sha=$HEAD_SHA&event=pull_request&per_page=100") || exit 1 + run=$(newest pr-require-checks.yml <<<"$runs") + if [ -z "$run" ]; then + echo "No pr-require-checks run on $HEAD_SHA yet; its ci-status reads the reviews when it runs." + exit 0 + fi + id=$(jq -r .id <<<"$run") + jobs=$(read_api "the jobs of run $id" "repos/$GITHUB_REPOSITORY/actions/runs/$id/jobs?filter=latest&per_page=100") || exit 1 + job=$(jq -c '[.jobs[] | select(.name == "ci-status")] | max_by(.id) // empty' <<<"$jobs") + if [ "$(jq -r .status <<<"$run")" != completed ]; then case $(jq -r '.status // "absent"' <<<"${job:-null}") in in_progress | completed) ;; *) @@ -345,17 +378,31 @@ jobs: exit 0 ;; esac + fi + started_at=$(jq -r '.started_at // ""' <<<"${job:-null}") + if [[ $started_at > $done_at ]]; then + echo "ci-status of run $id started at $started_at, after both lanes finished ($done_at); it read them." + exit 0 + fi + waited=0 + while [ "$(jq -r .status <<<"$run")" != completed ]; do if [ "$waited" -ge "$POLL_LIMIT_SECONDS" ]; then echo "::error::run $id on $HEAD_SHA has not finished after ${waited}s; re-run its ci-status job once it does." exit 1 fi sleep "$POLL_SECONDS" waited=$((waited + POLL_SECONDS)) + run=$(read_api "run $id" "repos/$GITHUB_REPOSITORY/actions/runs/$id") || exit 1 done job_id=$(jq -r '.id // empty' <<<"${job:-null}") if [[ ! $job_id =~ ^[1-9][0-9]{0,14}$ ]]; then echo "::error::run $id on $HEAD_SHA has no ci-status job to re-run." exit 1 fi + pr=$(read_api "#$PR_NUMBER" "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER") || exit 1 + if [ "$(jq -r '"\(.state) \(.head.sha)"' <<<"$pr")" != "open $HEAD_SHA" ]; then + echo "#$PR_NUMBER is closed or no longer at $HEAD_SHA; nothing to re-run." + exit 0 + fi gh api -X POST "repos/$GITHUB_REPOSITORY/actions/jobs/$job_id/rerun" >/dev/null echo "Re-ran ci-status (job $job_id) of run $id on $HEAD_SHA." diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index 75f6719b2d..bcba1c34b0 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -1434,9 +1434,12 @@ them on every `pull_request` run, contract-only runs included: A review usually finishes after ci-status has read it, so `pr-feedback-dispatch.yml`'s `rerun-ci-status` job re-runs the ci-status job of the newest `pr-require-checks` run on the head -whenever either review caller completes (not when GitHub skipped it) on an open same-repository PR -still at that head. A ci-status not started yet is left to read the reviews itself; one in progress -is polled every 60 seconds for at most 10 minutes first. The merge-group run re-checks the labels. +when a review caller completes (not when GitHub skipped it) on an open same-repository PR still at +that head and the other lane's newest run on the head has finished or never ran. While the other +lane is still going a re-run could only be red, and that lane's completion re-runs it, so a head +gets one re-run, not one per lane. A ci-status not started yet, or one that started after both +lanes finished, is left to read the reviews itself; an earlier one in progress is polled every 60 +seconds for at most 10 minutes first. The merge-group run re-checks the labels. `fix-ci` keys on `ci-lanes`, so a hold or a pending review never summons it. ## Trust-root paths diff --git a/scripts/ci-rerun-ci-status.test.sh b/scripts/ci-rerun-ci-status.test.sh index 9d1bfbd649..a400346328 100755 --- a/scripts/ci-rerun-ci-status.test.sh +++ b/scripts/ci-rerun-ci-status.test.sh @@ -4,10 +4,13 @@ # shell Actions gives a step with no `shell:` (bash -e), against stub `gh` and # `sleep`. # -# When a review run completes, the job re-runs only the ci-status job of the -# newest pr-require-checks pull_request run on the reviewed head, after that -# run finishes, polling at most POLL_LIMIT_SECONDS. A closed PR, a moved head, -# no run yet or a ci-status not started is left alone. Unreadable reads fail. +# When a review run completes and the other lane's newest run on the head has +# finished or never ran, the job re-runs only the ci-status job of the newest +# pr-require-checks pull_request run on the reviewed head, after that run +# finishes, polling at most POLL_LIMIT_SECONDS. The other lane still going, a +# closed PR, a moved head, no run yet, a ci-status not started or one started +# after both lanes finished is left alone. Unreadable reads fail. Each path's +# API call count is pinned. # test-scope: .github/workflows/pr-feedback-dispatch.yml set -uo pipefail @@ -36,7 +39,8 @@ if [[ ! -s "$TMP_ROOT/body.sh" ]]; then exit 1 fi # shellcheck disable=SC2016 # the ${{ }} text is workflow source, not shell -for env in 'HEAD_SHA: ${{ github.event.workflow_run.head_sha }}' 'PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }}' "POLL_LIMIT_SECONDS: '600'"; do +for env in 'HEAD_SHA: ${{ github.event.workflow_run.head_sha }}' 'PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }}' \ + 'LANE_PATH: ${{ github.event.workflow_run.path }}' 'LANE_DONE_AT: ${{ github.event.workflow_run.updated_at }}' "POLL_LIMIT_SECONDS: '600'"; do if [[ "$step" == *"$env"* ]]; then ok "step env carries $env"; else fail "step env lacks $env"; fi done job="$(awk '/^ rerun-ci-status:$/ { on = 1; next } on && /^ [a-z]/ { exit } on { print }' "$WORKFLOW")" @@ -46,23 +50,32 @@ else fail "the job's permissions changed" fi -# The stub logs its arguments. A pulls read prints STUB_PR. A runs read prints -# the next line of STUB_RUNS_SEQ (the last line once used up), failing when -# STUB_RUNS_FAIL is set. A jobs read prints STUB_JOBS. A POST re-run fails when -# STUB_RERUN_FAIL is set. The sleep stub logs its seconds. +# The stub logs its arguments. A pulls read prints STUB_PR. A review lane's +# runs read prints STUB_LANE_RUNS and a pr-require-checks runs read STUB_RUNS, +# each failing when its *_FAIL is set. A jobs read prints STUB_JOBS. A read of +# one run prints the next line of STUB_RUN_SEQ (the last line once used up). +# A POST re-run fails when STUB_RERUN_FAIL is set. The sleep stub logs its +# seconds. mkdir -p "$TMP_ROOT/bin" cat >"$TMP_ROOT/bin/gh" <<'EOF' #!/usr/bin/env bash printf '%s\n' "$*" >>"$STUB_LOG" case "$*" in */pulls/*) printf '%s\n' "$STUB_PR" ;; +*/actions/workflows/pr-review-hosted.yml/runs* | */actions/workflows/pr-review-security-hosted.yml/runs*) + [[ -z "${STUB_LANE_FAIL:-}" ]] || exit 1 + printf '%s\n' "$STUB_LANE_RUNS" + ;; */actions/workflows/pr-require-checks.yml/runs*) [[ -z "${STUB_RUNS_FAIL:-}" ]] || exit 1 - n=$(grep -c '/runs?' "$STUB_LOG") - total=$(wc -l <<<"$STUB_RUNS_SEQ") - sed -n "$((n < total ? n : total))p" <<<"$STUB_RUNS_SEQ" + printf '%s\n' "$STUB_RUNS" ;; */actions/runs/*/jobs*) printf '%s\n' "$STUB_JOBS" ;; +"api repos/o/r/actions/runs/"[0-9]*) + n=$(grep -c '^api repos/o/r/actions/runs/[0-9]*$' "$STUB_LOG") + total=$(wc -l <<<"$STUB_RUN_SEQ") + sed -n "$((n < total ? n : total))p" <<<"$STUB_RUN_SEQ" + ;; "api -X POST "*/rerun) [[ -z "${STUB_RERUN_FAIL:-}" ]] || exit 1 echo '{}' @@ -76,19 +89,31 @@ chmod +x "$TMP_ROOT/bin/gh" "$TMP_ROOT/bin/sleep" HEAD=0123456789abcdef0123456789abcdef01234567 RCF=.github/workflows/pr-require-checks.yml +CODE=.github/workflows/pr-review-hosted.yml +SEC=.github/workflows/pr-review-security-hosted.yml +# This lane finished at DONE, the other lane at LATER; ci-status started at +# EARLY unless a case says otherwise. +EARLY=2026-10-23T09:50:00Z DONE=2026-10-23T10:00:00Z LATER=2026-10-23T10:05:00Z AFTER=2026-10-23T10:06:00Z pr() { jq -cn --arg st "${1:-open}" --arg sha "${2:-$HEAD}" '{state: $st, head: {sha: $sha}}'; } -# run [status] [sha] [path] [event] -run() { jq -cn --argjson id "$1" --arg st "${2:-completed}" --arg sha "${3:-$HEAD}" --arg p "${4:-$RCF}" --arg ev "${5:-pull_request}" \ - '{id: $id, status: $st, head_sha: $sha, path: $p, event: $ev}'; } -runs() { local IFS=,; printf '{"workflow_runs":[%s]}' "$*"; } -cijob() { jq -cn --arg st "$1" '{jobs: [{id: 7, name: "lint-repo", status: "completed"}, {id: 77, name: "ci-status", status: $st}]}'; } +# run [status] [sha] [path] [event] [updated_at] +run() { + jq -cn --argjson id "$1" --arg st "${2:-completed}" --arg sha "${3:-$HEAD}" --arg p "${4:-$RCF}" --arg ev "${5:-pull_request}" --arg at "${6:-$LATER}" \ + '{id: $id, status: $st, head_sha: $sha, path: $p, event: $ev, updated_at: $at}' +} +runs() { + local IFS=, + printf '{"workflow_runs":[%s]}' "$*" +} +# cijob [started_at] +cijob() { jq -cn --arg st "$1" --arg at "${2:-$EARLY}" '{jobs: [{id: 7, name: "lint-repo", status: "completed"}, {id: 77, name: "ci-status", status: $st, started_at: $at}]}'; } -export STUB_PR STUB_RUNS_SEQ STUB_JOBS +export STUB_PR STUB_LANE_RUNS STUB_RUNS STUB_JOBS STUB_RUN_SEQ expect() { local label="$1" want_rc="$2" out rc : >"$TMP_ROOT/log" out="$(PATH="$TMP_ROOT/bin:$PATH" STUB_LOG="$TMP_ROOT/log" GITHUB_REPOSITORY=o/r HEAD_SHA="${HEAD_SHA-$HEAD}" \ - PR_NUMBER="${PR_NUMBER-42}" POLL_SECONDS=60 POLL_LIMIT_SECONDS=600 bash -e "$TMP_ROOT/body.sh" 2>&1)" && rc=0 || rc=$? + PR_NUMBER="${PR_NUMBER-42}" LANE_PATH="${LANE_PATH-$CODE}" LANE_DONE_AT="${LANE_DONE_AT-$DONE}" \ + POLL_SECONDS=60 POLL_LIMIT_SECONDS=600 bash -e "$TMP_ROOT/body.sh" 2>&1)" && rc=0 || rc=$? if [[ "$rc" -ne "$want_rc" ]]; then fail "$label: expected rc=$want_rc got rc=$rc out='$out'" elif [[ -n "${3:-}" && "$out" != *"$3"* ]]; then @@ -105,56 +130,123 @@ check_reran() { fail "$2: re-runs: $(grep '^api -X POST' "$TMP_ROOT/log" | paste -sd'|')" fi } +# The API calls the last case made, against the count its path is pinned to. +check_calls() { + local n + n=$(grep -c '^api ' "$TMP_ROOT/log") + if [[ "$n" == "$1" ]]; then ok "$2 ($1 API calls)"; else fail "$2: $n API calls, want $1: $(paste -sd'|' "$TMP_ROOT/log")"; fi +} -STUB_PR="$(pr)" STUB_RUNS_SEQ="$(runs "$(run 5)" "$(run 9)" "$(run 8)")" STUB_JOBS="$(cijob completed)" -expect "a finished run's ci-status is re-run" 0 "Re-ran ci-status (job 77) of run 9 on $HEAD." +# Both lanes finished: the head's last lane re-runs ci-status once. +STUB_PR="$(pr)" STUB_LANE_RUNS="$(runs "$(run 3 completed "$HEAD" "$SEC")")" +STUB_RUNS="$(runs "$(run 5)" "$(run 9)" "$(run 8)")" STUB_JOBS="$(cijob completed)" STUB_RUN_SEQ='' +expect "both lanes finished: a finished run's ci-status is re-run" 0 "Re-ran ci-status (job 77) of run 9 on $HEAD." check_reran 1 "only the ci-status job of the newest run is re-run" -if grep -qxF "api repos/o/r/actions/workflows/pr-require-checks.yml/runs?head_sha=$HEAD&event=pull_request&per_page=100" "$TMP_ROOT/log" && +check_calls 5 "the last lane reads the other lane, the runs, the jobs and the PR, then re-runs" +if grep -qxF "api repos/o/r/actions/workflows/pr-review-security-hosted.yml/runs?head_sha=$HEAD&event=pull_request&per_page=100" "$TMP_ROOT/log" && + grep -qxF "api repos/o/r/actions/workflows/pr-require-checks.yml/runs?head_sha=$HEAD&event=pull_request&per_page=100" "$TMP_ROOT/log" && grep -qxF "api repos/o/r/actions/runs/9/jobs?filter=latest&per_page=100" "$TMP_ROOT/log"; then - ok "the head's pull_request runs and the newest run's latest jobs are read" + ok "the other lane's, then the head's pull_request runs and the newest run's latest jobs are read" else fail "unexpected reads: '$(sort -u "$TMP_ROOT/log")'" fi -STUB_RUNS_SEQ="$(runs "$(run 5)" "$(run 9 completed "${HEAD//0/f}")" "$(run 10 completed "$HEAD" .github/workflows/other.yml)" "$(run 11 completed "$HEAD" "$RCF" merge_group)")" +STUB_LANE_RUNS="$(runs "$(run 4 completed "$HEAD" "$CODE")")" +LANE_PATH=$SEC expect "the security lane finishing last reads the code lane" 0 "Re-ran ci-status (job 77)" +if grep -q '/actions/workflows/pr-review-hosted.yml/runs?' "$TMP_ROOT/log"; then ok "the security lane's run reads the code lane's runs"; else fail "reads: $(paste -sd'|' "$TMP_ROOT/log")"; fi +STUB_RUNS="$(runs "$(run 5)" "$(run 9 completed "${HEAD//0/f}")" "$(run 10 completed "$HEAD" .github/workflows/other.yml)" "$(run 11 completed "$HEAD" "$RCF" merge_group)")" expect "runs on another head, of another file or event are not the newest" 0 "of run 5 on $HEAD" +STUB_RUNS="$(runs "$(run 9)")" + +# The other lane still going: no re-run, one API call; its completion re-runs ci-status. +for st in in_progress queued waiting; do + STUB_LANE_RUNS="$(runs "$(run 2 completed "$HEAD" "$SEC")" "$(run 3 "$st" "$HEAD" "$SEC")")" + expect "the other lane's newest run $st: left for its completion" 0 "pr-review-security-hosted.yml has not finished on $HEAD; its completion re-runs ci-status." + check_reran 0 "nothing is re-run while the other lane is $st" + check_calls 1 "the first lane to finish stops after one read" +done +STUB_LANE_RUNS="$(runs "$(run 3 in_progress "${HEAD//0/f}" "$SEC")" "$(run 4 in_progress "$HEAD" "$SEC" merge_group)" "$(run 5 in_progress "$HEAD" "$CODE")")" +expect "other heads, events or files are not the other lane's newest run" 0 "Re-ran ci-status (job 77)" +check_reran 1 "a head with no other-lane run of its own is re-run" + +# One lane only on the head (an exempt author, a lane that never ran), or the +# other lane skipped, cancelled or failed: this lane is the last, so re-run. +STUB_LANE_RUNS="$(runs)" +expect "no run of the other lane: re-run on this lane alone" 0 "Re-ran ci-status (job 77) of run 9" +check_reran 1 "a single-lane head is re-run" +check_calls 5 "a single-lane head costs the same as the last lane" +for c in skipped cancelled failure; do + STUB_LANE_RUNS="$(runs "$(jq -c --arg c "$c" '. + {conclusion: $c}' <<<"$(run 3 completed "$HEAD" "$SEC")")")" + expect "the other lane completed $c counts as finished" 0 "Re-ran ci-status (job 77)" + check_reran 1 "the other lane $c: re-run" +done +STUB_LANE_RUNS="$(runs "$(run 3 completed "$HEAD" "$SEC")")" + +# ci-status that already read both finished lanes is left alone. +STUB_JOBS="$(cijob completed "$AFTER")" +expect "a ci-status that started after both lanes finished is left alone" 0 "started at $AFTER, after both lanes finished ($LATER)" +check_reran 0 "a fresh ci-status is not re-run" +check_calls 3 "a fresh ci-status costs no PR read and no re-run" +STUB_JOBS="$(cijob completed 2026-10-23T10:03:00Z)" +expect "a ci-status that started between the lanes is re-run" 0 "Re-ran ci-status (job 77)" +STUB_LANE_RUNS="$(runs "$(run 3 completed "$HEAD" "$SEC" pull_request 2026-10-23T09:40:00Z)")" STUB_JOBS="$(cijob completed 2026-10-23T09:55:00Z)" +expect "a ci-status that started before this lane finished is re-run" 0 "Re-ran ci-status (job 77)" +STUB_LANE_RUNS="$(runs "$(run 3 completed "$HEAD" "$SEC")")" STUB_JOBS="$(cijob completed "$LATER")" +expect "a ci-status that started the second the last lane finished is re-run" 0 "Re-ran ci-status (job 77)" +STUB_RUNS="$(runs "$(run 9 queued)")" STUB_JOBS="$(cijob queued "")" +expect "a ci-status re-run already queued is left alone" 0 "ci-status of run 9 has not started" +check_reran 0 "the second of two racing lanes adds no re-run" +check_calls 3 "a queued ci-status costs no PR read" +STUB_JOBS="$(cijob in_progress "$AFTER")" STUB_RUNS="$(runs "$(run 9 in_progress)")" +expect "a ci-status in progress that started after both lanes is left alone" 0 "after both lanes finished" +if grep -q '^sleep' "$TMP_ROOT/log"; then fail "a fresh ci-status was polled"; else ok "a fresh ci-status in progress is not polled"; fi -STUB_RUNS_SEQ="$(runs "$(run 9 in_progress)") -$(runs "$(run 9 in_progress)") -$(runs "$(run 9)")" STUB_JOBS="$(cijob in_progress)" +# A stale ci-status in progress waits for its run, reading only the run. +STUB_RUNS="$(runs "$(run 9 in_progress)")" STUB_JOBS="$(cijob in_progress)" +STUB_RUN_SEQ="$(run 9 in_progress) +$(run 9)" expect "a ci-status in progress is polled, then re-run once its run finishes" 0 "Re-ran ci-status (job 77) of run 9" check_reran 1 "the polled run's ci-status is re-run once" if [[ "$(grep -c '^sleep 60$' "$TMP_ROOT/log")" == 2 ]]; then ok "it polled every 60 s"; else fail "polls: $(grep '^sleep' "$TMP_ROOT/log" | paste -sd'|')"; fi -STUB_RUNS_SEQ="$(runs "$(run 9 in_progress)")" +check_calls 7 "each poll reads the run only" +if [[ "$(grep -c '/jobs?' "$TMP_ROOT/log")" == 1 ]]; then ok "the job list is read once"; else fail "job list reads: $(grep -c '/jobs?' "$TMP_ROOT/log")"; fi +STUB_RUN_SEQ="$(run 9 in_progress)" expect "a run still going after the limit fails" 1 "::error::run 9 on $HEAD has not finished after 600s" check_reran 0 "nothing is re-run past the poll limit" if [[ "$(grep -c '^sleep 60$' "$TMP_ROOT/log")" == 10 ]]; then ok "it stops after 10 minutes of polls"; else fail "polls: $(grep -c '^sleep 60$' "$TMP_ROOT/log")"; fi +check_calls 13 "a 10-minute wait reads the run 10 times" for st in queued waiting pending; do - STUB_JOBS="$(cijob "$st")" + STUB_JOBS="$(cijob "$st" "")" expect "a ci-status $st is left to read the reviews when it starts" 0 "ci-status of run 9 has not started" check_reran 0 "a $st ci-status is not re-run" done STUB_JOBS='{"jobs":[]}' expect "a run with no ci-status job listed yet is left alone" 0 "has not started" -STUB_RUNS_SEQ="$(runs "$(run 9)")" +STUB_RUNS="$(runs "$(run 9)")" expect "a finished run with no ci-status job fails" 1 "::error::run 9 on $HEAD has no ci-status job to re-run." -STUB_RUNS_SEQ="$(runs)" STUB_JOBS="$(cijob completed)" +STUB_RUNS="$(runs)" STUB_JOBS="$(cijob completed)" expect "no run on the head yet is left alone" 0 "No pr-require-checks run on $HEAD yet" check_reran 0 "nothing is re-run with no run" -STUB_RUNS_SEQ="$(runs "$(run 9)")" +STUB_RUNS="$(runs "$(run 9)")" STUB_PR="$(pr closed)" expect "a closed PR is left alone" 0 "#42 is closed or no longer at $HEAD" check_reran 0 "a closed PR's ci-status is not re-run" STUB_PR="$(pr open "${HEAD//0/f}")" expect "a head the PR moved past is left alone" 0 "no longer at $HEAD" -if grep -q '/runs?' "$TMP_ROOT/log"; then fail "a moved head's runs were read"; else ok "a moved head reads no runs"; fi +check_reran 0 "a moved head's ci-status is not re-run" STUB_PR="$(pr)" PR_NUMBER='' expect "a review run with no pull request is left alone" 0 "names no pull request" if [[ -s "$TMP_ROOT/log" ]]; then fail "a run with no pull request read the API"; else ok "a run with no pull request reads nothing"; fi PR_NUMBER='1;id' expect "a pull request number that is not a number is left alone" 0 "names no pull request" HEAD_SHA=main expect "a head that is not a commit SHA fails" 1 "::error::the review run's head 'main' is not a commit SHA." +LANE_PATH=.github/workflows/pr-verify.yml expect "a run of another file fails" 1 "::error::'.github/workflows/pr-verify.yml' is not a review lane." +LANE_DONE_AT='2026-10-23 10:00' expect "a finish time that is not a UTC timestamp fails" 1 "is not a UTC timestamp." +if [[ -s "$TMP_ROOT/log" ]]; then fail "a bad finish time read the API"; else ok "a bad finish time reads nothing"; fi +export STUB_LANE_FAIL=1 +expect "an unreadable other-lane run list fails" 1 "::error::could not read the pr-review-security-hosted.yml runs on $HEAD; re-run this run." +unset STUB_LANE_FAIL export STUB_RUNS_FAIL=1 expect "an unreadable run list fails" 1 "::error::could not read the pr-require-checks runs on $HEAD; re-run this run." unset STUB_RUNS_FAIL From d5b9b368f357b7fc392f1a69543aaa8ed27e889d Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:06:38 -0400 Subject: [PATCH 2/8] chore(pr-pipeline): add the changelog fragment for fewer ci-status re-runs Co-Authored-By: Claude Opus 5.5 --- .../perf-feedback-dispatch-fewer-ci-reruns-3e7a91c4.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/pr-pipeline/perf-feedback-dispatch-fewer-ci-reruns-3e7a91c4.md diff --git a/.changes/pr-pipeline/perf-feedback-dispatch-fewer-ci-reruns-3e7a91c4.md b/.changes/pr-pipeline/perf-feedback-dispatch-fewer-ci-reruns-3e7a91c4.md new file mode 100644 index 0000000000..dcad0a6a80 --- /dev/null +++ b/.changes/pr-pipeline/perf-feedback-dispatch-fewer-ci-reruns-3e7a91c4.md @@ -0,0 +1,7 @@ +--- +bump: patch +--- + +### Changed + +- `pr-feedback-dispatch.yml`'s `rerun-ci-status` re-runs a head's ci-status once, after the last review lane finishes, instead of once per lane: while the other lane's newest run on the head is still going it stops after one read, since ci-status would still be red and that lane's completion starts the job again. A head with no run of the other lane re-runs on one lane. A ci-status that started after both lanes finished, or a re-run already queued, is left alone. The job list is read once, and each 60-second poll reads only the run. From 3fa371c38f7f2c761204da7620b6d962df96c325 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:12:44 -0400 Subject: [PATCH 3/8] fix(pr-pipeline): keep a 60 s margin before trusting a fresh ci-status A ci-status job counts as having read both finished lanes only when it started more than 60 s after the later one finished, so a lagging runs read costs an extra re-run, not a stale result. The job comment now gives the right reason the later lane finds the earlier one finished. The test pins that only the other lane's newest run decides. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/pr-feedback-dispatch.yml | 19 +++++++++++-------- scripts/ci-rerun-ci-status.test.sh | 12 ++++++++++-- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr-feedback-dispatch.yml b/.github/workflows/pr-feedback-dispatch.yml index fbe5e833f3..7dd6ddbbea 100644 --- a/.github/workflows/pr-feedback-dispatch.yml +++ b/.github/workflows/pr-feedback-dispatch.yml @@ -272,13 +272,15 @@ jobs: # # One re-run per head, after the last lane: ci-status fails while the # other lane's newest run has not finished, so a re-run before then can - # only be red, and that lane's own completion starts this job again. The - # workflow's per-head concurrency group runs the two lanes' runs of this - # job one after the other, so the later one finds the earlier lane - # finished. A head with no run of the other lane (one lane only) re-runs - # on this lane alone. A head the PR has moved past, or a closed PR, is left - # alone. A ci-status job not started yet, or one that started after both - # lanes finished, reads them anyway, so it is left alone too; one that + # only be red, and that lane's own completion starts this job again. Each + # run of this job starts after its own lane completed, so the later lane's + # run finds the earlier lane finished; the per-head concurrency group keeps + # the two from re-running ci-status at once. A head with no run of the + # other lane (one lane only) re-runs on this lane alone. A head the PR has + # moved past, or a closed PR, is left alone. A ci-status job not started + # yet, or one that started more than 60 s after both lanes finished (the + # margin covers a lagging read), reads them anyway, so it is left alone + # too; one that # started earlier and is still in progress waits for its run, polled every # 60 s for at most 10 minutes, then is re-run. No checkout, no model: the # token can only read runs and the PR and re-run a job. @@ -380,7 +382,8 @@ jobs: esac fi started_at=$(jq -r '.started_at // ""' <<<"${job:-null}") - if [[ $started_at > $done_at ]]; then + if [ "$(jq -r --arg s "$started_at" --arg d "$done_at" \ + 'try (($s | fromdateiso8601) > ($d | fromdateiso8601) + 60) catch false' << [status] [sha] [path] [event] [updated_at] run() { @@ -165,6 +165,9 @@ for st in in_progress queued waiting; do check_calls 1 "the first lane to finish stops after one read" done STUB_LANE_RUNS="$(runs "$(run 3 in_progress "${HEAD//0/f}" "$SEC")" "$(run 4 in_progress "$HEAD" "$SEC" merge_group)" "$(run 5 in_progress "$HEAD" "$CODE")")" +STUB_LANE_RUNS="$(runs "$(run 3 in_progress "$HEAD" "$SEC")" "$(run 4 completed "$HEAD" "$SEC")")" +expect "an older other-lane run still going does not hold the newest, finished one" 0 "Re-ran ci-status (job 77)" +check_reran 1 "only the other lane's newest run decides" expect "other heads, events or files are not the other lane's newest run" 0 "Re-ran ci-status (job 77)" check_reran 1 "a head with no other-lane run of its own is re-run" @@ -192,6 +195,11 @@ STUB_LANE_RUNS="$(runs "$(run 3 completed "$HEAD" "$SEC" pull_request 2026-10-23 expect "a ci-status that started before this lane finished is re-run" 0 "Re-ran ci-status (job 77)" STUB_LANE_RUNS="$(runs "$(run 3 completed "$HEAD" "$SEC")")" STUB_JOBS="$(cijob completed "$LATER")" expect "a ci-status that started the second the last lane finished is re-run" 0 "Re-ran ci-status (job 77)" +STUB_JOBS="$(cijob completed 2026-10-23T10:06:00Z)" +expect "a ci-status that started within 60 s of the last lane is re-run" 0 "Re-ran ci-status (job 77)" +STUB_JOBS="$(cijob completed "$AFTER")" STUB_LANE_RUNS="$(runs "$(run 3 completed "$HEAD" "$SEC" pull_request 2026-10-23T10:05)")" +expect "an unreadable finish time of the other lane means a re-run" 0 "Re-ran ci-status (job 77)" +STUB_LANE_RUNS="$(runs "$(run 3 completed "$HEAD" "$SEC")")" STUB_RUNS="$(runs "$(run 9 queued)")" STUB_JOBS="$(cijob queued "")" expect "a ci-status re-run already queued is left alone" 0 "ci-status of run 9 has not started" check_reran 0 "the second of two racing lanes adds no re-run" From 02b117afd16f5277b028fe8e33e552b2862b2c3a Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:31:13 -0400 Subject: [PATCH 4/8] fix(pr-pipeline): wait out a primary rate limit in rerun-ci-status A read refused by the token's primary rate limit (403 or 429 with x-ratelimit-remaining 0) now waits until one second past x-ratelimit-reset when that fits in what is left of the job's 10-minute cap, shared with the ci-status poll, then reads again. A reset past the cap ends the job green with a warning: the next review lane completion re-runs ci-status. Before, the job failed inside the empty window and asked for a re-run that would spend the same empty bucket. Co-Authored-By: Claude Opus 5.5 --- ...dback-dispatch-fewer-ci-reruns-3e7a91c4.md | 2 +- .github/workflows/pr-feedback-dispatch.yml | 48 ++++++++++---- docs/conventions/pr-pipeline/README.md | 4 +- scripts/ci-rerun-ci-status.test.sh | 63 ++++++++++++++----- 4 files changed, 88 insertions(+), 29 deletions(-) diff --git a/.changes/pr-pipeline/perf-feedback-dispatch-fewer-ci-reruns-3e7a91c4.md b/.changes/pr-pipeline/perf-feedback-dispatch-fewer-ci-reruns-3e7a91c4.md index dcad0a6a80..cf24c7e6df 100644 --- a/.changes/pr-pipeline/perf-feedback-dispatch-fewer-ci-reruns-3e7a91c4.md +++ b/.changes/pr-pipeline/perf-feedback-dispatch-fewer-ci-reruns-3e7a91c4.md @@ -4,4 +4,4 @@ bump: patch ### Changed -- `pr-feedback-dispatch.yml`'s `rerun-ci-status` re-runs a head's ci-status once, after the last review lane finishes, instead of once per lane: while the other lane's newest run on the head is still going it stops after one read, since ci-status would still be red and that lane's completion starts the job again. A head with no run of the other lane re-runs on one lane. A ci-status that started after both lanes finished, or a re-run already queued, is left alone. The job list is read once, and each 60-second poll reads only the run. +- `pr-feedback-dispatch.yml`'s `rerun-ci-status` re-runs a head's ci-status once, after the last review lane finishes, instead of once per lane: while the other lane's newest run on the head is still going it stops after one read, since ci-status would still be red and that lane's completion starts the job again. A head with no run of the other lane re-runs on one lane. A ci-status that started after both lanes finished, or a re-run already queued, is left alone. The job list is read once, and each 60-second poll reads only the run. A read refused by the token's primary rate limit (403 or 429 with `x-ratelimit-remaining: 0`) waits until one second past `x-ratelimit-reset` inside the same 10-minute cap instead of failing; a reset past the cap ends the job green with a warning. diff --git a/.github/workflows/pr-feedback-dispatch.yml b/.github/workflows/pr-feedback-dispatch.yml index 7dd6ddbbea..d0ce89b4cb 100644 --- a/.github/workflows/pr-feedback-dispatch.yml +++ b/.github/workflows/pr-feedback-dispatch.yml @@ -331,16 +331,41 @@ jobs: exit 1 ;; esac + # read_api : GET into . A primary rate + # limit (403 or 429 with x-ratelimit-remaining 0) waits until one + # second past x-ratelimit-reset when that fits in what is left of + # POLL_LIMIT_SECONDS, shared with the poll below; a reset past it + # ends the job green, since the next lane completion re-runs + # ci-status. Any other failure is read three times, then fails. + waited=0 read_api() { - local attempt body + local attempt out hdr status left reset wait for attempt in 1 2 3; do - if body=$(gh api "$2"); then - printf '%s\n' "$body" + if out=$(gh api -i "$3"); then + printf -v "$1" '%s' "$(sed '1,/^\r\?$/d' <<<"$out")" return 0 fi - [ "$attempt" -lt 3 ] || { echo "::error::could not read $1; re-run this run." >&2; return 1; } + hdr=$(sed '/^\r\?$/q' <<<"$out") + status=$(awk 'NR == 1 { print $2 }' <<<"$hdr") + left=$(awk -F': *' 'tolower($1) == "x-ratelimit-remaining" { sub(/\r$/, "", $2); print $2; exit }' <<<"$hdr") + reset=$(awk -F': *' 'tolower($1) == "x-ratelimit-reset" { sub(/\r$/, "", $2); print $2; exit }' <<<"$hdr") + if [[ $status =~ ^(403|429)$ && $left == 0 && $reset =~ ^[0-9]+$ ]]; then + wait=$((reset - $(date +%s))) + wait=$((wait < 0 ? 1 : wait + 1)) + if [ $((waited + wait)) -gt "$POLL_LIMIT_SECONDS" ]; then + echo "::warning::the token's rate limit is spent until $(date -u -d "@$reset" +%H:%M:%SZ), past this job's ${POLL_LIMIT_SECONDS}s cap; the next review lane completion re-runs ci-status." + exit 0 + fi + echo "Rate limited reading $2; waiting ${wait}s for the reset." + sleep "$wait" + waited=$((waited + wait)) + continue + fi + [ "$attempt" -lt 3 ] || { echo "::error::could not read $2; re-run this run." >&2; exit 1; } sleep $((attempt * 5)) done + echo "::error::could not read $2 after the rate-limit waits; re-run this run." >&2 + exit 1 } # ci-status's own choice of run: the newest pull_request run of the file on the head. newest() { @@ -348,8 +373,8 @@ jobs: | select(.path == $path and .head_sha == $sha and .event == "pull_request")] | max_by(.id) // empty' } - runs=$(read_api "the $other runs on $HEAD_SHA" \ - "repos/$GITHUB_REPOSITORY/actions/workflows/$other/runs?head_sha=$HEAD_SHA&event=pull_request&per_page=100") || exit 1 + read_api runs "the $other runs on $HEAD_SHA" \ + "repos/$GITHUB_REPOSITORY/actions/workflows/$other/runs?head_sha=$HEAD_SHA&event=pull_request&per_page=100" lane=$(newest "$other" <<<"$runs") done_at=$LANE_DONE_AT if [ -n "$lane" ]; then @@ -362,15 +387,15 @@ jobs: done_at=$other_at fi fi - runs=$(read_api "the pr-require-checks runs on $HEAD_SHA" \ - "repos/$GITHUB_REPOSITORY/actions/workflows/pr-require-checks.yml/runs?head_sha=$HEAD_SHA&event=pull_request&per_page=100") || exit 1 + read_api runs "the pr-require-checks runs on $HEAD_SHA" \ + "repos/$GITHUB_REPOSITORY/actions/workflows/pr-require-checks.yml/runs?head_sha=$HEAD_SHA&event=pull_request&per_page=100" run=$(newest pr-require-checks.yml <<<"$runs") if [ -z "$run" ]; then echo "No pr-require-checks run on $HEAD_SHA yet; its ci-status reads the reviews when it runs." exit 0 fi id=$(jq -r .id <<<"$run") - jobs=$(read_api "the jobs of run $id" "repos/$GITHUB_REPOSITORY/actions/runs/$id/jobs?filter=latest&per_page=100") || exit 1 + read_api jobs "the jobs of run $id" "repos/$GITHUB_REPOSITORY/actions/runs/$id/jobs?filter=latest&per_page=100" job=$(jq -c '[.jobs[] | select(.name == "ci-status")] | max_by(.id) // empty' <<<"$jobs") if [ "$(jq -r .status <<<"$run")" != completed ]; then case $(jq -r '.status // "absent"' <<<"${job:-null}") in @@ -387,7 +412,6 @@ jobs: echo "ci-status of run $id started at $started_at, after both lanes finished ($done_at); it read them." exit 0 fi - waited=0 while [ "$(jq -r .status <<<"$run")" != completed ]; do if [ "$waited" -ge "$POLL_LIMIT_SECONDS" ]; then echo "::error::run $id on $HEAD_SHA has not finished after ${waited}s; re-run its ci-status job once it does." @@ -395,14 +419,14 @@ jobs: fi sleep "$POLL_SECONDS" waited=$((waited + POLL_SECONDS)) - run=$(read_api "run $id" "repos/$GITHUB_REPOSITORY/actions/runs/$id") || exit 1 + read_api run "run $id" "repos/$GITHUB_REPOSITORY/actions/runs/$id" done job_id=$(jq -r '.id // empty' <<<"${job:-null}") if [[ ! $job_id =~ ^[1-9][0-9]{0,14}$ ]]; then echo "::error::run $id on $HEAD_SHA has no ci-status job to re-run." exit 1 fi - pr=$(read_api "#$PR_NUMBER" "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER") || exit 1 + read_api pr "#$PR_NUMBER" "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" if [ "$(jq -r '"\(.state) \(.head.sha)"' <<<"$pr")" != "open $HEAD_SHA" ]; then echo "#$PR_NUMBER is closed or no longer at $HEAD_SHA; nothing to re-run." exit 0 diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index ed2ea92019..4345126b81 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -1451,7 +1451,9 @@ that head and the other lane's newest run on the head has finished or never ran. lane is still going a re-run could only be red, and that lane's completion re-runs it, so a head gets one re-run, not one per lane. A ci-status not started yet, or one that started after both lanes finished, is left to read the reviews itself; an earlier one in progress is polled every 60 -seconds for at most 10 minutes first. The merge-group run re-checks the labels. +seconds for at most 10 minutes first. A read refused by the token's primary rate limit waits for +the reset inside the same 10 minutes; a reset past them ends the job green, and the next lane +completion re-runs ci-status. The merge-group run re-checks the labels. `fix-ci` keys on `ci-lanes`, so a hold or a pending review never summons it. ## Trust-root paths diff --git a/scripts/ci-rerun-ci-status.test.sh b/scripts/ci-rerun-ci-status.test.sh index 58cfd2962e..f7de8ee997 100755 --- a/scripts/ci-rerun-ci-status.test.sh +++ b/scripts/ci-rerun-ci-status.test.sh @@ -9,8 +9,9 @@ # pr-require-checks pull_request run on the reviewed head, after that run # finishes, polling at most POLL_LIMIT_SECONDS. The other lane still going, a # closed PR, a moved head, no run yet, a ci-status not started or one started -# after both lanes finished is left alone. Unreadable reads fail. Each path's -# API call count is pinned. +# after both lanes finished is left alone. A primary rate limit waits for its +# reset within the cap, or ends green past it. Unreadable reads fail. Each +# path's API call count is pinned. # test-scope: .github/workflows/pr-feedback-dispatch.yml set -uo pipefail @@ -54,27 +55,37 @@ fi # runs read prints STUB_LANE_RUNS and a pr-require-checks runs read STUB_RUNS, # each failing when its *_FAIL is set. A jobs read prints STUB_JOBS. A read of # one run prints the next line of STUB_RUN_SEQ (the last line once used up). -# A POST re-run fails when STUB_RERUN_FAIL is set. The sleep stub logs its -# seconds. +# A POST re-run fails when STUB_RERUN_FAIL is set. While STUB_LIMIT_FILE +# holds a count above 0, a pr-require-checks runs read is refused as a +# primary rate limit resetting at STUB_RESET, and the count drops by one. The +# sleep stub logs its seconds. mkdir -p "$TMP_ROOT/bin" cat >"$TMP_ROOT/bin/gh" <<'EOF' #!/usr/bin/env bash printf '%s\n' "$*" >>"$STUB_LOG" +# A GET read with -i prints a status line and a blank line before the body. +ok200() { printf 'HTTP/2.0 200 OK\r\nX-Ratelimit-Remaining: 9\r\n\r\n%s\n' "$1"; } +if [[ -n "${STUB_LIMIT_FILE:-}" && "$*" == *"/actions/workflows/pr-require-checks.yml/runs"* ]] && + [[ "$(cat "$STUB_LIMIT_FILE")" -gt 0 ]]; then + echo $(($(cat "$STUB_LIMIT_FILE") - 1)) >"$STUB_LIMIT_FILE" + printf 'HTTP/2.0 403 Forbidden\r\nX-Ratelimit-Remaining: 0\r\nX-Ratelimit-Reset: %s\r\n\r\n{"message":"API rate limit exceeded for installation"}\n' "$STUB_RESET" + exit 1 +fi case "$*" in -*/pulls/*) printf '%s\n' "$STUB_PR" ;; +*/pulls/*) ok200 "$STUB_PR" ;; */actions/workflows/pr-review-hosted.yml/runs* | */actions/workflows/pr-review-security-hosted.yml/runs*) [[ -z "${STUB_LANE_FAIL:-}" ]] || exit 1 - printf '%s\n' "$STUB_LANE_RUNS" + ok200 "$STUB_LANE_RUNS" ;; */actions/workflows/pr-require-checks.yml/runs*) [[ -z "${STUB_RUNS_FAIL:-}" ]] || exit 1 - printf '%s\n' "$STUB_RUNS" + ok200 "$STUB_RUNS" ;; -*/actions/runs/*/jobs*) printf '%s\n' "$STUB_JOBS" ;; -"api repos/o/r/actions/runs/"[0-9]*) - n=$(grep -c '^api repos/o/r/actions/runs/[0-9]*$' "$STUB_LOG") +*/actions/runs/*/jobs*) ok200 "$STUB_JOBS" ;; +"api -i repos/o/r/actions/runs/"[0-9]*) + n=$(grep -c '^api -i repos/o/r/actions/runs/[0-9]*$' "$STUB_LOG") total=$(wc -l <<<"$STUB_RUN_SEQ") - sed -n "$((n < total ? n : total))p" <<<"$STUB_RUN_SEQ" + ok200 "$(sed -n "$((n < total ? n : total))p" <<<"$STUB_RUN_SEQ")" ;; "api -X POST "*/rerun) [[ -z "${STUB_RERUN_FAIL:-}" ]] || exit 1 @@ -85,7 +96,11 @@ esac EOF # shellcheck disable=SC2016 # the stub expands $* and $STUB_LOG when it runs printf '#!/usr/bin/env bash\necho "sleep $*" >>"$STUB_LOG"\n' >"$TMP_ROOT/bin/sleep" -chmod +x "$TMP_ROOT/bin/gh" "$TMP_ROOT/bin/sleep" +# The clock reads 1000 s past the epoch; any other date call runs the real date. +REAL_DATE="$(command -v date)" +# shellcheck disable=SC2016 # the stub expands its arguments when it runs +printf '#!/usr/bin/env bash\nif [ "$*" = +%%s ]; then echo 1000; else exec %q "$@"; fi\n' "$REAL_DATE" >"$TMP_ROOT/bin/date" +chmod +x "$TMP_ROOT/bin/gh" "$TMP_ROOT/bin/sleep" "$TMP_ROOT/bin/date" HEAD=0123456789abcdef0123456789abcdef01234567 RCF=.github/workflows/pr-require-checks.yml @@ -143,9 +158,9 @@ STUB_RUNS="$(runs "$(run 5)" "$(run 9)" "$(run 8)")" STUB_JOBS="$(cijob complete expect "both lanes finished: a finished run's ci-status is re-run" 0 "Re-ran ci-status (job 77) of run 9 on $HEAD." check_reran 1 "only the ci-status job of the newest run is re-run" check_calls 5 "the last lane reads the other lane, the runs, the jobs and the PR, then re-runs" -if grep -qxF "api repos/o/r/actions/workflows/pr-review-security-hosted.yml/runs?head_sha=$HEAD&event=pull_request&per_page=100" "$TMP_ROOT/log" && - grep -qxF "api repos/o/r/actions/workflows/pr-require-checks.yml/runs?head_sha=$HEAD&event=pull_request&per_page=100" "$TMP_ROOT/log" && - grep -qxF "api repos/o/r/actions/runs/9/jobs?filter=latest&per_page=100" "$TMP_ROOT/log"; then +if grep -qxF "api -i repos/o/r/actions/workflows/pr-review-security-hosted.yml/runs?head_sha=$HEAD&event=pull_request&per_page=100" "$TMP_ROOT/log" && + grep -qxF "api -i repos/o/r/actions/workflows/pr-require-checks.yml/runs?head_sha=$HEAD&event=pull_request&per_page=100" "$TMP_ROOT/log" && + grep -qxF "api -i repos/o/r/actions/runs/9/jobs?filter=latest&per_page=100" "$TMP_ROOT/log"; then ok "the other lane's, then the head's pull_request runs and the newest run's latest jobs are read" else fail "unexpected reads: '$(sort -u "$TMP_ROOT/log")'" @@ -252,6 +267,24 @@ LANE_PATH=.github/workflows/pr-verify.yml expect "a run of another file fails" 1 LANE_DONE_AT='2026-10-23 10:00' expect "a finish time that is not a UTC timestamp fails" 1 "is not a UTC timestamp." if [[ -s "$TMP_ROOT/log" ]]; then fail "a bad finish time read the API"; else ok "a bad finish time reads nothing"; fi +# A primary rate limit waits for its reset inside the cap, then reads again. +export STUB_LIMIT_FILE="$TMP_ROOT/limit" STUB_RESET=1119 +STUB_PR="$(pr)" STUB_LANE_RUNS="$(runs "$(run 3 completed "$HEAD" "$SEC")")" STUB_RUNS="$(runs "$(run 9)")" STUB_JOBS="$(cijob completed)" +echo 1 >"$STUB_LIMIT_FILE" +expect "a rate-limited read waits for the reset, then re-runs" 0 "Rate limited reading the pr-require-checks runs on $HEAD; waiting 120s for the reset." +check_reran 1 "the read after the reset goes on to the re-run" +if [[ "$(grep -c '^sleep 120$' "$TMP_ROOT/log")" == 1 ]]; then ok "it slept until one second past the reset"; else fail "sleeps: $(grep '^sleep' "$TMP_ROOT/log" | paste -sd'|')"; fi +echo 1 >"$STUB_LIMIT_FILE" +STUB_RESET=1700 expect "a reset past the cap ends green with a warning" 0 "::warning::the token's rate limit is spent until" +check_reran 0 "nothing is re-run when the reset is past the cap" +if grep -q '^sleep' "$TMP_ROOT/log"; then fail "it waited for a reset past the cap"; else ok "a reset past the cap is not waited for"; fi +echo 1 >"$STUB_LIMIT_FILE" +STUB_RUNS="$(runs "$(run 9 in_progress)")" STUB_JOBS="$(cijob in_progress)" STUB_RUN_SEQ="$(run 9 in_progress)" +expect "the reset wait counts toward the poll cap" 1 "has not finished after 600s" +if [[ "$(grep -c '^sleep 60$' "$TMP_ROOT/log")" == 8 ]]; then ok "a 120 s reset wait leaves 8 polls"; else fail "polls: $(grep -c '^sleep 60$' "$TMP_ROOT/log")"; fi +unset STUB_LIMIT_FILE STUB_RESET +STUB_RUNS="$(runs "$(run 9)")" STUB_JOBS="$(cijob completed)" + export STUB_LANE_FAIL=1 expect "an unreadable other-lane run list fails" 1 "::error::could not read the pr-review-security-hosted.yml runs on $HEAD; re-run this run." unset STUB_LANE_FAIL From 7e622deeca134368c9869da09230de7bad973e3c Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:08:47 -0400 Subject: [PATCH 5/8] test(pr-pipeline): give the other-heads case its own other-lane fixture The newest-run case sat between the other-heads fixture and its expect, so the other-heads case ran against the newest-run fixture and could not fail on a missing head, event or path filter. Co-Authored-By: Claude Opus 5.5 --- scripts/ci-rerun-ci-status.test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci-rerun-ci-status.test.sh b/scripts/ci-rerun-ci-status.test.sh index f7de8ee997..303451eaba 100755 --- a/scripts/ci-rerun-ci-status.test.sh +++ b/scripts/ci-rerun-ci-status.test.sh @@ -180,10 +180,10 @@ for st in in_progress queued waiting; do check_calls 1 "the first lane to finish stops after one read" done STUB_LANE_RUNS="$(runs "$(run 3 in_progress "${HEAD//0/f}" "$SEC")" "$(run 4 in_progress "$HEAD" "$SEC" merge_group)" "$(run 5 in_progress "$HEAD" "$CODE")")" +expect "other heads, events or files are not the other lane's newest run" 0 "Re-ran ci-status (job 77)" STUB_LANE_RUNS="$(runs "$(run 3 in_progress "$HEAD" "$SEC")" "$(run 4 completed "$HEAD" "$SEC")")" expect "an older other-lane run still going does not hold the newest, finished one" 0 "Re-ran ci-status (job 77)" check_reran 1 "only the other lane's newest run decides" -expect "other heads, events or files are not the other lane's newest run" 0 "Re-ran ci-status (job 77)" check_reran 1 "a head with no other-lane run of its own is re-run" # One lane only on the head (an exempt author, a lane that never ran), or the From 991f1a244b2cb7e9b2ecd66a06fa54e7e6ae3c49 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:09:28 -0400 Subject: [PATCH 6/8] fix(pr-pipeline): honor retry-after in rerun-ci-status's rate-limit wait A 403 or 429 with retry-after waits that many seconds before the x-ratelimit-reset rule, the order GitHub documents and gh_read and pr-merge's dispatch step follow. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/pr-feedback-dispatch.yml | 29 ++++++++++++++-------- scripts/ci-rerun-ci-status.test.sh | 12 +++++++-- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pr-feedback-dispatch.yml b/.github/workflows/pr-feedback-dispatch.yml index d0ce89b4cb..2c65a098b0 100644 --- a/.github/workflows/pr-feedback-dispatch.yml +++ b/.github/workflows/pr-feedback-dispatch.yml @@ -331,15 +331,17 @@ jobs: exit 1 ;; esac - # read_api : GET into . A primary rate - # limit (403 or 429 with x-ratelimit-remaining 0) waits until one - # second past x-ratelimit-reset when that fits in what is left of - # POLL_LIMIT_SECONDS, shared with the poll below; a reset past it - # ends the job green, since the next lane completion re-runs - # ci-status. Any other failure is read three times, then fails. + # read_api : GET into . A rate limit + # (403 or 429) waits retry-after seconds, else, with + # x-ratelimit-remaining 0, until one second past x-ratelimit-reset, + # both read from the refused response's headers, when the wait fits + # in what is left of POLL_LIMIT_SECONDS, shared with the poll below. + # A wait past it ends the job green, since the next lane completion + # re-runs ci-status. Any other failure is read three times, then + # fails. waited=0 read_api() { - local attempt out hdr status left reset wait + local attempt out hdr status left reset after wait for attempt in 1 2 3; do if out=$(gh api -i "$3"); then printf -v "$1" '%s' "$(sed '1,/^\r\?$/d' <<<"$out")" @@ -347,11 +349,18 @@ jobs: fi hdr=$(sed '/^\r\?$/q' <<<"$out") status=$(awk 'NR == 1 { print $2 }' <<<"$hdr") - left=$(awk -F': *' 'tolower($1) == "x-ratelimit-remaining" { sub(/\r$/, "", $2); print $2; exit }' <<<"$hdr") - reset=$(awk -F': *' 'tolower($1) == "x-ratelimit-reset" { sub(/\r$/, "", $2); print $2; exit }' <<<"$hdr") - if [[ $status =~ ^(403|429)$ && $left == 0 && $reset =~ ^[0-9]+$ ]]; then + read -r left reset after < <(tr -d '\r' <<<"$hdr" | awk -F': *' '{ h[tolower($1)] = $2 } + END { print h["x-ratelimit-remaining"] "-", h["x-ratelimit-reset"] "-", h["retry-after"] "-" }') + left=${left%-} reset=${reset%-} after=${after%-} + if [[ $status =~ ^(403|429)$ && $after =~ ^[0-9]{1,5}$ ]]; then + wait=$((10#$after)) reset=$(($(date +%s) + 10#$after)) + elif [[ $status =~ ^(403|429)$ && $left == 0 && $reset =~ ^[0-9]{1,10}$ ]]; then wait=$((reset - $(date +%s))) wait=$((wait < 0 ? 1 : wait + 1)) + else + wait= + fi + if [ -n "$wait" ]; then if [ $((waited + wait)) -gt "$POLL_LIMIT_SECONDS" ]; then echo "::warning::the token's rate limit is spent until $(date -u -d "@$reset" +%H:%M:%SZ), past this job's ${POLL_LIMIT_SECONDS}s cap; the next review lane completion re-runs ci-status." exit 0 diff --git a/scripts/ci-rerun-ci-status.test.sh b/scripts/ci-rerun-ci-status.test.sh index 303451eaba..0af3d9e5d6 100755 --- a/scripts/ci-rerun-ci-status.test.sh +++ b/scripts/ci-rerun-ci-status.test.sh @@ -57,7 +57,8 @@ fi # one run prints the next line of STUB_RUN_SEQ (the last line once used up). # A POST re-run fails when STUB_RERUN_FAIL is set. While STUB_LIMIT_FILE # holds a count above 0, a pr-require-checks runs read is refused as a -# primary rate limit resetting at STUB_RESET, and the count drops by one. The +# primary rate limit resetting at STUB_RESET (a 429 with retry-after +# STUB_RETRY_AFTER when that is set), and the count drops by one. The # sleep stub logs its seconds. mkdir -p "$TMP_ROOT/bin" cat >"$TMP_ROOT/bin/gh" <<'EOF' @@ -68,7 +69,11 @@ ok200() { printf 'HTTP/2.0 200 OK\r\nX-Ratelimit-Remaining: 9\r\n\r\n%s\n' "$1"; if [[ -n "${STUB_LIMIT_FILE:-}" && "$*" == *"/actions/workflows/pr-require-checks.yml/runs"* ]] && [[ "$(cat "$STUB_LIMIT_FILE")" -gt 0 ]]; then echo $(($(cat "$STUB_LIMIT_FILE") - 1)) >"$STUB_LIMIT_FILE" - printf 'HTTP/2.0 403 Forbidden\r\nX-Ratelimit-Remaining: 0\r\nX-Ratelimit-Reset: %s\r\n\r\n{"message":"API rate limit exceeded for installation"}\n' "$STUB_RESET" + if [[ -n "${STUB_RETRY_AFTER:-}" ]]; then + printf 'HTTP/2.0 429 Too Many Requests\r\nRetry-After: %s\r\n\r\n{"message":"You have exceeded a secondary rate limit"}\n' "$STUB_RETRY_AFTER" + else + printf 'HTTP/2.0 403 Forbidden\r\nX-Ratelimit-Remaining: 0\r\nX-Ratelimit-Reset: %s\r\n\r\n{"message":"API rate limit exceeded for installation"}\n' "$STUB_RESET" + fi exit 1 fi case "$*" in @@ -275,6 +280,9 @@ expect "a rate-limited read waits for the reset, then re-runs" 0 "Rate limited r check_reran 1 "the read after the reset goes on to the re-run" if [[ "$(grep -c '^sleep 120$' "$TMP_ROOT/log")" == 1 ]]; then ok "it slept until one second past the reset"; else fail "sleeps: $(grep '^sleep' "$TMP_ROOT/log" | paste -sd'|')"; fi echo 1 >"$STUB_LIMIT_FILE" +STUB_RETRY_AFTER=30 expect "a retry-after refusal waits that many seconds" 0 "waiting 30s for the reset." +check_reran 1 "the read after retry-after goes on to the re-run" +echo 1 >"$STUB_LIMIT_FILE" STUB_RESET=1700 expect "a reset past the cap ends green with a warning" 0 "::warning::the token's rate limit is spent until" check_reran 0 "nothing is re-run when the reset is past the cap" if grep -q '^sleep' "$TMP_ROOT/log"; then fail "it waited for a reset past the cap"; else ok "a reset past the cap is not waited for"; fi From 540b163a6216d65615644298547b04b8b1ae5b41 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:37:13 -0400 Subject: [PATCH 7/8] fix(pr-pipeline): fail rerun-ci-status when a rate limit outlasts its cap A green exit past the cap left ci-status stale on the last or only lane: no later lane completion exists there, and maintenance-reconcile-lost-runs leaves a run where only ci-status failed alone. The job now fails with an error naming the reset, as it did before the wait, so the stale head is visible and re-run after the reset. Co-Authored-By: Claude Opus 5.5 --- ...erf-feedback-dispatch-fewer-ci-reruns-3e7a91c4.md | 2 +- .github/workflows/pr-feedback-dispatch.yml | 12 +++++++----- docs/conventions/pr-pipeline/README.md | 4 ++-- scripts/ci-rerun-ci-status.test.sh | 6 +++--- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.changes/pr-pipeline/perf-feedback-dispatch-fewer-ci-reruns-3e7a91c4.md b/.changes/pr-pipeline/perf-feedback-dispatch-fewer-ci-reruns-3e7a91c4.md index cf24c7e6df..170d5dbaab 100644 --- a/.changes/pr-pipeline/perf-feedback-dispatch-fewer-ci-reruns-3e7a91c4.md +++ b/.changes/pr-pipeline/perf-feedback-dispatch-fewer-ci-reruns-3e7a91c4.md @@ -4,4 +4,4 @@ bump: patch ### Changed -- `pr-feedback-dispatch.yml`'s `rerun-ci-status` re-runs a head's ci-status once, after the last review lane finishes, instead of once per lane: while the other lane's newest run on the head is still going it stops after one read, since ci-status would still be red and that lane's completion starts the job again. A head with no run of the other lane re-runs on one lane. A ci-status that started after both lanes finished, or a re-run already queued, is left alone. The job list is read once, and each 60-second poll reads only the run. A read refused by the token's primary rate limit (403 or 429 with `x-ratelimit-remaining: 0`) waits until one second past `x-ratelimit-reset` inside the same 10-minute cap instead of failing; a reset past the cap ends the job green with a warning. +- `pr-feedback-dispatch.yml`'s `rerun-ci-status` re-runs a head's ci-status once, after the last review lane finishes, instead of once per lane: while the other lane's newest run on the head is still going it stops after one read, since ci-status would still be red and that lane's completion starts the job again. A head with no run of the other lane re-runs on one lane. A ci-status that started after both lanes finished, or a re-run already queued, is left alone. The job list is read once, and each 60-second poll reads only the run. A read refused by the token's primary rate limit (403 or 429) waits `retry-after` seconds, else, with `x-ratelimit-remaining: 0`, until one second past `x-ratelimit-reset` inside the same 10-minute cap instead of failing; a reset past the cap fails the job with an error naming the reset, since nothing else re-runs a ci-status left stale. diff --git a/.github/workflows/pr-feedback-dispatch.yml b/.github/workflows/pr-feedback-dispatch.yml index eddfc37dc0..1681301bf9 100644 --- a/.github/workflows/pr-feedback-dispatch.yml +++ b/.github/workflows/pr-feedback-dispatch.yml @@ -345,9 +345,11 @@ jobs: # x-ratelimit-remaining 0, until one second past x-ratelimit-reset, # both read from the refused response's headers, when the wait fits # in what is left of POLL_LIMIT_SECONDS, shared with the poll below. - # A wait past it ends the job green, since the next lane completion - # re-runs ci-status. Any other failure is read three times, then - # fails. + # A wait past it fails the job, naming the reset: no other run + # re-runs a ci-status this job leaves stale (a later lane's + # completion exists only while the other lane is running, and + # maintenance-reconcile-lost-runs leaves a run where only ci-status + # failed alone). Any other failure is read three times, then fails. waited=0 read_api() { local attempt out hdr status left reset after wait @@ -371,8 +373,8 @@ jobs: fi if [ -n "$wait" ]; then if [ $((waited + wait)) -gt "$POLL_LIMIT_SECONDS" ]; then - echo "::warning::the token's rate limit is spent until $(date -u -d "@$reset" +%H:%M:%SZ), past this job's ${POLL_LIMIT_SECONDS}s cap; the next review lane completion re-runs ci-status." - exit 0 + echo "::error::the token's rate limit blocks reading $2 until $(date -u -d "@$reset" +%H:%M:%SZ), past this job's ${POLL_LIMIT_SECONDS}s cap; ci-status on $HEAD_SHA is not re-run. Re-run this run after the reset." + exit 1 fi echo "Rate limited reading $2; waiting ${wait}s for the reset." sleep "$wait" diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index a11f756fd6..4ffba9d220 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -1525,8 +1525,8 @@ lane is still going a re-run could only be red, and that lane's completion re-ru gets one re-run, not one per lane. A ci-status not started yet, or one that started after both lanes finished, is left to read the reviews itself; an earlier one in progress is polled every 60 seconds for at most 10 minutes first. A read refused by the token's primary rate limit waits for -the reset inside the same 10 minutes; a reset past them ends the job green, and the next lane -completion re-runs ci-status. The merge-group run re-checks the labels. +the reset inside the same 10 minutes; a reset past them fails the job, naming the reset, since +nothing else re-runs a ci-status left stale. The merge-group run re-checks the labels. `fix-ci` keys on `ci-lanes`, so a hold or a pending review never summons it. ## Trust-root paths diff --git a/scripts/ci-rerun-ci-status.test.sh b/scripts/ci-rerun-ci-status.test.sh index 0af3d9e5d6..08823d2736 100755 --- a/scripts/ci-rerun-ci-status.test.sh +++ b/scripts/ci-rerun-ci-status.test.sh @@ -9,8 +9,8 @@ # pr-require-checks pull_request run on the reviewed head, after that run # finishes, polling at most POLL_LIMIT_SECONDS. The other lane still going, a # closed PR, a moved head, no run yet, a ci-status not started or one started -# after both lanes finished is left alone. A primary rate limit waits for its -# reset within the cap, or ends green past it. Unreadable reads fail. Each +# after both lanes finished is left alone. A rate limit waits for its reset +# within the cap, or fails past it. Unreadable reads fail. Each # path's API call count is pinned. # test-scope: .github/workflows/pr-feedback-dispatch.yml set -uo pipefail @@ -283,7 +283,7 @@ echo 1 >"$STUB_LIMIT_FILE" STUB_RETRY_AFTER=30 expect "a retry-after refusal waits that many seconds" 0 "waiting 30s for the reset." check_reran 1 "the read after retry-after goes on to the re-run" echo 1 >"$STUB_LIMIT_FILE" -STUB_RESET=1700 expect "a reset past the cap ends green with a warning" 0 "::warning::the token's rate limit is spent until" +STUB_RESET=1700 expect "a reset past the cap fails, naming the reset" 1 "::error::the token's rate limit blocks reading the pr-require-checks runs on $HEAD until" check_reran 0 "nothing is re-run when the reset is past the cap" if grep -q '^sleep' "$TMP_ROOT/log"; then fail "it waited for a reset past the cap"; else ok "a reset past the cap is not waited for"; fi echo 1 >"$STUB_LIMIT_FILE" From 41be7ff9ae561836c768cef78b2c49d9ad3d8b4c Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:40:54 -0400 Subject: [PATCH 8/8] test(pr-pipeline): pin the other-heads case's re-run on its own log The other-heads case's check_reran had moved below the newest-run case and re-read that case's log. It now follows its own expect. Co-Authored-By: Claude Opus 5.5 --- scripts/ci-rerun-ci-status.test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci-rerun-ci-status.test.sh b/scripts/ci-rerun-ci-status.test.sh index 08823d2736..5797bd5671 100755 --- a/scripts/ci-rerun-ci-status.test.sh +++ b/scripts/ci-rerun-ci-status.test.sh @@ -186,10 +186,10 @@ for st in in_progress queued waiting; do done STUB_LANE_RUNS="$(runs "$(run 3 in_progress "${HEAD//0/f}" "$SEC")" "$(run 4 in_progress "$HEAD" "$SEC" merge_group)" "$(run 5 in_progress "$HEAD" "$CODE")")" expect "other heads, events or files are not the other lane's newest run" 0 "Re-ran ci-status (job 77)" +check_reran 1 "a head with no other-lane run of its own is re-run" STUB_LANE_RUNS="$(runs "$(run 3 in_progress "$HEAD" "$SEC")" "$(run 4 completed "$HEAD" "$SEC")")" expect "an older other-lane run still going does not hold the newest, finished one" 0 "Re-ran ci-status (job 77)" check_reran 1 "only the other lane's newest run decides" -check_reran 1 "a head with no other-lane run of its own is re-run" # One lane only on the head (an exempt author, a lane that never ran), or the # other lane skipped, cancelled or failed: this lane is the last, so re-run.