diff --git a/.changes/pr-pipeline/perf-feedback-dispatch-fewer-ci-reruns-3e7a91c4.md b/.changes/pr-pipeline/perf-feedback-dispatch-fewer-ci-reruns-3e7a91c4.md new file mode 100644 index 0000000000..170d5dbaab --- /dev/null +++ b/.changes/pr-pipeline/perf-feedback-dispatch-fewer-ci-reruns-3e7a91c4.md @@ -0,0 +1,7 @@ +--- +bump: patch +--- + +### Changed + +- `pr-feedback-dispatch.yml`'s `rerun-ci-status` re-runs a head's ci-status once, after the last review lane finishes, instead of once per lane: while the other lane's newest run on the head is still going it stops after one read, since ci-status would still be red and that lane's completion starts the job again. A head with no run of the other lane re-runs on one lane. A ci-status that started after both lanes finished, or a re-run already queued, is left alone. The job list is read once, and each 60-second poll reads only the run. A read refused by the token's primary rate limit (403 or 429) waits `retry-after` seconds, else, with `x-ratelimit-remaining: 0`, until one second past `x-ratelimit-reset` inside the same 10-minute cap instead of failing; a reset past the cap fails the job with an error naming the reset, since nothing else re-runs a ci-status left stale. diff --git a/.github/workflows/pr-feedback-dispatch.yml b/.github/workflows/pr-feedback-dispatch.yml index ed11772734..1681301bf9 100644 --- a/.github/workflows/pr-feedback-dispatch.yml +++ b/.github/workflows/pr-feedback-dispatch.yml @@ -25,8 +25,8 @@ name: pr-feedback-dispatch # run either way. # # A completed review run on any same-repository PR also starts the -# rerun-ci-status job, which re-runs the head's ci-status so it reads the -# finished review (see that job). +# rerun-ci-status job, which re-runs the head's ci-status once the head's +# last review lane has finished, so it reads both reviews (see that job). # # Every completed review run at a head starts this file, so a run leaves a # record for the next one at that head in the Actions cache @@ -271,17 +271,28 @@ jobs: tokens: github-token ${{ github.token }} # ci-status's `Require both review lanes on this head` step reads the - # review runs when it runs, and a review usually finishes after it. So - # when either review caller completes on a pull request of this - # repository, this job re-runs the ci-status job of the newest - # pr-require-checks run on that head, which reads both lanes and the live - # labels again. Every PR's review starts it, not only a lanes App PR's. - # A run GitHub skipped (a draft) reviewed nothing, and a fork's review - # never runs, so neither starts it. A head the PR has moved past, or a - # closed PR, is left alone. A ci-status job not started yet reads the - # reviews when it starts, so it is left alone too; one in progress is - # polled every 60 s for at most 10 minutes, then re-run. No checkout, no - # model: the token can only read runs and the PR and re-run a job. + # newest pull_request run of each review lane on the head when it runs, and + # a review usually finishes after it. So when either review caller + # completes on a pull request of this repository, this job re-runs the + # ci-status job of the newest pr-require-checks run on that head, which + # reads both lanes and the live labels again. Every PR's review starts it, + # not only a lanes App PR's. A run GitHub skipped (a draft) reviewed + # nothing, and a fork's review never runs, so neither starts it. + # + # One re-run per head, after the last lane: ci-status fails while the + # other lane's newest run has not finished, so a re-run before then can + # only be red, and that lane's own completion starts this job again. Each + # run of this job starts after its own lane completed, so the later lane's + # run finds the earlier lane finished; the per-head concurrency group keeps + # the two from re-running ci-status at once. A head with no run of the + # other lane (one lane only) re-runs on this lane alone. A head the PR has + # moved past, or a closed PR, is left alone. A ci-status job not started + # yet, or one that started more than 60 s after both lanes finished (the + # margin covers a lagging read), reads them anyway, so it is left alone + # too; one that + # started earlier and is still in progress waits for its run, polled every + # 60 s for at most 10 minutes, then is re-run. No checkout, no model: the + # token can only read runs and the PR and re-run a job. # Pinned by scripts/ci-rerun-ci-status.test.sh, which runs this body. rerun-ci-status: if: >- @@ -295,7 +306,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 permissions: - actions: write # list pr-require-checks runs and jobs; re-run the ci-status job + actions: write # list review and pr-require-checks runs and jobs; re-run the ci-status job pull-requests: read # the PR's state and live head steps: - name: Re-run ci-status on the reviewed head @@ -303,6 +314,8 @@ jobs: GH_TOKEN: ${{ github.token }} HEAD_SHA: ${{ github.event.workflow_run.head_sha }} PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} + LANE_PATH: ${{ github.event.workflow_run.path }} + LANE_DONE_AT: ${{ github.event.workflow_run.updated_at }} POLL_SECONDS: '60' POLL_LIMIT_SECONDS: '600' run: | @@ -314,39 +327,97 @@ jobs: echo "::notice::the review run names no pull request; nothing to re-run." exit 0 fi + utc='^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$' + if [[ ! $LANE_DONE_AT =~ $utc ]]; then + echo "::error::the review run's finish time '$LANE_DONE_AT' is not a UTC timestamp." + exit 1 + fi + case $LANE_PATH in + .github/workflows/pr-review-hosted.yml) other='pr-review-security-hosted.yml' ;; + .github/workflows/pr-review-security-hosted.yml) other='pr-review-hosted.yml' ;; + *) + echo "::error::'$LANE_PATH' is not a review lane." + exit 1 + ;; + esac + # read_api : GET into . A rate limit + # (403 or 429) waits retry-after seconds, else, with + # x-ratelimit-remaining 0, until one second past x-ratelimit-reset, + # both read from the refused response's headers, when the wait fits + # in what is left of POLL_LIMIT_SECONDS, shared with the poll below. + # A wait past it fails the job, naming the reset: no other run + # re-runs a ci-status this job leaves stale (a later lane's + # completion exists only while the other lane is running, and + # maintenance-reconcile-lost-runs leaves a run where only ci-status + # failed alone). Any other failure is read three times, then fails. + waited=0 read_api() { - local attempt body + local attempt out hdr status left reset after wait for attempt in 1 2 3; do - if body=$(gh api "$2"); then - printf '%s\n' "$body" + if out=$(gh api -i "$3"); then + printf -v "$1" '%s' "$(sed '1,/^\r\?$/d' <<<"$out")" return 0 fi - [ "$attempt" -lt 3 ] || { echo "::error::could not read $1; re-run this run." >&2; return 1; } + hdr=$(sed '/^\r\?$/q' <<<"$out") + status=$(awk 'NR == 1 { print $2 }' <<<"$hdr") + read -r left reset after < <(tr -d '\r' <<<"$hdr" | awk -F': *' '{ h[tolower($1)] = $2 } + END { print h["x-ratelimit-remaining"] "-", h["x-ratelimit-reset"] "-", h["retry-after"] "-" }') + left=${left%-} reset=${reset%-} after=${after%-} + if [[ $status =~ ^(403|429)$ && $after =~ ^[0-9]{1,5}$ ]]; then + wait=$((10#$after)) reset=$(($(date +%s) + 10#$after)) + elif [[ $status =~ ^(403|429)$ && $left == 0 && $reset =~ ^[0-9]{1,10}$ ]]; then + wait=$((reset - $(date +%s))) + wait=$((wait < 0 ? 1 : wait + 1)) + else + wait= + fi + if [ -n "$wait" ]; then + if [ $((waited + wait)) -gt "$POLL_LIMIT_SECONDS" ]; then + echo "::error::the token's rate limit blocks reading $2 until $(date -u -d "@$reset" +%H:%M:%SZ), past this job's ${POLL_LIMIT_SECONDS}s cap; ci-status on $HEAD_SHA is not re-run. Re-run this run after the reset." + exit 1 + fi + echo "Rate limited reading $2; waiting ${wait}s for the reset." + sleep "$wait" + waited=$((waited + wait)) + continue + fi + [ "$attempt" -lt 3 ] || { echo "::error::could not read $2; re-run this run." >&2; exit 1; } sleep $((attempt * 5)) done + echo "::error::could not read $2 after the rate-limit waits; re-run this run." >&2 + exit 1 } - pr=$(read_api "#$PR_NUMBER" "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER") || exit 1 - if [ "$(jq -r '"\(.state) \(.head.sha)"' <<<"$pr")" != "open $HEAD_SHA" ]; then - echo "#$PR_NUMBER is closed or no longer at $HEAD_SHA; nothing to re-run." - exit 0 - fi - waited=0 - while :; do - runs=$(read_api "the pr-require-checks runs on $HEAD_SHA" \ - "repos/$GITHUB_REPOSITORY/actions/workflows/pr-require-checks.yml/runs?head_sha=$HEAD_SHA&event=pull_request&per_page=100") || exit 1 - run=$(jq -c --arg sha "$HEAD_SHA" '[.workflow_runs[] - | select(.path == ".github/workflows/pr-require-checks.yml" and .head_sha == $sha and .event == "pull_request")] - | max_by(.id) // empty' <<<"$runs") - if [ -z "$run" ]; then - echo "No pr-require-checks run on $HEAD_SHA yet; its ci-status reads the reviews when it runs." + # ci-status's own choice of run: the newest pull_request run of the file on the head. + newest() { + jq -c --arg path ".github/workflows/$1" --arg sha "$HEAD_SHA" '[.workflow_runs[] + | select(.path == $path and .head_sha == $sha and .event == "pull_request")] + | max_by(.id) // empty' + } + read_api runs "the $other runs on $HEAD_SHA" \ + "repos/$GITHUB_REPOSITORY/actions/workflows/$other/runs?head_sha=$HEAD_SHA&event=pull_request&per_page=100" + lane=$(newest "$other" <<<"$runs") + done_at=$LANE_DONE_AT + if [ -n "$lane" ]; then + if [ "$(jq -r .status <<<"$lane")" != completed ]; then + echo "$other has not finished on $HEAD_SHA; its completion re-runs ci-status." exit 0 fi - id=$(jq -r .id <<<"$run") - jobs=$(read_api "the jobs of run $id" "repos/$GITHUB_REPOSITORY/actions/runs/$id/jobs?filter=latest&per_page=100") || exit 1 - job=$(jq -c '[.jobs[] | select(.name == "ci-status")] | max_by(.id) // empty' <<<"$jobs") - if [ "$(jq -r .status <<<"$run")" = completed ]; then - break + other_at=$(jq -r '.updated_at // ""' <<<"$lane") + if [[ $other_at > $done_at ]]; then + done_at=$other_at fi + fi + read_api runs "the pr-require-checks runs on $HEAD_SHA" \ + "repos/$GITHUB_REPOSITORY/actions/workflows/pr-require-checks.yml/runs?head_sha=$HEAD_SHA&event=pull_request&per_page=100" + run=$(newest pr-require-checks.yml <<<"$runs") + if [ -z "$run" ]; then + echo "No pr-require-checks run on $HEAD_SHA yet; its ci-status reads the reviews when it runs." + exit 0 + fi + id=$(jq -r .id <<<"$run") + read_api jobs "the jobs of run $id" "repos/$GITHUB_REPOSITORY/actions/runs/$id/jobs?filter=latest&per_page=100" + job=$(jq -c '[.jobs[] | select(.name == "ci-status")] | max_by(.id) // empty' <<<"$jobs") + if [ "$(jq -r .status <<<"$run")" != completed ]; then case $(jq -r '.status // "absent"' <<<"${job:-null}") in in_progress | completed) ;; *) @@ -354,17 +425,31 @@ jobs: exit 0 ;; esac + fi + started_at=$(jq -r '.started_at // ""' <<<"${job:-null}") + if [ "$(jq -r --arg s "$started_at" --arg d "$done_at" \ + 'try (($s | fromdateiso8601) > ($d | fromdateiso8601) + 60) catch false' <</dev/null echo "Re-ran ci-status (job $job_id) of run $id on $HEAD_SHA." diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index d9447db86a..4ffba9d220 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -1519,9 +1519,14 @@ them on every `pull_request` run, contract-only runs included: A review usually finishes after ci-status has read it, so `pr-feedback-dispatch.yml`'s `rerun-ci-status` job re-runs the ci-status job of the newest `pr-require-checks` run on the head -whenever either review caller completes (not when GitHub skipped it) on an open same-repository PR -still at that head. A ci-status not started yet is left to read the reviews itself; one in progress -is polled every 60 seconds for at most 10 minutes first. The merge-group run re-checks the labels. +when a review caller completes (not when GitHub skipped it) on an open same-repository PR still at +that head and the other lane's newest run on the head has finished or never ran. While the other +lane is still going a re-run could only be red, and that lane's completion re-runs it, so a head +gets one re-run, not one per lane. A ci-status not started yet, or one that started after both +lanes finished, is left to read the reviews itself; an earlier one in progress is polled every 60 +seconds for at most 10 minutes first. A read refused by the token's primary rate limit waits for +the reset inside the same 10 minutes; a reset past them fails the job, naming the reset, since +nothing else re-runs a ci-status left stale. The merge-group run re-checks the labels. `fix-ci` keys on `ci-lanes`, so a hold or a pending review never summons it. ## Trust-root paths diff --git a/scripts/ci-rerun-ci-status.test.sh b/scripts/ci-rerun-ci-status.test.sh index 9d1bfbd649..5797bd5671 100755 --- a/scripts/ci-rerun-ci-status.test.sh +++ b/scripts/ci-rerun-ci-status.test.sh @@ -4,10 +4,14 @@ # shell Actions gives a step with no `shell:` (bash -e), against stub `gh` and # `sleep`. # -# When a review run completes, the job re-runs only the ci-status job of the -# newest pr-require-checks pull_request run on the reviewed head, after that -# run finishes, polling at most POLL_LIMIT_SECONDS. A closed PR, a moved head, -# no run yet or a ci-status not started is left alone. Unreadable reads fail. +# When a review run completes and the other lane's newest run on the head has +# finished or never ran, the job re-runs only the ci-status job of the newest +# pr-require-checks pull_request run on the reviewed head, after that run +# finishes, polling at most POLL_LIMIT_SECONDS. The other lane still going, a +# closed PR, a moved head, no run yet, a ci-status not started or one started +# after both lanes finished is left alone. A rate limit waits for its reset +# within the cap, or fails past it. Unreadable reads fail. Each +# path's API call count is pinned. # test-scope: .github/workflows/pr-feedback-dispatch.yml set -uo pipefail @@ -36,7 +40,8 @@ if [[ ! -s "$TMP_ROOT/body.sh" ]]; then exit 1 fi # shellcheck disable=SC2016 # the ${{ }} text is workflow source, not shell -for env in 'HEAD_SHA: ${{ github.event.workflow_run.head_sha }}' 'PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }}' "POLL_LIMIT_SECONDS: '600'"; do +for env in 'HEAD_SHA: ${{ github.event.workflow_run.head_sha }}' 'PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }}' \ + 'LANE_PATH: ${{ github.event.workflow_run.path }}' 'LANE_DONE_AT: ${{ github.event.workflow_run.updated_at }}' "POLL_LIMIT_SECONDS: '600'"; do if [[ "$step" == *"$env"* ]]; then ok "step env carries $env"; else fail "step env lacks $env"; fi done job="$(awk '/^ rerun-ci-status:$/ { on = 1; next } on && /^ [a-z]/ { exit } on { print }' "$WORKFLOW")" @@ -46,23 +51,47 @@ else fail "the job's permissions changed" fi -# The stub logs its arguments. A pulls read prints STUB_PR. A runs read prints -# the next line of STUB_RUNS_SEQ (the last line once used up), failing when -# STUB_RUNS_FAIL is set. A jobs read prints STUB_JOBS. A POST re-run fails when -# STUB_RERUN_FAIL is set. The sleep stub logs its seconds. +# The stub logs its arguments. A pulls read prints STUB_PR. A review lane's +# runs read prints STUB_LANE_RUNS and a pr-require-checks runs read STUB_RUNS, +# each failing when its *_FAIL is set. A jobs read prints STUB_JOBS. A read of +# one run prints the next line of STUB_RUN_SEQ (the last line once used up). +# A POST re-run fails when STUB_RERUN_FAIL is set. While STUB_LIMIT_FILE +# holds a count above 0, a pr-require-checks runs read is refused as a +# primary rate limit resetting at STUB_RESET (a 429 with retry-after +# STUB_RETRY_AFTER when that is set), and the count drops by one. The +# sleep stub logs its seconds. mkdir -p "$TMP_ROOT/bin" cat >"$TMP_ROOT/bin/gh" <<'EOF' #!/usr/bin/env bash printf '%s\n' "$*" >>"$STUB_LOG" +# A GET read with -i prints a status line and a blank line before the body. +ok200() { printf 'HTTP/2.0 200 OK\r\nX-Ratelimit-Remaining: 9\r\n\r\n%s\n' "$1"; } +if [[ -n "${STUB_LIMIT_FILE:-}" && "$*" == *"/actions/workflows/pr-require-checks.yml/runs"* ]] && + [[ "$(cat "$STUB_LIMIT_FILE")" -gt 0 ]]; then + echo $(($(cat "$STUB_LIMIT_FILE") - 1)) >"$STUB_LIMIT_FILE" + if [[ -n "${STUB_RETRY_AFTER:-}" ]]; then + printf 'HTTP/2.0 429 Too Many Requests\r\nRetry-After: %s\r\n\r\n{"message":"You have exceeded a secondary rate limit"}\n' "$STUB_RETRY_AFTER" + else + printf 'HTTP/2.0 403 Forbidden\r\nX-Ratelimit-Remaining: 0\r\nX-Ratelimit-Reset: %s\r\n\r\n{"message":"API rate limit exceeded for installation"}\n' "$STUB_RESET" + fi + exit 1 +fi case "$*" in -*/pulls/*) printf '%s\n' "$STUB_PR" ;; +*/pulls/*) ok200 "$STUB_PR" ;; +*/actions/workflows/pr-review-hosted.yml/runs* | */actions/workflows/pr-review-security-hosted.yml/runs*) + [[ -z "${STUB_LANE_FAIL:-}" ]] || exit 1 + ok200 "$STUB_LANE_RUNS" + ;; */actions/workflows/pr-require-checks.yml/runs*) [[ -z "${STUB_RUNS_FAIL:-}" ]] || exit 1 - n=$(grep -c '/runs?' "$STUB_LOG") - total=$(wc -l <<<"$STUB_RUNS_SEQ") - sed -n "$((n < total ? n : total))p" <<<"$STUB_RUNS_SEQ" + ok200 "$STUB_RUNS" + ;; +*/actions/runs/*/jobs*) ok200 "$STUB_JOBS" ;; +"api -i repos/o/r/actions/runs/"[0-9]*) + n=$(grep -c '^api -i repos/o/r/actions/runs/[0-9]*$' "$STUB_LOG") + total=$(wc -l <<<"$STUB_RUN_SEQ") + ok200 "$(sed -n "$((n < total ? n : total))p" <<<"$STUB_RUN_SEQ")" ;; -*/actions/runs/*/jobs*) printf '%s\n' "$STUB_JOBS" ;; "api -X POST "*/rerun) [[ -z "${STUB_RERUN_FAIL:-}" ]] || exit 1 echo '{}' @@ -72,23 +101,39 @@ esac EOF # shellcheck disable=SC2016 # the stub expands $* and $STUB_LOG when it runs printf '#!/usr/bin/env bash\necho "sleep $*" >>"$STUB_LOG"\n' >"$TMP_ROOT/bin/sleep" -chmod +x "$TMP_ROOT/bin/gh" "$TMP_ROOT/bin/sleep" +# The clock reads 1000 s past the epoch; any other date call runs the real date. +REAL_DATE="$(command -v date)" +# shellcheck disable=SC2016 # the stub expands its arguments when it runs +printf '#!/usr/bin/env bash\nif [ "$*" = +%%s ]; then echo 1000; else exec %q "$@"; fi\n' "$REAL_DATE" >"$TMP_ROOT/bin/date" +chmod +x "$TMP_ROOT/bin/gh" "$TMP_ROOT/bin/sleep" "$TMP_ROOT/bin/date" HEAD=0123456789abcdef0123456789abcdef01234567 RCF=.github/workflows/pr-require-checks.yml +CODE=.github/workflows/pr-review-hosted.yml +SEC=.github/workflows/pr-review-security-hosted.yml +# This lane finished at DONE, the other lane at LATER; ci-status started at +# EARLY unless a case says otherwise. AFTER is just past the 60 s margin. +EARLY=2026-10-23T09:50:00Z DONE=2026-10-23T10:00:00Z LATER=2026-10-23T10:05:00Z AFTER=2026-10-23T10:06:01Z pr() { jq -cn --arg st "${1:-open}" --arg sha "${2:-$HEAD}" '{state: $st, head: {sha: $sha}}'; } -# run [status] [sha] [path] [event] -run() { jq -cn --argjson id "$1" --arg st "${2:-completed}" --arg sha "${3:-$HEAD}" --arg p "${4:-$RCF}" --arg ev "${5:-pull_request}" \ - '{id: $id, status: $st, head_sha: $sha, path: $p, event: $ev}'; } -runs() { local IFS=,; printf '{"workflow_runs":[%s]}' "$*"; } -cijob() { jq -cn --arg st "$1" '{jobs: [{id: 7, name: "lint-repo", status: "completed"}, {id: 77, name: "ci-status", status: $st}]}'; } +# run [status] [sha] [path] [event] [updated_at] +run() { + jq -cn --argjson id "$1" --arg st "${2:-completed}" --arg sha "${3:-$HEAD}" --arg p "${4:-$RCF}" --arg ev "${5:-pull_request}" --arg at "${6:-$LATER}" \ + '{id: $id, status: $st, head_sha: $sha, path: $p, event: $ev, updated_at: $at}' +} +runs() { + local IFS=, + printf '{"workflow_runs":[%s]}' "$*" +} +# cijob [started_at] +cijob() { jq -cn --arg st "$1" --arg at "${2:-$EARLY}" '{jobs: [{id: 7, name: "lint-repo", status: "completed"}, {id: 77, name: "ci-status", status: $st, started_at: $at}]}'; } -export STUB_PR STUB_RUNS_SEQ STUB_JOBS +export STUB_PR STUB_LANE_RUNS STUB_RUNS STUB_JOBS STUB_RUN_SEQ expect() { local label="$1" want_rc="$2" out rc : >"$TMP_ROOT/log" out="$(PATH="$TMP_ROOT/bin:$PATH" STUB_LOG="$TMP_ROOT/log" GITHUB_REPOSITORY=o/r HEAD_SHA="${HEAD_SHA-$HEAD}" \ - PR_NUMBER="${PR_NUMBER-42}" POLL_SECONDS=60 POLL_LIMIT_SECONDS=600 bash -e "$TMP_ROOT/body.sh" 2>&1)" && rc=0 || rc=$? + PR_NUMBER="${PR_NUMBER-42}" LANE_PATH="${LANE_PATH-$CODE}" LANE_DONE_AT="${LANE_DONE_AT-$DONE}" \ + POLL_SECONDS=60 POLL_LIMIT_SECONDS=600 bash -e "$TMP_ROOT/body.sh" 2>&1)" && rc=0 || rc=$? if [[ "$rc" -ne "$want_rc" ]]; then fail "$label: expected rc=$want_rc got rc=$rc out='$out'" elif [[ -n "${3:-}" && "$out" != *"$3"* ]]; then @@ -105,56 +150,152 @@ check_reran() { fail "$2: re-runs: $(grep '^api -X POST' "$TMP_ROOT/log" | paste -sd'|')" fi } +# The API calls the last case made, against the count its path is pinned to. +check_calls() { + local n + n=$(grep -c '^api ' "$TMP_ROOT/log") + if [[ "$n" == "$1" ]]; then ok "$2 ($1 API calls)"; else fail "$2: $n API calls, want $1: $(paste -sd'|' "$TMP_ROOT/log")"; fi +} -STUB_PR="$(pr)" STUB_RUNS_SEQ="$(runs "$(run 5)" "$(run 9)" "$(run 8)")" STUB_JOBS="$(cijob completed)" -expect "a finished run's ci-status is re-run" 0 "Re-ran ci-status (job 77) of run 9 on $HEAD." +# Both lanes finished: the head's last lane re-runs ci-status once. +STUB_PR="$(pr)" STUB_LANE_RUNS="$(runs "$(run 3 completed "$HEAD" "$SEC")")" +STUB_RUNS="$(runs "$(run 5)" "$(run 9)" "$(run 8)")" STUB_JOBS="$(cijob completed)" STUB_RUN_SEQ='' +expect "both lanes finished: a finished run's ci-status is re-run" 0 "Re-ran ci-status (job 77) of run 9 on $HEAD." check_reran 1 "only the ci-status job of the newest run is re-run" -if grep -qxF "api repos/o/r/actions/workflows/pr-require-checks.yml/runs?head_sha=$HEAD&event=pull_request&per_page=100" "$TMP_ROOT/log" && - grep -qxF "api repos/o/r/actions/runs/9/jobs?filter=latest&per_page=100" "$TMP_ROOT/log"; then - ok "the head's pull_request runs and the newest run's latest jobs are read" +check_calls 5 "the last lane reads the other lane, the runs, the jobs and the PR, then re-runs" +if grep -qxF "api -i repos/o/r/actions/workflows/pr-review-security-hosted.yml/runs?head_sha=$HEAD&event=pull_request&per_page=100" "$TMP_ROOT/log" && + grep -qxF "api -i repos/o/r/actions/workflows/pr-require-checks.yml/runs?head_sha=$HEAD&event=pull_request&per_page=100" "$TMP_ROOT/log" && + grep -qxF "api -i repos/o/r/actions/runs/9/jobs?filter=latest&per_page=100" "$TMP_ROOT/log"; then + ok "the other lane's, then the head's pull_request runs and the newest run's latest jobs are read" else fail "unexpected reads: '$(sort -u "$TMP_ROOT/log")'" fi -STUB_RUNS_SEQ="$(runs "$(run 5)" "$(run 9 completed "${HEAD//0/f}")" "$(run 10 completed "$HEAD" .github/workflows/other.yml)" "$(run 11 completed "$HEAD" "$RCF" merge_group)")" +STUB_LANE_RUNS="$(runs "$(run 4 completed "$HEAD" "$CODE")")" +LANE_PATH=$SEC expect "the security lane finishing last reads the code lane" 0 "Re-ran ci-status (job 77)" +if grep -q '/actions/workflows/pr-review-hosted.yml/runs?' "$TMP_ROOT/log"; then ok "the security lane's run reads the code lane's runs"; else fail "reads: $(paste -sd'|' "$TMP_ROOT/log")"; fi +STUB_RUNS="$(runs "$(run 5)" "$(run 9 completed "${HEAD//0/f}")" "$(run 10 completed "$HEAD" .github/workflows/other.yml)" "$(run 11 completed "$HEAD" "$RCF" merge_group)")" expect "runs on another head, of another file or event are not the newest" 0 "of run 5 on $HEAD" +STUB_RUNS="$(runs "$(run 9)")" + +# The other lane still going: no re-run, one API call; its completion re-runs ci-status. +for st in in_progress queued waiting; do + STUB_LANE_RUNS="$(runs "$(run 2 completed "$HEAD" "$SEC")" "$(run 3 "$st" "$HEAD" "$SEC")")" + expect "the other lane's newest run $st: left for its completion" 0 "pr-review-security-hosted.yml has not finished on $HEAD; its completion re-runs ci-status." + check_reran 0 "nothing is re-run while the other lane is $st" + check_calls 1 "the first lane to finish stops after one read" +done +STUB_LANE_RUNS="$(runs "$(run 3 in_progress "${HEAD//0/f}" "$SEC")" "$(run 4 in_progress "$HEAD" "$SEC" merge_group)" "$(run 5 in_progress "$HEAD" "$CODE")")" +expect "other heads, events or files are not the other lane's newest run" 0 "Re-ran ci-status (job 77)" +check_reran 1 "a head with no other-lane run of its own is re-run" +STUB_LANE_RUNS="$(runs "$(run 3 in_progress "$HEAD" "$SEC")" "$(run 4 completed "$HEAD" "$SEC")")" +expect "an older other-lane run still going does not hold the newest, finished one" 0 "Re-ran ci-status (job 77)" +check_reran 1 "only the other lane's newest run decides" -STUB_RUNS_SEQ="$(runs "$(run 9 in_progress)") -$(runs "$(run 9 in_progress)") -$(runs "$(run 9)")" STUB_JOBS="$(cijob in_progress)" +# One lane only on the head (an exempt author, a lane that never ran), or the +# other lane skipped, cancelled or failed: this lane is the last, so re-run. +STUB_LANE_RUNS="$(runs)" +expect "no run of the other lane: re-run on this lane alone" 0 "Re-ran ci-status (job 77) of run 9" +check_reran 1 "a single-lane head is re-run" +check_calls 5 "a single-lane head costs the same as the last lane" +for c in skipped cancelled failure; do + STUB_LANE_RUNS="$(runs "$(jq -c --arg c "$c" '. + {conclusion: $c}' <<<"$(run 3 completed "$HEAD" "$SEC")")")" + expect "the other lane completed $c counts as finished" 0 "Re-ran ci-status (job 77)" + check_reran 1 "the other lane $c: re-run" +done +STUB_LANE_RUNS="$(runs "$(run 3 completed "$HEAD" "$SEC")")" + +# ci-status that already read both finished lanes is left alone. +STUB_JOBS="$(cijob completed "$AFTER")" +expect "a ci-status that started after both lanes finished is left alone" 0 "started at $AFTER, after both lanes finished ($LATER)" +check_reran 0 "a fresh ci-status is not re-run" +check_calls 3 "a fresh ci-status costs no PR read and no re-run" +STUB_JOBS="$(cijob completed 2026-10-23T10:03:00Z)" +expect "a ci-status that started between the lanes is re-run" 0 "Re-ran ci-status (job 77)" +STUB_LANE_RUNS="$(runs "$(run 3 completed "$HEAD" "$SEC" pull_request 2026-10-23T09:40:00Z)")" STUB_JOBS="$(cijob completed 2026-10-23T09:55:00Z)" +expect "a ci-status that started before this lane finished is re-run" 0 "Re-ran ci-status (job 77)" +STUB_LANE_RUNS="$(runs "$(run 3 completed "$HEAD" "$SEC")")" STUB_JOBS="$(cijob completed "$LATER")" +expect "a ci-status that started the second the last lane finished is re-run" 0 "Re-ran ci-status (job 77)" +STUB_JOBS="$(cijob completed 2026-10-23T10:06:00Z)" +expect "a ci-status that started within 60 s of the last lane is re-run" 0 "Re-ran ci-status (job 77)" +STUB_JOBS="$(cijob completed "$AFTER")" STUB_LANE_RUNS="$(runs "$(run 3 completed "$HEAD" "$SEC" pull_request 2026-10-23T10:05)")" +expect "an unreadable finish time of the other lane means a re-run" 0 "Re-ran ci-status (job 77)" +STUB_LANE_RUNS="$(runs "$(run 3 completed "$HEAD" "$SEC")")" +STUB_RUNS="$(runs "$(run 9 queued)")" STUB_JOBS="$(cijob queued "")" +expect "a ci-status re-run already queued is left alone" 0 "ci-status of run 9 has not started" +check_reran 0 "the second of two racing lanes adds no re-run" +check_calls 3 "a queued ci-status costs no PR read" +STUB_JOBS="$(cijob in_progress "$AFTER")" STUB_RUNS="$(runs "$(run 9 in_progress)")" +expect "a ci-status in progress that started after both lanes is left alone" 0 "after both lanes finished" +if grep -q '^sleep' "$TMP_ROOT/log"; then fail "a fresh ci-status was polled"; else ok "a fresh ci-status in progress is not polled"; fi + +# A stale ci-status in progress waits for its run, reading only the run. +STUB_RUNS="$(runs "$(run 9 in_progress)")" STUB_JOBS="$(cijob in_progress)" +STUB_RUN_SEQ="$(run 9 in_progress) +$(run 9)" expect "a ci-status in progress is polled, then re-run once its run finishes" 0 "Re-ran ci-status (job 77) of run 9" check_reran 1 "the polled run's ci-status is re-run once" if [[ "$(grep -c '^sleep 60$' "$TMP_ROOT/log")" == 2 ]]; then ok "it polled every 60 s"; else fail "polls: $(grep '^sleep' "$TMP_ROOT/log" | paste -sd'|')"; fi -STUB_RUNS_SEQ="$(runs "$(run 9 in_progress)")" +check_calls 7 "each poll reads the run only" +if [[ "$(grep -c '/jobs?' "$TMP_ROOT/log")" == 1 ]]; then ok "the job list is read once"; else fail "job list reads: $(grep -c '/jobs?' "$TMP_ROOT/log")"; fi +STUB_RUN_SEQ="$(run 9 in_progress)" expect "a run still going after the limit fails" 1 "::error::run 9 on $HEAD has not finished after 600s" check_reran 0 "nothing is re-run past the poll limit" if [[ "$(grep -c '^sleep 60$' "$TMP_ROOT/log")" == 10 ]]; then ok "it stops after 10 minutes of polls"; else fail "polls: $(grep -c '^sleep 60$' "$TMP_ROOT/log")"; fi +check_calls 13 "a 10-minute wait reads the run 10 times" for st in queued waiting pending; do - STUB_JOBS="$(cijob "$st")" + STUB_JOBS="$(cijob "$st" "")" expect "a ci-status $st is left to read the reviews when it starts" 0 "ci-status of run 9 has not started" check_reran 0 "a $st ci-status is not re-run" done STUB_JOBS='{"jobs":[]}' expect "a run with no ci-status job listed yet is left alone" 0 "has not started" -STUB_RUNS_SEQ="$(runs "$(run 9)")" +STUB_RUNS="$(runs "$(run 9)")" expect "a finished run with no ci-status job fails" 1 "::error::run 9 on $HEAD has no ci-status job to re-run." -STUB_RUNS_SEQ="$(runs)" STUB_JOBS="$(cijob completed)" +STUB_RUNS="$(runs)" STUB_JOBS="$(cijob completed)" expect "no run on the head yet is left alone" 0 "No pr-require-checks run on $HEAD yet" check_reran 0 "nothing is re-run with no run" -STUB_RUNS_SEQ="$(runs "$(run 9)")" +STUB_RUNS="$(runs "$(run 9)")" STUB_PR="$(pr closed)" expect "a closed PR is left alone" 0 "#42 is closed or no longer at $HEAD" check_reran 0 "a closed PR's ci-status is not re-run" STUB_PR="$(pr open "${HEAD//0/f}")" expect "a head the PR moved past is left alone" 0 "no longer at $HEAD" -if grep -q '/runs?' "$TMP_ROOT/log"; then fail "a moved head's runs were read"; else ok "a moved head reads no runs"; fi +check_reran 0 "a moved head's ci-status is not re-run" STUB_PR="$(pr)" PR_NUMBER='' expect "a review run with no pull request is left alone" 0 "names no pull request" if [[ -s "$TMP_ROOT/log" ]]; then fail "a run with no pull request read the API"; else ok "a run with no pull request reads nothing"; fi PR_NUMBER='1;id' expect "a pull request number that is not a number is left alone" 0 "names no pull request" HEAD_SHA=main expect "a head that is not a commit SHA fails" 1 "::error::the review run's head 'main' is not a commit SHA." +LANE_PATH=.github/workflows/pr-verify.yml expect "a run of another file fails" 1 "::error::'.github/workflows/pr-verify.yml' is not a review lane." +LANE_DONE_AT='2026-10-23 10:00' expect "a finish time that is not a UTC timestamp fails" 1 "is not a UTC timestamp." +if [[ -s "$TMP_ROOT/log" ]]; then fail "a bad finish time read the API"; else ok "a bad finish time reads nothing"; fi + +# A primary rate limit waits for its reset inside the cap, then reads again. +export STUB_LIMIT_FILE="$TMP_ROOT/limit" STUB_RESET=1119 +STUB_PR="$(pr)" STUB_LANE_RUNS="$(runs "$(run 3 completed "$HEAD" "$SEC")")" STUB_RUNS="$(runs "$(run 9)")" STUB_JOBS="$(cijob completed)" +echo 1 >"$STUB_LIMIT_FILE" +expect "a rate-limited read waits for the reset, then re-runs" 0 "Rate limited reading the pr-require-checks runs on $HEAD; waiting 120s for the reset." +check_reran 1 "the read after the reset goes on to the re-run" +if [[ "$(grep -c '^sleep 120$' "$TMP_ROOT/log")" == 1 ]]; then ok "it slept until one second past the reset"; else fail "sleeps: $(grep '^sleep' "$TMP_ROOT/log" | paste -sd'|')"; fi +echo 1 >"$STUB_LIMIT_FILE" +STUB_RETRY_AFTER=30 expect "a retry-after refusal waits that many seconds" 0 "waiting 30s for the reset." +check_reran 1 "the read after retry-after goes on to the re-run" +echo 1 >"$STUB_LIMIT_FILE" +STUB_RESET=1700 expect "a reset past the cap fails, naming the reset" 1 "::error::the token's rate limit blocks reading the pr-require-checks runs on $HEAD until" +check_reran 0 "nothing is re-run when the reset is past the cap" +if grep -q '^sleep' "$TMP_ROOT/log"; then fail "it waited for a reset past the cap"; else ok "a reset past the cap is not waited for"; fi +echo 1 >"$STUB_LIMIT_FILE" +STUB_RUNS="$(runs "$(run 9 in_progress)")" STUB_JOBS="$(cijob in_progress)" STUB_RUN_SEQ="$(run 9 in_progress)" +expect "the reset wait counts toward the poll cap" 1 "has not finished after 600s" +if [[ "$(grep -c '^sleep 60$' "$TMP_ROOT/log")" == 8 ]]; then ok "a 120 s reset wait leaves 8 polls"; else fail "polls: $(grep -c '^sleep 60$' "$TMP_ROOT/log")"; fi +unset STUB_LIMIT_FILE STUB_RESET +STUB_RUNS="$(runs "$(run 9)")" STUB_JOBS="$(cijob completed)" +export STUB_LANE_FAIL=1 +expect "an unreadable other-lane run list fails" 1 "::error::could not read the pr-review-security-hosted.yml runs on $HEAD; re-run this run." +unset STUB_LANE_FAIL export STUB_RUNS_FAIL=1 expect "an unreadable run list fails" 1 "::error::could not read the pr-require-checks runs on $HEAD; re-run this run." unset STUB_RUNS_FAIL