diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02decedf3..673789e53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,13 +106,13 @@ jobs: # fail. Baseline growth vs the PR base is the next step below. bash scripts/ci-gates.sh rules --baseline scripts/rule-enforcement-baseline.tsv . - # Three legs, SHARD=i/3 like selftests: each runs only the scripts/ci-tests.d registrations that scripts/ci-tests-shard.sh lists + # Four legs, SHARD=i/4 (the i/N spelling selftests uses): each runs only the scripts/ci-tests.d registrations that scripts/ci-tests-shard.sh lists # for it, so the job takes as long as its slowest slice, not the sum. `gates` needs `perpr`, so it fails unless every leg passes. perpr: strategy: fail-fast: false matrix: - shard: [1, 2, 3] + shard: [1, 2, 3, 4] runs-on: ubuntu-latest timeout-minutes: 35 steps: @@ -133,7 +133,7 @@ jobs: # This leg's registrations (serial and background) are listed once, here, into $RUNNER_TEMP/bg/leg; both steps read that list. - name: Start background per-PR tests (scripts/ci-tests.d/*.bg.sh) env: - SHARD: ${{ matrix.shard }}/3 + SHARD: ${{ matrix.shard }}/4 run: | mkdir -p "$RUNNER_TEMP/bg" date +%s >"$RUNNER_TEMP/bg/started" @@ -145,7 +145,7 @@ jobs: f="$RUNNER_TEMP/bg/$(basename "$t")" : >"$f.log" # shellcheck disable=SC2016 # $1 and $2 expand in the child shell, on purpose - nohup bash -c 'timeout 900 bash -e "$1" >"$2.log" 2>&1; echo $? >"$2.rc.tmp"; mv "$2.rc.tmp" "$2.rc"' _ "$t" "$f" >/dev/null 2>&1 & + nohup bash -c 'timeout 900 bash -e "$1" >"$2.log" 2>&1; echo $? >"$2.rc.tmp"; date +%s >"$2.end"; mv "$2.rc.tmp" "$2.rc"' _ "$t" "$f" >/dev/null 2>&1 & done # Per-PR tests: a PR registers its test by adding scripts/ci-tests.d/.sh (one command line), @@ -171,6 +171,8 @@ jobs: end=$(($(cat "$RUNNER_TEMP/bg/started") + ${BG_WAIT_S:-960})) # counted from the background start, not from here while [ ! -f "$f.rc" ] && [ "$(date +%s)" -lt "$end" ]; do sleep 1; done cat "$f.log" + took='?'; [ -f "$f.end" ] && took=$(($(cat "$f.end") - $(cat "$RUNNER_TEMP/bg/started"))) + echo "bg elapsed: ${f##*/} $took s" [ -f "$f.rc" ] || { echo "FAIL: $t did not finish in time"; exit 1; } [ "$(cat "$f.rc")" = 0 ] done diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d5a06d15a..98081f4ba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,14 +37,24 @@ Parallel lanes must not bump the version: each one claims "main + 1" and all but or `size-budgets.tsv` rows: committing them makes any two PRs conflict once one merges. CI rebuilds them for an unlanded PR (`scripts/regen-unlanded.sh`; the size ratchet still compares the rebuilt rows to main), and the land commit carries them and is checked byte-for-byte. Register a new test by adding - `scripts/ci-tests.d/.sh` (one command line), not a line in `ci.yml`. CI splits the registrations over 3 - parallel legs by sorted position (`SHARD=i/3 bash scripts/ci-tests-shard.sh` lists leg i's), so a test must not rely on - another registration having run first. + `scripts/ci-tests.d/.sh` (one command line), not a line in `ci.yml`. CI splits the registrations over the + parallel `perpr` legs by recorded seconds (`scripts/ci-durations.tsv`: the longest test goes first, to the least-loaded leg; a + test with no row counts as the median, so adding one needs no table edit; `*.bg.sh` ones go round-robin; + `SHARD=i/N bash scripts/ci-tests-shard.sh` lists leg i's of the N legs), so a test must not rely on another registration having run first. `bash scripts/run-ci-tests.sh --affected` runs only the registered tests a changed file can reach (the pre-push default), so run it before pushing instead of just your own new test. `bash scripts/ci-gates.sh fragments --config scripts/size-budgets.tsv changelog.d` checks the fragment's marker lines and headings (the pre-commit and pre-push hooks, land and CI run it), so a malformed fragment fails your commit instead of the release. +- **Test-time table:** refresh `scripts/ci-durations.tsv` when the slowest `perpr` leg's `Per-PR tests` step is more than 20% + over the fastest on a PR run (until then a new test counts as the median, so only the balance drifts). Save the job log of + every `perpr` leg as `leg1.log` ... `legN.log` (`gh api repos///actions/jobs//logs`; the job id is in + the job's URL on the run page), rebuild the rows (the seconds between consecutive `== ` lines, rounded, at least 1) + and put them under the table's header with its run id and date updated: + + ```bash + awk 'FNR == 1 { p = "" } $2 == "==" && $3 ~ /^scripts\/ci-tests\.d\// { t = substr($1, 12, 2) * 3600 + substr($1, 15, 2) * 60 + substr($1, 18); d = t - q; if (d < 0) d += 86400; if (p != "") print p "\t" (d < 1.5 ? 1 : int(d + 0.5)); p = ($3 ~ /\.bg\.sh$/ ? "" : $3); q = t }' leg*.log | LC_ALL=C sort + ``` - **Land:** on the branch, run `bash scripts/land-release.sh`. Under a lock it merges `origin/main` (generated-file conflicts are regenerated, anything else aborts), stamps the next minor across the lockstep set, folds the fragments into `CHANGELOG.md`, regenerates, and commits. Push and merge as diff --git a/changelog.d/perpr-shard-balance.md b/changelog.d/perpr-shard-balance.md new file mode 100644 index 000000000..dcd67b42a --- /dev/null +++ b/changelog.d/perpr-shard-balance.md @@ -0,0 +1,4 @@ +### Changed +- The CI `perpr` job now runs four legs instead of three (`shard: [1, 2, 3, 4]`, `SHARD=i/4`), and its legs are balanced by recorded test time instead of by sorted position. `scripts/ci-tests-shard.sh` reads `scripts/ci-durations.tsv` (whole seconds per `scripts/ci-tests.d/*.sh` registration: the seconds between consecutive `== ` lines of the `perpr` job logs of public CI run 41148962559 (2026-10-31), rounded, at least 1), takes the serial registrations longest first (ties by name, C locale) and gives each to the least-loaded leg (ties to the lowest index). A registration with no row counts as the table's median, so adding a test never needs a table edit and two PRs never conflict on the table. `*.bg.sh` registrations keep the sorted round-robin, because they run beside the serial ones, so only serial seconds are balanced. The contract is unchanged: the `i/N` slices are disjoint and cover every registration for any N, a malformed `SHARD` exits 2, an empty slice exits 1, the listing is sorted and does not depend on the locale. On the committed table, which counts every test at least 1 s (so these are predictions from the table, not CI measurements), the old helper with three legs on the tree before this change gives legs of 283, 489 and 682 s of serial tests, and the new split over four legs gives 364, 364, 364 and 363 s. +- The `Per-PR tests` step also prints one `bg elapsed: s` line per background test (its wall time from the background start, `?` when it left no end time), because the `== ` lines show only how long the step waited for it, not how long it ran. +- Test: `scripts/test-perpr-shard-balance.sh` (registered as `scripts/ci-tests.d/perpr-shard-balance.sh`) checks the committed table's format, that the slices are disjoint and complete for N = 1 to 5, that the slowest of the legs `ci.yml` runs (the N its `perpr` legs pass in `SHARD=i/N`, read from `ci.yml`) is predicted within 1.1x of the mean (it fails on the old split), and, on a fixture, the exact longest-first assignment, the median default for a malformed or missing row, and the round-robin of background tests. `scripts/test-perpr-shard.sh` no longer asserts that the slices hold nearly equal numbers of registrations, which only the old split guaranteed, and now checks that each leg logs its `bg elapsed:` line. `CONTRIBUTING.md` describes the new split, says when to refresh the table (the slowest leg's `Per-PR tests` step more than 20% over the fastest on a PR run) and gives the awk command that rebuilds its rows from the job logs of the legs; the test runs that command on synthetic logs. diff --git a/scripts/ci-durations.tsv b/scripts/ci-durations.tsv new file mode 100644 index 000000000..b4554c44d --- /dev/null +++ b/scripts/ci-durations.tsv @@ -0,0 +1,287 @@ +# Whole seconds each scripts/ci-tests.d serial registration took in the perpr legs of public CI run 41148962559 (2026-10-31): +# the seconds between consecutive "== " lines of the three job logs, rounded, at least 1 (CONTRIBUTING.md has the rebuild command). +# scripts/ci-tests-shard.sh balances the perpr legs with it. A registration missing here counts as the median, so a new test needs no row. +scripts/ci-tests.d/admission-check.sh 1 +scripts/ci-tests.d/affected-receipt.sh 14 +scripts/ci-tests.d/always-loaded-bytes.sh 1 +scripts/ci-tests.d/always-loaded-surface.sh 1 +scripts/ci-tests.d/audit-pr-body.sh 5 +scripts/ci-tests.d/audit-pr-budget.sh 1 +scripts/ci-tests.d/audit-pr-footer-markup.sh 1 +scripts/ci-tests.d/audit-pr-footer-rule.sh 1 +scripts/ci-tests.d/audit-skills-control-char.sh 1 +scripts/ci-tests.d/audit-skills.sh 14 +scripts/ci-tests.d/background-work-guard.sh 1 +scripts/ci-tests.d/bash3-portability.sh 1 +scripts/ci-tests.d/bench-check-frozen.sh 2 +scripts/ci-tests.d/bench-check-ledger.sh 1 +scripts/ci-tests.d/bench-cost-capture.sh 1 +scripts/ci-tests.d/bg-wait-claim.sh 1 +scripts/ci-tests.d/bizops-check.sh 1 +scripts/ci-tests.d/bizops-fab-bench.sh 1 +scripts/ci-tests.d/budget-below-timeout.sh 1 +scripts/ci-tests.d/ceo-altitude-guard.sh 7 +scripts/ci-tests.d/ceo-guard-default-warn.sh 12 +scripts/ci-tests.d/ceo-start-load.sh 1 +scripts/ci-tests.d/ci-cost-advisor.sh 24 +scripts/ci-tests.d/ci-gates-sandbox-skip.sh 1 +scripts/ci-tests.d/ci-shards.sh 1 +scripts/ci-tests.d/clock-lint-jstiming.sh 3 +scripts/ci-tests.d/clock-lint-reltext.sh 1 +scripts/ci-tests.d/clock-lint-timing.sh 2 +scripts/ci-tests.d/committed-personal-data-fixture-label.sh 1 +scripts/ci-tests.d/committed-personal-data.sh 10 +scripts/ci-tests.d/comms-e4-confirm.sh 2 +scripts/ci-tests.d/comms-lint-comments.sh 1 +scripts/ci-tests.d/comms-lint-placeholder-template.sh 1 +scripts/ci-tests.d/comms-outcome-bench.sh 1 +scripts/ci-tests.d/comms-outcome-r2.sh 1 +scripts/ci-tests.d/comms-quality.sh 1 +scripts/ci-tests.d/comms-reader-check.sh 1 +scripts/ci-tests.d/confirm-v2.sh 15 +scripts/ci-tests.d/context-audit.sh 1 +scripts/ci-tests.d/cost-advice-levers.sh 1 +scripts/ci-tests.d/cost-advice.sh 6 +scripts/ci-tests.d/cost-statusline-window.sh 1 +scripts/ci-tests.d/council-bench.sh 2 +scripts/ci-tests.d/critic-findings.sh 1 +scripts/ci-tests.d/delivery-peer-mechanisms.sh 1 +scripts/ci-tests.d/design-drift.sh 2 +scripts/ci-tests.d/destructive-guard-w21-wait.sh 1 +scripts/ci-tests.d/differentiator-bench.sh 1 +scripts/ci-tests.d/discovery-check.sh 1 +scripts/ci-tests.d/doctor-orphan-dbs.sh 3 +scripts/ci-tests.d/doctor-stale-local-skills.sh 1 +scripts/ci-tests.d/dynamic-probe.sh 1 +scripts/ci-tests.d/evil-twin-bench-r2.sh 1 +scripts/ci-tests.d/evil-twin-bench.sh 1 +scripts/ci-tests.d/feeder-order.sh 1 +scripts/ci-tests.d/finding-verdict.sh 1 +scripts/ci-tests.d/flake-ledger.sh 57 +scripts/ci-tests.d/flake-serial.sh 1 +scripts/ci-tests.d/flow-guard-fixes.sh 2 +scripts/ci-tests.d/focus-kit.sh 1 +scripts/ci-tests.d/ga-readout-check.sh 1 +scripts/ci-tests.d/gate-cache.sh 18 +scripts/ci-tests.d/gh-shim.sh 7 +scripts/ci-tests.d/growth-outcome-1019.sh 2 +scripts/ci-tests.d/growth-outcome-r2-1019.sh 3 +scripts/ci-tests.d/guard-registry.sh 4 +scripts/ci-tests.d/handback-dirty.sh 1 +scripts/ci-tests.d/handback-merge.sh 5 +scripts/ci-tests.d/handback-mode-coverage.sh 1 +scripts/ci-tests.d/handback-spec.sh 5 +scripts/ci-tests.d/handback-tests-count.sh 4 +scripts/ci-tests.d/handback.sh 6 +scripts/ci-tests.d/heavy-gate-holders.sh 1 +scripts/ci-tests.d/heavy-gate-scanners.sh 1 +scripts/ci-tests.d/hermetic-env-lint.sh 1 +scripts/ci-tests.d/hidden-test-bench-r2.sh 1 +scripts/ci-tests.d/hidden-test-bench.sh 1 +scripts/ci-tests.d/hidden-test-pool.sh 1 +scripts/ci-tests.d/hold-reread.sh 1 +scripts/ci-tests.d/hook-friction.sh 1 +scripts/ci-tests.d/host-accept.sh 1 +scripts/ci-tests.d/host-scaling.sh 1 +scripts/ci-tests.d/idea-commands.sh 1 +scripts/ci-tests.d/idea-critic-calibration.sh 1 +scripts/ci-tests.d/idea-critic-outcome.sh 1 +scripts/ci-tests.d/idea-critic-v2.sh 1 +scripts/ci-tests.d/idea-critic-v3.sh 1 +scripts/ci-tests.d/idea-critic-v4-xmodel.sh 1 +scripts/ci-tests.d/idea-critic-v4.sh 1 +scripts/ci-tests.d/idea-critic-v5.sh 1 +scripts/ci-tests.d/identity-gate-rebase.sh 1 +scripts/ci-tests.d/identity-gate.sh 8 +scripts/ci-tests.d/install-copy-drift.sh 1 +scripts/ci-tests.d/install-hygiene.sh 6 +scripts/ci-tests.d/install-policy-no-disown.sh 5 +scripts/ci-tests.d/install-update-hygiene.sh 15 +scripts/ci-tests.d/invisible-chars-hardening.sh 6 +scripts/ci-tests.d/invisible-chars.sh 1 +scripts/ci-tests.d/job-lease.sh 23 +scripts/ci-tests.d/land-ready.sh 3 +scripts/ci-tests.d/landing-watchdog-once.sh 1 +scripts/ci-tests.d/landing-watchdog.sh 4 +scripts/ci-tests.d/lane-guard-shadow-fix.sh 1 +scripts/ci-tests.d/lane-stall.sh 1 +scripts/ci-tests.d/lane-worktree-reap.sh 2 +scripts/ci-tests.d/lease-sandbox-fallback.sh 1 +scripts/ci-tests.d/lease-sandbox-hardening.sh 1 +scripts/ci-tests.d/lint-ui-slop.sh 4 +scripts/ci-tests.d/lkg-pointer.sh 2 +scripts/ci-tests.d/lkg-runner-provenance.sh 1 +scripts/ci-tests.d/low-context-index.sh 35 +scripts/ci-tests.d/memory-handoff-hygiene.sh 1 +scripts/ci-tests.d/mode-coverage.sh 5 +scripts/ci-tests.d/mutation-equivalent.sh 1 +scripts/ci-tests.d/mutation-kills.sh 3 +scripts/ci-tests.d/nested-worktree-liveness.sh 1 +scripts/ci-tests.d/orphan-tests-misc.sh 9 +scripts/ci-tests.d/orphan-tests.sh 6 +scripts/ci-tests.d/outcome-claim-guard-ui.sh 1 +scripts/ci-tests.d/outcome-claim-guard.sh 2 +scripts/ci-tests.d/output-safety-check.sh 1 +scripts/ci-tests.d/overclaim-bench-r3.sh 1 +scripts/ci-tests.d/overclaim-bench-r4.sh 4 +scripts/ci-tests.d/overclaim-bench.sh 1 +scripts/ci-tests.d/overclaim-r4-labels.sh 1 +scripts/ci-tests.d/parity-differ-focus-gate.sh 1 +scripts/ci-tests.d/partial-pass.sh 1 +scripts/ci-tests.d/peer-learnings-1019.sh 2 +scripts/ci-tests.d/peer-lessons-batch.sh 4 +scripts/ci-tests.d/peer-send-alarm-relay.sh 1 +scripts/ci-tests.d/perf-tools.sh 4 +scripts/ci-tests.d/perpr-shard.sh 1 +scripts/ci-tests.d/pipeline-hub.sh 1 +scripts/ci-tests.d/pipeline-optimizer.sh 5 +scripts/ci-tests.d/pipeline-profile-tests-cmd.sh 1 +scripts/ci-tests.d/plan-gate.sh 1 +scripts/ci-tests.d/positioning-check.sh 1 +scripts/ci-tests.d/pr-state.sh 4 +scripts/ci-tests.d/precheck-order.sh 1 +scripts/ci-tests.d/premise-check.sh 1 +scripts/ci-tests.d/prepush-lint-nice.sh 3 +scripts/ci-tests.d/prepush-nice.sh 1 +scripts/ci-tests.d/privacy-guard.sh 10 +scripts/ci-tests.d/provision-semgrep-warm.sh 1 +scripts/ci-tests.d/provision-tools.sh 19 +scripts/ci-tests.d/rails.sh 2 +scripts/ci-tests.d/ready-pr-age.sh 1 +scripts/ci-tests.d/red-main-watchdog.sh 1 +scripts/ci-tests.d/red-main.sh 1 +scripts/ci-tests.d/repo-settings-audit.sh 1 +scripts/ci-tests.d/review-appsec-split.sh 1 +scripts/ci-tests.d/review-checks-dep-fallback.sh 9 +scripts/ci-tests.d/review-checks-full.sh 6 +scripts/ci-tests.d/review-checks-gitleaks-allow.sh 9 +scripts/ci-tests.d/review-checks-gitleaks-fixture-bucket.sh 20 +scripts/ci-tests.d/review-checks-gitleaks-fixture-hint.sh 4 +scripts/ci-tests.d/review-checks-gitleaks-template.sh 1 +scripts/ci-tests.d/review-checks-guarddog.sh 28 +scripts/ci-tests.d/review-checks-history-range.sh 1 +scripts/ci-tests.d/review-checks-lease-degraded.sh 2 +scripts/ci-tests.d/review-checks-lease-policy.sh 2 +scripts/ci-tests.d/review-checks-new-rows.sh 3 +scripts/ci-tests.d/review-checks-scan-summary-hermetic.sh 5 +scripts/ci-tests.d/review-checks-scan-summary-registry-env.sh 5 +scripts/ci-tests.d/review-checks-scan-summary.sh 6 +scripts/ci-tests.d/review-checks-scanner-gaps.sh 12 +scripts/ci-tests.d/review-checks-security-folds.sh 11 +scripts/ci-tests.d/review-checks-security.sh 4 +scripts/ci-tests.d/review-checks-semgrep-default.sh 2 +scripts/ci-tests.d/review-checks-tool-hub.sh 56 +scripts/ci-tests.d/review-checks-trufflehog.sh 23 +scripts/ci-tests.d/review-cost-matched.sh 2 +scripts/ci-tests.d/review-cost-per-recall.sh 1 +scripts/ci-tests.d/review-cost-v2.sh 1 +scripts/ci-tests.d/review-fullmap-core.sh 1 +scripts/ci-tests.d/review-gate-rereview.sh 1 +scripts/ci-tests.d/review-latency-split.sh 1 +scripts/ci-tests.d/review-load-bytes.sh 1 +scripts/ci-tests.d/review-method-split.sh 1 +scripts/ci-tests.d/review-read-budget.sh 1 +scripts/ci-tests.d/review-skips-index.sh 1 +scripts/ci-tests.d/rival-answer-keys.sh 1 +scripts/ci-tests.d/rival-eval.sh 1 +scripts/ci-tests.d/rival-h2h.sh 1 +scripts/ci-tests.d/rivals-outcome-audit.sh 1 +scripts/ci-tests.d/route-auth-map.sh 25 +scripts/ci-tests.d/routing-dedupe.sh 1 +scripts/ci-tests.d/rules-locale.sh 1 +scripts/ci-tests.d/run-ci-gates.sh 4 +scripts/ci-tests.d/run-ci-tests-affected.sh 3 +scripts/ci-tests.d/run-ci-tests-always.sh 1 +scripts/ci-tests.d/run-ledger.sh 6 +scripts/ci-tests.d/safety-bias-r1-1019.sh 1 +scripts/ci-tests.d/safety-outcome-1019.sh 3 +scripts/ci-tests.d/scanner-reporting-fixes.sh 6 +scripts/ci-tests.d/scanner-sandbox-fallback.sh 7 +scripts/ci-tests.d/scoreboard-token-voice.sh 1 +scripts/ci-tests.d/scoreboard.sh 1 +scripts/ci-tests.d/secret-anthropic-airtable.sh 1 +scripts/ci-tests.d/secret-rule-waves.sh 14 +scripts/ci-tests.d/secret-scan-union.sh 2 +scripts/ci-tests.d/security-gates-corpus.sh 26 +scripts/ci-tests.d/security-gates-heldout-fixes.sh 10 +scripts/ci-tests.d/security-gates-heldout.sh 39 +scripts/ci-tests.d/security-gates-heldout2.sh 23 +scripts/ci-tests.d/security-gates-heldout3.sh 49 +scripts/ci-tests.d/security-heldout-table.sh 55 +scripts/ci-tests.d/self-approval-guard.sh 1 +scripts/ci-tests.d/semgrep-gha-actor-allowlist.sh 1 +scripts/ci-tests.d/semgrep-hardcoded-cred.sh 1 +scripts/ci-tests.d/semgrep-js-date-gcm.sh 1 +scripts/ci-tests.d/semgrep-pack-fallback.sh 11 +scripts/ci-tests.d/semgrep-registry-timeout.sh 19 +scripts/ci-tests.d/semgrep-taint.sh 1 +scripts/ci-tests.d/semgrep-typeof-compare.sh 1 +scripts/ci-tests.d/serial-project.sh 2 +scripts/ci-tests.d/sibling-sweep.sh 1 +scripts/ci-tests.d/skill-index-split.sh 1 +scripts/ci-tests.d/slim-bizops-fast-path.sh 1 +scripts/ci-tests.d/slim-ceo-fast-path.sh 1 +scripts/ci-tests.d/slim-contribution-fast-path.sh 1 +scripts/ci-tests.d/slim-critic-fast-path.sh 1 +scripts/ci-tests.d/slim-delivery-skill.sh 1 +scripts/ci-tests.d/slim-ga-fast-path.sh 1 +scripts/ci-tests.d/slim-pd-fast-path.sh 1 +scripts/ci-tests.d/slim-pos-fast-path.sh 1 +scripts/ci-tests.d/slim-positioning-fast-path.sh 1 +scripts/ci-tests.d/slim-skill-fast-path.sh 1 +scripts/ci-tests.d/spend-hub.sh 1 +scripts/ci-tests.d/spend-nudge.sh 3 +scripts/ci-tests.d/spend-rollup-groups.sh 1 +scripts/ci-tests.d/spend-rollup-nits.sh 1 +scripts/ci-tests.d/spend-rollup.sh 1 +scripts/ci-tests.d/stage-run.sh 118 +scripts/ci-tests.d/stage-weights.sh 1 +scripts/ci-tests.d/stall-guard.sh 5 +scripts/ci-tests.d/supply-chain-check.sh 2 +scripts/ci-tests.d/supply-chain-osv-fallback.sh 1 +scripts/ci-tests.d/supply-chain-reach.sh 1 +scripts/ci-tests.d/surface-check-deps.sh 6 +scripts/ci-tests.d/surface-check-skip-token.sh 1 +scripts/ci-tests.d/teardown.sh 1 +scripts/ci-tests.d/tenant-scope-check.sh 1 +scripts/ci-tests.d/tests-wired.sh 1 +scripts/ci-tests.d/token-active-window.sh 1 +scripts/ci-tests.d/token-apply.sh 1 +scripts/ci-tests.d/token-burn-blocks.sh 1 +scripts/ci-tests.d/token-cache-gap.sh 1 +scripts/ci-tests.d/token-compact-saving.sh 1 +scripts/ci-tests.d/token-haiku-tier.sh 1 +scripts/ci-tests.d/token-lever-rank.sh 1 +scripts/ci-tests.d/token-levers.sh 1 +scripts/ci-tests.d/token-nits.sh 1 +scripts/ci-tests.d/token-outcome.sh 1 +scripts/ci-tests.d/token-rollup-buckets.sh 1 +scripts/ci-tests.d/token-startup-files.sh 1 +scripts/ci-tests.d/token-tail-cap.sh 1 +scripts/ci-tests.d/token-tool-results.sh 1 +scripts/ci-tests.d/train-budget.sh 32 +scripts/ci-tests.d/train-disjoint.sh 15 +scripts/ci-tests.d/train-flake-yield.sh 1 +scripts/ci-tests.d/train-gate.sh 7 +scripts/ci-tests.d/train-refusal.sh 1 +scripts/ci-tests.d/train-solo-port.sh 89 +scripts/ci-tests.d/train-stale-cancel.sh 9 +scripts/ci-tests.d/train-stale-self.sh 34 +scripts/ci-tests.d/train-vs-base.sh 1 +scripts/ci-tests.d/train-yield.sh 1 +scripts/ci-tests.d/veche-doc-claims.sh 1 +scripts/ci-tests.d/veche-failrecord.sh 1 +scripts/ci-tests.d/veche-input.sh 2 +scripts/ci-tests.d/veche-json-out-exit-code.sh 1 +scripts/ci-tests.d/veche-json-out-late-write.sh 1 +scripts/ci-tests.d/veche-record.sh 1 +scripts/ci-tests.d/veche-runtime.sh 5 +scripts/ci-tests.d/veche-surface.sh 2 +scripts/ci-tests.d/veche-uniq-comparators.sh 1 +scripts/ci-tests.d/veche-uniq-unicode-fold.sh 1 +scripts/ci-tests.d/wave-ab.sh 1 +scripts/ci-tests.d/wave-ae-mechanisms.sh 1 +scripts/ci-tests.d/wave-ak-mechanisms.sh 1 +scripts/ci-tests.d/wip-cap.sh 1 +scripts/ci-tests.d/xvendor-judge.sh 1 +scripts/ci-tests.d/zero-chat-lanes.sh 3 diff --git a/scripts/ci-tests-shard.sh b/scripts/ci-tests-shard.sh index 2b1f78f3d..ec9492807 100644 --- a/scripts/ci-tests-shard.sh +++ b/scripts/ci-tests-shard.sh @@ -1,15 +1,45 @@ #!/usr/bin/env bash # ci-tests-shard.sh — list the scripts/ci-tests.d registrations that shard SHARD=i/N runs, one path per line, sorted. -# Same SHARD=i/N spelling as scripts/test-ci-gates.sh (CI runs SHARD=i/3, one `perpr` matrix leg per slice); unset lists all. -# Sorted in the C locale, serial (*.sh) and background (*.bg.sh) registrations numbered separately from 0 so the slow -# background ones spread over the legs too; shard i takes the numbers equal to i-1 modulo N. The N slices are disjoint and -# cover every registration (a dotfile is not one: the step globs skip it). A malformed SHARD exits 2 and an empty slice -# exits 1, never a silently empty list. +# Same SHARD=i/N spelling as scripts/test-ci-gates.sh (CI runs one `perpr` matrix leg per slice); unset lists all. +# Serial (*.sh) registrations are balanced by recorded time: scripts/ci-durations.tsv (pathwhole seconds, from one CI run) +# is taken longest first (ties by name, C locale) and each test goes to the least-loaded leg (ties to the lowest index). A +# registration with no valid row counts as the table's median (1 for an empty or missing table), so adding a test never needs +# a table edit. Background (*.bg.sh) registrations run beside the serial ones, so they are not weighed: sorted in the C locale +# and numbered from 0, shard i takes the numbers equal to i-1 modulo N. The N slices are disjoint and cover every registration +# (a dotfile is not one: the step globs skip it) whatever the table says. A malformed SHARD exits 2 and an empty slice exits 1, +# never a silently empty list. # Run from anywhere; paths are relative to the repo root, as the CI steps and run-ci-tests.sh spell them. set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." s="${SHARD:-1/1}" [[ $s =~ ^([1-9][0-9]*)/([1-9][0-9]*)$ ]] && [ "${BASH_REMATCH[1]}" -le "${BASH_REMATCH[2]}" ] \ || { echo "ci-tests-shard: SHARD must be i/N with 1 <= i <= N, got: $s" >&2; exit 2; } +# ponytail: the pick below is an O(tests^2) scan, fine for hundreds of registrations; sort the weights first if it reaches thousands +# awk-v-ok: i and n are positive integers checked by the regex above find scripts/ci-tests.d -maxdepth 1 -name '*.sh' ! -name '.*' | LC_ALL=C sort \ - | awk -v i="${BASH_REMATCH[1]}" -v n="${BASH_REMATCH[2]}" '{ k = /\.bg\.sh$/ } c[k]++ % n == i - 1' | grep . # awk-v-ok: i and n are positive integers checked by the regex above + | awk -F'\t' -v i="${BASH_REMATCH[1]}" -v n="${BASH_REMATCH[2]}" ' + BEGIN { + i += 0; n += 0 # i and n are compared below, so force numbers whatever the awk does with -v values + while ((getline < "scripts/ci-durations.tsv") > 0) + if ($2 ~ /^[1-9][0-9]*$/) { t[$1] = $2 + 0; v[++k] = $2 + 0 } + med = 1 # the (k/2+1)-th smallest row of the table + for (a = 1; a <= k; a++) { + lt = le = 0 + for (b = 1; b <= k; b++) { lt += (v[b] < v[a]); le += (v[b] <= v[a]) } + if (lt <= int(k / 2) && le > int(k / 2)) { med = v[a]; break } + } + } + /\.bg\.sh$/ { if (g++ % n == i - 1) print; next } + { nm[++s] = $0; w[s] = ($0 in t) ? t[$0] : med } + END { + for (r = 0; r < s; r++) { + x = 0 + for (j = 1; j <= s; j++) if (!d[j] && (!x || w[j] > w[x])) x = j # the first of the heaviest left is the lowest name + d[x] = 1 + m = 1 + # the least loaded leg, the lowest index on a tie; a leg past s gets no test (weights are >= 1), so the scan stops at s + for (j = 2; j <= n && j <= s; j++) if (L[j] + 0 < L[m] + 0) m = j + L[m] += w[x] + if (m == i) print nm[x] + } + }' | LC_ALL=C sort | grep . diff --git a/scripts/ci-tests.d/perpr-shard-balance.sh b/scripts/ci-tests.d/perpr-shard-balance.sh new file mode 100644 index 000000000..2a2a6307f --- /dev/null +++ b/scripts/ci-tests.d/perpr-shard-balance.sh @@ -0,0 +1 @@ +bash scripts/test-perpr-shard-balance.sh diff --git a/scripts/test-perpr-shard-balance.sh b/scripts/test-perpr-shard-balance.sh new file mode 100644 index 000000000..31df1a1e9 --- /dev/null +++ b/scripts/test-perpr-shard-balance.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Tests for the time balance of the CI `perpr` legs: scripts/ci-tests-shard.sh weighs every serial scripts/ci-tests.d +# registration with scripts/ci-durations.tsv. On the committed table the legs ci.yml's perpr job runs (the N its legs pass in +# SHARD=i/N, read from ci.yml, not assumed) are predicted within 10% of equal, the +# slices stay disjoint and complete for N = 1..5, and a registration the table lacks (or has no valid row for) counts as +# the table's median, so adding a test never needs a table edit. Background (*.bg.sh) registrations run beside the serial +# ones and stay round-robin. The awk command CONTRIBUTING.md gives for rebuilding the table runs on synthetic job logs. +# Hermetic: fixtures live under $TMPDIR; the checkout is only read. The malformed-SHARD and empty-slice exits are pinned in +# scripts/test-perpr-shard.sh. +# Prints PASS/FAIL per case and a final "Tests: P/N"; exits non-zero on any failure. +# shellcheck disable=SC2319 # each `ok $?` reads the status of the check on the line before it, on purpose +set -uo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +H="$ROOT/scripts/ci-tests-shard.sh" +T="$ROOT/scripts/ci-durations.tsv" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/dcr-shard-balance.XXXXXX")" || exit 1 +trap 'rm -rf "$WORK"' EXIT +pass=0 total=0 +ok() { total=$((total + 1)); if [ "$1" -eq 0 ]; then pass=$((pass + 1)); echo "PASS $2"; else echo "FAIL $2"; fi; } +slice() { env -u SHARD SHARD="$1" bash "${2:-$H}"; } # [helper]: the dry listing of one slice +rows() { grep -v '^#' "$T" | grep .; } # the table's data rows (comment lines dropped) + +# The committed table: one `pathwhole seconds >= 1` row per registration, no path twice, C-sorted so a refresh diffs row by row. +[ "$(rows | wc -l)" -gt 0 ] && ! rows | grep -qvE "^scripts/ci-tests\.d/[^[:space:]]+\.sh$(printf '\t')[1-9][0-9]*$" \ + && [ "$(rows | cut -f1)" = "$(rows | cut -f1 | LC_ALL=C sort -u)" ] +ok $? "scripts/ci-durations.tsv rows are pathseconds (>= 1), sorted, no path twice" + +# Disjoint and complete for any N: the N slices together list each registration exactly once, and none is empty. +all="$(cd "$ROOT" && printf '%s\n' scripts/ci-tests.d/*.sh | LC_ALL=C sort)" +bad="" +for N in 1 2 3 4 5; do + got=""; i=1 + while [ "$i" -le "$N" ]; do + s="$(slice "$i/$N")"; [ -n "$s" ] || bad="$bad $i/$N" + got="$got$s"$'\n'; i=$((i + 1)) + done + [ "$(grep . <<<"$got" | LC_ALL=C sort)" = "$all" ] || bad="$bad N=$N" +done +[ -z "$bad" ]; ok $? "N = 1..5: the slices of the real tree are non-empty, disjoint and cover every registration${bad:+ (bad:$bad)}" + +# Balance: predicted serial seconds per leg = the table seconds of the leg's *.sh registrations, a registration without a row at +# the table's median (the (n/2+1)-th smallest). Background tests are not counted: they run beside the serial ones. +med="$(rows | cut -f2 | sort -n | awk '{ a[NR] = $1 } END { print NR ? a[int(NR / 2) + 1] : 1 }')" +secs() { slice "$1" | awk -F'\t' -v t="$T" -v med="$med" 'BEGIN { while ((getline < t) > 0) w[$1] = $2 } !/\.bg\.sh$/ { s += ($0 in w) ? w[$0] : med } END { print s + 0 }'; } +# N = what ci.yml's perpr legs pass in SHARD=i/N (the same read as scripts/test-perpr-shard.sh), so a matrix change re-aims this case. +nlegs="$(awk '/^ [a-z]+:$/ { j = $1 } j == "perpr:"' "$ROOT/.github/workflows/ci.yml" | grep -oE 'SHARD: \$\{\{ matrix.shard \}\}/[0-9]+' | grep -oE '[0-9]+$' | sort -u)" +case "$nlegs" in '' | *[!0-9]* | 0*) nlegs=0 ;; esac # not exactly one positive N: the case below fails instead of dividing by it +tot=0 max=0 per=""; i=1 +while [ "$i" -le "$nlegs" ]; do s="$(secs "$i/$nlegs")"; per="$per${per:+/}$s"; tot=$((tot + s)); [ "$s" -gt "$max" ] && max=$s; i=$((i + 1)); done +[ "$tot" -gt 0 ] && [ $((max * nlegs * 10)) -le $((tot * 11)) ] # max <= 1.1 * tot / nlegs, in integers +ok $? "N=$nlegs, the legs of ci.yml's perpr job: the slowest leg is predicted within 1.1x of the mean (legs $per s, slowest $max, mean $((tot / (nlegs ? nlegs : 1))))" + +# A fixture root. Seconds: x1 10, x2 6, x3 6, x4 2, x5 2; u.sh has a malformed row (so no row); y.bg.sh and a stale row (gone.sh) carry +# big seconds and weigh no leg. They are valid rows, so they count in the median like any other (7 rows: the 4th smallest is 6) but +# do not move it here (5 rows: the 3rd smallest is 6), so u weighs 6. Longest first, ties by name: x1 u x2 x3 x4 x5; N=2 puts x1 on +# leg 1, u and x2 on leg 2, x3 on leg 1, x4 and x5 on leg 2 (a sorted-position split, or u at 0, 1 or 5, differs). +FX="$WORK/fx"; D=scripts/ci-tests.d; mkdir -p "$FX/$D"; cp "$H" "$FX/scripts/"; FH="$FX/scripts/ci-tests-shard.sh" +for n in u x1 x2 x3 x4 x5 y.bg z.bg; do echo : >"$FX/$D/$n.sh"; done +tab=$'\t' +printf '%s\n' "# a comment" "" "$D/x1.sh${tab}10" "$D/x2.sh${tab}6" "$D/x3.sh${tab}6" "$D/x4.sh${tab}2" "$D/x5.sh${tab}2" \ + "$D/u.sh${tab}fast" "$D/y.bg.sh${tab}99" "$D/gone.sh${tab}50" >"$FX/scripts/ci-durations.tsv" +[ "$(slice 1/2 "$FH")" = "$D/x1.sh"$'\n'"$D/x3.sh"$'\n'"$D/y.bg.sh" ] \ + && [ "$(slice 2/2 "$FH")" = "$D/u.sh"$'\n'"$D/x2.sh"$'\n'"$D/x4.sh"$'\n'"$D/x5.sh"$'\n'"$D/z.bg.sh" ] +ok $? "fixture N=2: longest first onto the least-loaded leg, a malformed row weighs the median, background stays round-robin" +echo : >"$FX/$D/zz-new.sh" # a new registration with no table row +got="$(slice 1/2 "$FH"; slice 2/2 "$FH")" +[ "$(grep -c . <<<"$got")" -eq 9 ] && [ "$(grep -c 'zz-new\.sh$' <<<"$got")" -eq 1 ] +ok $? "fixture: a registration with no table row is listed by exactly one slice (adding a test needs no table edit)" + +# CONTRIBUTING.md's rebuild command on synthetic logs. a.sh crosses midnight (5.4 s -> 5), b.sh is 0.2 s (at least 1), c.sh 4.4 s (4), +# d.sh 1.5 s (2), f.sh 7 s; the background waits (two in a row, as on leg 1) are no rows, and e.sh, the last serial line of leg2.log, +# gets no row (it must not pair with the first line of leg3.log). +LG="$WORK/logs"; mkdir -p "$LG" +printf '%s\n' "2026-10-30T23:59:58.2000000Z Current runner version" "2026-10-30T23:59:59.0000000Z == $D/a.sh" "2026-10-31T00:00:03.6000000Z output" \ + "2026-10-31T00:00:04.4000000Z == $D/b.sh" "2026-10-31T00:00:04.6000000Z == $D/c.sh" \ + "2026-10-31T00:00:09.0000000Z == $D/q.bg.sh (started in the background)" \ + "2026-10-31T00:00:12.0000000Z == $D/r.bg.sh (started in the background)" >"$LG/leg1.log" +printf '%s\n' "2026-10-31T00:01:00.0000000Z == $D/d.sh" "2026-10-31T00:01:01.5000000Z == $D/e.sh" >"$LG/leg2.log" +printf '%s\n' "2026-10-31T00:05:00.0000000Z == $D/f.sh" "2026-10-31T00:05:07.0000000Z == $D/g.bg.sh (started in the background)" >"$LG/leg3.log" +prog="$(grep -m1 "^ awk '" "$ROOT/CONTRIBUTING.md" | sed "s/^ awk '//; s/' leg.*//")" +[ -n "$prog" ] && [ "$(cd "$LG" && awk "$prog" leg1.log leg2.log leg3.log | LC_ALL=C sort)" \ + = "$(printf '%s\t%s\n' "$D/a.sh" 5 "$D/b.sh" 1 "$D/c.sh" 4 "$D/d.sh" 2 "$D/f.sh" 7)" ] +ok $? "CONTRIBUTING.md's table rebuild command: seconds between == lines, rounded, at least 1, midnight wrap, no row for a background wait or a file's last test" + +echo "Tests: $pass/$total" +[ "$pass" -eq "$total" ] diff --git a/scripts/test-perpr-shard.sh b/scripts/test-perpr-shard.sh index 1f8392c27..8f70b0894 100644 --- a/scripts/test-perpr-shard.sh +++ b/scripts/test-perpr-shard.sh @@ -15,7 +15,7 @@ pass=0 total=0 ok() { total=$((total + 1)); if [ "$1" -eq 0 ]; then pass=$((pass + 1)); echo "PASS $2"; else echo "FAIL $2"; fi; } slice() { env -u SHARD SHARD="$1" bash "${2:-$H}"; } # [helper]: the dry listing of one slice -# The real tree: 3 slices, disjoint, covering every registration (serial and .bg.sh), spread evenly. +# The real tree: 3 slices, disjoint, covering every registration (serial and .bg.sh); their time balance is in scripts/test-perpr-shard-balance.sh. all="$(cd "$ROOT" && printf '%s\n' scripts/ci-tests.d/*.sh | LC_ALL=C sort)" s1="$(slice 1/3)"; s2="$(slice 2/3)"; s3="$(slice 3/3)" joined="$(printf '%s\n%s\n%s\n' "$s1" "$s2" "$s3")" @@ -23,8 +23,7 @@ joined="$(printf '%s\n%s\n%s\n' "$s1" "$s2" "$s3")" [ "$(LC_ALL=C sort <<<"$joined")" = "$all" ]; ok $? "slices 1/3, 2/3 and 3/3 together cover every registration" [ "$(env -u SHARD bash "$H")" = "$all" ] && [ "$(slice 1/1)" = "$all" ]; ok $? "SHARD unset or 1/1 lists every registration" n1=$(grep -c . <<<"$s1"); n2=$(grep -c . <<<"$s2"); n3=$(grep -c . <<<"$s3") -[ "$n1" -gt 0 ] && [ $((n1 - n2)) -le 2 ] && [ $((n2 - n1)) -le 2 ] && [ $((n1 - n3)) -le 2 ] && [ $((n3 - n1)) -le 2 ] && [ $((n2 - n3)) -le 2 ] && [ $((n3 - n2)) -le 2 ] -ok $? "the 3 slices differ by at most 2 registrations ($n1/$n2/$n3)" +[ "$n1" -gt 0 ] && [ "$n2" -gt 0 ] && [ "$n3" -gt 0 ]; ok $? "the 3 slices are all non-empty ($n1/$n2/$n3 registrations)" [ "$s1" = "$(LC_ALL=C sort <<<"$s1")" ]; ok $? "a slice is listed sorted" # Fail closed: a bad SHARD is exit 2 with nothing on stdout, never an empty or full list a leg would run on. @@ -84,6 +83,8 @@ for i in 1 2 3; do [ "$(sed -n 's/^== \([^ ]*\).*/\1/p' "$WORK/leg$i.out" | LC_ALL=C sort)" = "$(slice $i/3 "$FX/scripts/ci-tests-shard.sh")" ] || good=1 done [ "$good" -eq 0 ]; ok $? "each leg runs exactly the registrations its dry listing names" +el=0; for i in 1 2 3; do [ "$(grep -cE '^bg elapsed: [a-z]\.bg\.sh [0-9]+ s$' "$WORK/leg$i.out")" -eq 1 ] || el=1; done +[ "$el" -eq 0 ]; ok $? "each leg's wait for its background test logs one 'bg elapsed: s' line (the end time is written before the .rc the wait polls)" body "Per-PR tests (scripts/ci-tests.d)" >"$WORK/solo.sh"; : >"$WORK/log" # the per-PR step without the start step before it: no list solo=0; (cd "$FX" && PATH="$WORK/bin:$PATH" PERPR_LOG="$WORK/log" RUNNER_TEMP="$(mktemp -d "$WORK/rt.XXXXXX")" bash -e "$WORK/solo.sh") >"$WORK/solo.out" 2>&1 || solo=1 [ "$solo" -eq 1 ] && [ ! -s "$WORK/log" ] && grep -q 'wrote no test list' "$WORK/solo.out"