diff --git a/.github/workflows/arch-package.yml b/.github/workflows/arch-package.yml index e7e20ad511..4d5552d2bb 100644 --- a/.github/workflows/arch-package.yml +++ b/.github/workflows/arch-package.yml @@ -1,6 +1,7 @@ name: Arch package on: + merge_group: pull_request: paths: - "scripts/build-linux-arch-package*" @@ -11,7 +12,34 @@ permissions: contents: read jobs: + # GitHub ignores paths filters in the merge queue, so the queue run applies + # this workflow's pull_request paths to the queued change itself. + scope: + name: Scope + if: github.event_name == 'merge_group' + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + run: ${{ steps.queue.outputs.run }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 2 + filter: blob:none + sparse-checkout: | + .github/workflows + scripts/ci + - name: Merge queue scope + id: queue + env: + BASE: ${{ github.event.merge_group.base_sha }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py scope --workflow .github/workflows/arch-package.yml --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" + test: + needs: scope + if: ${{ !cancelled() && needs.scope.outputs.run != 'false' }} name: Arch package versions runs-on: blacksmith-2vcpu-ubuntu-2404 container: archlinux:base-devel@sha256:68bfc3b0d277b08a99101dc9b94aaa03e5ae70cf1b4fb965c03b2b87b915760d diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92711e5ab8..1451444164 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ on: branches: - main pull_request: + merge_group: workflow_dispatch: permissions: @@ -35,6 +36,10 @@ jobs: uses: dorny/paths-filter@v3 id: filter with: + # In the merge queue, compare the queued commit with the commit it + # was built on, so only this pull request's changes pick the jobs. + base: ${{ github.event.merge_group.base_sha }} + ref: ${{ github.event.merge_group.head_sha }} filters: | rust: - '.cargo/**' diff --git a/.github/workflows/database-profile-reliability.yml b/.github/workflows/database-profile-reliability.yml index c867954894..ebcdd0b9ab 100644 --- a/.github/workflows/database-profile-reliability.yml +++ b/.github/workflows/database-profile-reliability.yml @@ -1,6 +1,7 @@ name: Database Profile Reliability on: + merge_group: workflow_dispatch: pull_request: paths: @@ -26,7 +27,34 @@ concurrency: cancel-in-progress: true jobs: + # GitHub ignores paths filters in the merge queue, so the queue run applies + # this workflow's pull_request paths to the queued change itself. + scope: + name: Scope + if: github.event_name == 'merge_group' + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + run: ${{ steps.queue.outputs.run }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 2 + filter: blob:none + sparse-checkout: | + .github/workflows + scripts/ci + - name: Merge queue scope + id: queue + env: + BASE: ${{ github.event.merge_group.base_sha }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py scope --workflow .github/workflows/database-profile-reliability.yml --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" + profile-contract: + needs: scope + if: ${{ !cancelled() && needs.scope.outputs.run != 'false' }} name: Profile, activation, and consent compatibility runs-on: ubuntu-24.04 timeout-minutes: 10 diff --git a/.github/workflows/docker-build-media-server.yml b/.github/workflows/docker-build-media-server.yml index b31f9cfff1..ba163f23c4 100644 --- a/.github/workflows/docker-build-media-server.yml +++ b/.github/workflows/docker-build-media-server.yml @@ -1,6 +1,7 @@ name: "Docker Build Media Server" on: + merge_group: workflow_dispatch: inputs: tag: @@ -32,7 +33,34 @@ concurrency: cancel-in-progress: true jobs: + # GitHub ignores paths filters in the merge queue, so the queue run applies + # this workflow's pull_request paths to the queued change itself. + scope: + name: Scope + if: github.event_name == 'merge_group' + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + run: ${{ steps.queue.outputs.run }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 2 + filter: blob:none + sparse-checkout: | + .github/workflows + scripts/ci + - name: Merge queue scope + id: queue + env: + BASE: ${{ github.event.merge_group.base_sha }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py scope --workflow .github/workflows/docker-build-media-server.yml --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" + audio-metrics: + needs: scope + if: ${{ !cancelled() && needs.scope.outputs.run != 'false' }} name: Audio intelligibility alignment runs-on: ubuntu-24.04 timeout-minutes: 5 @@ -51,6 +79,8 @@ jobs: "$RUNNER_TEMP/audio-metrics/bin/python" scripts/test-audio-intelligibility.py railway-dockerfile: + needs: scope + if: ${{ !cancelled() && needs.scope.outputs.run != 'false' }} # The Railway media server builds apps/media-server/Dockerfile, whose test # chain differs from Dockerfile.standalone. Building it here keeps that # chain from failing unnoticed. @@ -67,6 +97,8 @@ jobs: run: docker build --progress=plain -f apps/media-server/Dockerfile -t cap-media-server:railway-check . build: + needs: scope + if: ${{ !cancelled() && needs.scope.outputs.run != 'false' }} name: Build Docker Image (${{ matrix.platform }}) runs-on: ${{ matrix.runner }} timeout-minutes: 25 @@ -90,7 +122,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - if: github.event_name != 'pull_request' + if: (github.event_name != 'pull_request' && github.event_name != 'merge_group') uses: docker/login-action@v3 with: registry: ghcr.io @@ -108,16 +140,16 @@ jobs: context: ./apps/media-server file: ./apps/media-server/Dockerfile.standalone platforms: linux/${{ matrix.platform }} - push: ${{ github.event_name != 'pull_request' }} - load: ${{ github.event_name == 'pull_request' }} - tags: ${{ github.event_name == 'pull_request' && 'cap-media-server:verification' || '' }} - outputs: ${{ github.event_name != 'pull_request' && format('type=image,name=ghcr.io/{0}/cap-media-server,push-by-digest=true', env.REPOSITORY_OWNER) || '' }} + push: ${{ (github.event_name != 'pull_request' && github.event_name != 'merge_group') }} + load: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') }} + tags: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && 'cap-media-server:verification' || '' }} + outputs: ${{ (github.event_name != 'pull_request' && github.event_name != 'merge_group') && format('type=image,name=ghcr.io/{0}/cap-media-server,push-by-digest=true', env.REPOSITORY_OWNER) || '' }} cache-from: type=gha,scope=media-server-${{ matrix.platform }} cache-to: type=gha,mode=max,scope=media-server-${{ matrix.platform }} - name: Verify recording decode in the production image env: - MEDIA_IMAGE: ${{ github.event_name == 'pull_request' && 'cap-media-server:verification' || format('ghcr.io/{0}/cap-media-server@{1}', env.REPOSITORY_OWNER, steps.build.outputs.digest) }} + MEDIA_IMAGE: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && 'cap-media-server:verification' || format('ghcr.io/{0}/cap-media-server@{1}', env.REPOSITORY_OWNER, steps.build.outputs.digest) }} run: | docker run --rm --network none --entrypoint bun "$MEDIA_IMAGE" test \ src/__tests__/lib/audio-quality-policy.test.ts \ @@ -176,7 +208,7 @@ jobs: - name: Verify the server runs without root in the production image env: - MEDIA_IMAGE: ${{ github.event_name == 'pull_request' && 'cap-media-server:verification' || format('ghcr.io/{0}/cap-media-server@{1}', env.REPOSITORY_OWNER, steps.build.outputs.digest) }} + MEDIA_IMAGE: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && 'cap-media-server:verification' || format('ghcr.io/{0}/cap-media-server@{1}', env.REPOSITORY_OWNER, steps.build.outputs.digest) }} run: | test "$(docker run --rm --network none "$MEDIA_IMAGE" id -u)" = 1000 docker volume create media-gpu-cache @@ -191,7 +223,7 @@ jobs: - name: Verify long recording performance in the production image env: - MEDIA_IMAGE: ${{ github.event_name == 'pull_request' && 'cap-media-server:verification' || format('ghcr.io/{0}/cap-media-server@{1}', env.REPOSITORY_OWNER, steps.build.outputs.digest) }} + MEDIA_IMAGE: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && 'cap-media-server:verification' || format('ghcr.io/{0}/cap-media-server@{1}', env.REPOSITORY_OWNER, steps.build.outputs.digest) }} run: | docker run --rm --network none --cpus 2 --memory 2g \ -e MEDIA_SERVER_RECORDING_PERFORMANCE_TESTS=1 \ @@ -205,14 +237,14 @@ jobs: src/__tests__/lib/drive-resumable-upload.test.ts - name: Export Digest - if: github.event_name != 'pull_request' + if: (github.event_name != 'pull_request' && github.event_name != 'merge_group') run: | mkdir -p /tmp/digests digest="${{ steps.build.outputs.digest }}" touch "/tmp/digests/${digest#sha256:}" - name: Upload Digest - if: github.event_name != 'pull_request' + if: (github.event_name != 'pull_request' && github.event_name != 'merge_group') uses: actions/upload-artifact@v4 with: name: digests-media-server-${{ matrix.platform }} @@ -223,7 +255,7 @@ jobs: merge: name: Create Multi-Architecture Manifest needs: build - if: github.event_name != 'pull_request' + if: (github.event_name != 'pull_request' && github.event_name != 'merge_group') runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/glass-probe.yml b/.github/workflows/glass-probe.yml index 57f960e09b..01733fa1a3 100644 --- a/.github/workflows/glass-probe.yml +++ b/.github/workflows/glass-probe.yml @@ -5,6 +5,7 @@ name: Glass probe (real app) # fitted to Apple's own glass. on: + merge_group: pull_request: types: [opened, synchronize, reopened, ready_for_review] paths: @@ -33,10 +34,27 @@ jobs: runs-on: blacksmith-2vcpu-ubuntu-2404 timeout-minutes: 5 outputs: - run: ${{ steps.scope.outputs.run }} + run: ${{ steps.queue.outputs.run || steps.scope.outputs.run }} steps: + - uses: actions/checkout@v4 + if: github.event_name == 'merge_group' + with: + persist-credentials: false + fetch-depth: 2 + filter: blob:none + sparse-checkout: | + .github/workflows + scripts/ci + - name: Merge queue scope + id: queue + if: github.event_name == 'merge_group' + env: + BASE: ${{ github.event.merge_group.base_sha }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py scope --workflow .github/workflows/glass-probe.yml --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" - name: Scope id: scope + if: github.event_name != 'merge_group' env: GH_TOKEN: ${{ github.token }} EVENT: ${{ github.event_name }} diff --git a/.github/workflows/gpui-app-bench.yml b/.github/workflows/gpui-app-bench.yml index 8e414de31c..7da1fc6a30 100644 --- a/.github/workflows/gpui-app-bench.yml +++ b/.github/workflows/gpui-app-bench.yml @@ -1,6 +1,7 @@ name: GPUI app benchmark on: + merge_group: pull_request: paths: - "apps/desktop-gpui/patches/**" @@ -26,11 +27,37 @@ permissions: contents: read jobs: + # GitHub ignores paths filters in the merge queue, so the queue run applies + # this workflow's pull_request paths to the queued change itself. + scope: + name: Scope + if: github.event_name == 'merge_group' + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + run: ${{ steps.queue.outputs.run }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 2 + filter: blob:none + sparse-checkout: | + .github/workflows + scripts/ci + - name: Merge queue scope + id: queue + env: + BASE: ${{ github.event.merge_group.base_sha }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py scope --workflow .github/workflows/gpui-app-bench.yml --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" + editor: + needs: scope + if: ${{ !cancelled() && needs.scope.outputs.run != 'false' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} name: Editor draw benchmark (macOS) # Benchmarks build GPUI twice and run for up to two hours, so pull requests # from forks do not start them; maintainers can run them with workflow_dispatch. - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: blacksmith-6vcpu-macos-15 timeout-minutes: 150 env: @@ -72,7 +99,7 @@ jobs: shell: bash env: BASE_INPUT: ${{ inputs.base }} - PR_BASE: ${{ github.event.pull_request.base.sha }} + PR_BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} run: | set -euo pipefail node scripts/sync-desktop-versions.mjs diff --git a/.github/workflows/gpui-bench.yml b/.github/workflows/gpui-bench.yml index 024f986805..c6960eb88e 100644 --- a/.github/workflows/gpui-bench.yml +++ b/.github/workflows/gpui-bench.yml @@ -1,6 +1,7 @@ name: GPUI frame benchmarks on: + merge_group: pull_request: paths: - "apps/desktop-gpui/patches/**" @@ -29,11 +30,37 @@ permissions: contents: read jobs: + # GitHub ignores paths filters in the merge queue, so the queue run applies + # this workflow's pull_request paths to the queued change itself. + scope: + name: Scope + if: github.event_name == 'merge_group' + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + run: ${{ steps.queue.outputs.run }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 2 + filter: blob:none + sparse-checkout: | + .github/workflows + scripts/ci + - name: Merge queue scope + id: queue + env: + BASE: ${{ github.event.merge_group.base_sha }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py scope --workflow .github/workflows/gpui-bench.yml --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" + bench: + needs: scope + if: ${{ !cancelled() && needs.scope.outputs.run != 'false' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} name: GPUI benchmarks (${{ matrix.settings.name }}) # Benchmarks build GPUI twice and run for up to two hours, so pull requests # from forks do not start them; maintainers can run them with workflow_dispatch. - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository timeout-minutes: 150 strategy: fail-fast: false @@ -84,7 +111,7 @@ jobs: shell: bash env: BASE_INPUT: ${{ inputs.base }} - PR_BASE: ${{ github.event.pull_request.base.sha }} + PR_BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} run: | set -euo pipefail ./scripts/prepare-gpui-dependency.sh @@ -170,9 +197,10 @@ jobs: if-no-files-found: ignore test: + needs: scope + if: ${{ !cancelled() && needs.scope.outputs.run != 'false' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} name: GPUI tests (${{ matrix.settings.name }}) # Same as the benchmarks: GPUI's test builds are long, so forks do not start them. - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository timeout-minutes: 120 strategy: fail-fast: false diff --git a/.github/workflows/infra-database.yml b/.github/workflows/infra-database.yml index 40b0af81dd..cd33320e02 100644 --- a/.github/workflows/infra-database.yml +++ b/.github/workflows/infra-database.yml @@ -1,6 +1,7 @@ name: Infra Database on: + merge_group: workflow_dispatch: pull_request: paths: @@ -60,7 +61,34 @@ concurrency: cancel-in-progress: true jobs: + # GitHub ignores paths filters in the merge queue, so the queue run applies + # this workflow's pull_request paths to the queued change itself. + scope: + name: Scope + if: github.event_name == 'merge_group' + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + run: ${{ steps.queue.outputs.run }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 2 + filter: blob:none + sparse-checkout: | + .github/workflows + scripts/ci + - name: Merge queue scope + id: queue + env: + BASE: ${{ github.event.merge_group.base_sha }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py scope --workflow .github/workflows/infra-database.yml --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" + infra-database: + needs: scope + if: ${{ !cancelled() && needs.scope.outputs.run != 'false' }} name: Infra billing and uploads against MySQL runs-on: ubuntu-24.04 timeout-minutes: 15 diff --git a/.github/workflows/instant-reliability.yml b/.github/workflows/instant-reliability.yml index 23922766cb..2804d9d37a 100644 --- a/.github/workflows/instant-reliability.yml +++ b/.github/workflows/instant-reliability.yml @@ -10,6 +10,7 @@ name: Instant reliability (real app) # Instant with a virtual camera drawn into the video. on: + merge_group: pull_request: types: [opened, synchronize, reopened, ready_for_review] paths: @@ -58,10 +59,27 @@ jobs: runs-on: blacksmith-2vcpu-ubuntu-2404 timeout-minutes: 5 outputs: - run: ${{ steps.scope.outputs.run }} + run: ${{ steps.queue.outputs.run || steps.scope.outputs.run }} steps: + - uses: actions/checkout@v4 + if: github.event_name == 'merge_group' + with: + persist-credentials: false + fetch-depth: 2 + filter: blob:none + sparse-checkout: | + .github/workflows + scripts/ci + - name: Merge queue scope + id: queue + if: github.event_name == 'merge_group' + env: + BASE: ${{ github.event.merge_group.base_sha }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py scope --workflow .github/workflows/instant-reliability.yml --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" - name: Scope id: scope + if: github.event_name != 'merge_group' env: GH_TOKEN: ${{ github.token }} EVENT: ${{ github.event_name }} diff --git a/.github/workflows/linux-desktop-reliability.yml b/.github/workflows/linux-desktop-reliability.yml index 6c3d3958c7..06bf4a39ad 100644 --- a/.github/workflows/linux-desktop-reliability.yml +++ b/.github/workflows/linux-desktop-reliability.yml @@ -7,6 +7,7 @@ name: Linux desktop reliability # the tray, approve the screen share), stops it, and checks it finished. on: + merge_group: pull_request: paths: - "apps/desktop-gpui/**" @@ -47,10 +48,27 @@ jobs: runs-on: blacksmith-2vcpu-ubuntu-2404 timeout-minutes: 5 outputs: - run: ${{ steps.scope.outputs.run }} + run: ${{ steps.queue.outputs.run || steps.scope.outputs.run }} steps: + - uses: actions/checkout@v4 + if: github.event_name == 'merge_group' + with: + persist-credentials: false + fetch-depth: 2 + filter: blob:none + sparse-checkout: | + .github/workflows + scripts/ci + - name: Merge queue scope + id: queue + if: github.event_name == 'merge_group' + env: + BASE: ${{ github.event.merge_group.base_sha }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py scope --workflow .github/workflows/linux-desktop-reliability.yml --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" - name: Scope id: scope + if: github.event_name != 'merge_group' env: GH_TOKEN: ${{ github.token }} EVENT: ${{ github.event_name }} diff --git a/.github/workflows/merge-queue.yml b/.github/workflows/merge-queue.yml new file mode 100644 index 0000000000..0de59337c6 --- /dev/null +++ b/.github/workflows/merge-queue.yml @@ -0,0 +1,46 @@ +name: Merge queue + +# "Merge queue" is the check to require. In the merge queue it waits for every +# workflow that declares merge_group (scripts/ci/merge_queue.py expected) to +# finish on the queued commit and fails as soon as one fails, so a workflow +# added to the queue is a merge gate without another ruleset change. On pull +# requests it only checks the queue helpers: the queue run is the one that +# decides. + +on: + pull_request: + merge_group: + +permissions: + actions: read + checks: read + contents: read + +concurrency: + group: merge-queue-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + gate: + name: Merge queue + # GitHub-hosted, because this job only waits on the API for up to the + # length of the slowest queue workflow. + runs-on: ubuntu-latest + timeout-minutes: 170 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + sparse-checkout: | + .github/workflows + scripts/ci + + - name: Check the queue helpers + run: python3 scripts/ci/merge_queue.py test + + - name: Wait for the queue workflows + if: github.event_name == 'merge_group' + env: + GITHUB_TOKEN: ${{ github.token }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py gate --head "$HEAD" diff --git a/.github/workflows/perf-budget.yml b/.github/workflows/perf-budget.yml index 40c05e684a..78c063c80e 100644 --- a/.github/workflows/perf-budget.yml +++ b/.github/workflows/perf-budget.yml @@ -5,6 +5,7 @@ name: Perf budget # drifts between jobs. "Perf budget" is the job to require. on: + merge_group: pull_request: types: [opened, synchronize, reopened, ready_for_review] workflow_dispatch: @@ -38,7 +39,9 @@ jobs: EVENT: ${{ github.event_name }} PR: ${{ github.event.pull_request.number }} DRAFT: ${{ github.event.pull_request.draft }} - SAME_REPO: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + SAME_REPO: ${{ github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'merge_group' }} + MQ_BASE: ${{ github.event.merge_group.base_sha }} + MQ_HEAD: ${{ github.event.merge_group.head_sha }} RUSTUP_TOOLCHAIN: stable run: | set -euo pipefail @@ -49,7 +52,11 @@ jobs: echo "Draft: the perf budget runs once the pull request is ready." | tee -a "$GITHUB_STEP_SUMMARY" echo "run=false" >> "$GITHUB_OUTPUT"; exit 0 fi - gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$RUNNER_TEMP/changed.txt" + if [[ "$EVENT" == merge_group ]]; then + python3 scripts/ci/merge_queue.py files --base "$MQ_BASE" --head "$MQ_HEAD" > "$RUNNER_TEMP/changed.txt" + else + gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$RUNNER_TEMP/changed.txt" + fi python3 scripts/verify/plan.py risk --files "$RUNNER_TEMP/changed.txt" --output "$RUNNER_TEMP/risk.txt" # A change to the budget itself is measured only from this # repository's branches, so a fork cannot queue the macOS job by @@ -105,7 +112,7 @@ jobs: - name: Base worktree env: BASE_INPUT: ${{ inputs.base }} - PR_BASE: ${{ github.event.pull_request.base.sha }} + PR_BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} run: | set -euo pipefail base="${BASE_INPUT:-}" diff --git a/.github/workflows/picker-smoke.yml b/.github/workflows/picker-smoke.yml index 8f4462fd19..4344352e1e 100644 --- a/.github/workflows/picker-smoke.yml +++ b/.github/workflows/picker-smoke.yml @@ -1,6 +1,7 @@ name: Picker smoke (real app) on: + merge_group: pull_request: types: [opened, synchronize, reopened, ready_for_review] paths: @@ -37,10 +38,27 @@ jobs: runs-on: blacksmith-2vcpu-ubuntu-2404 timeout-minutes: 5 outputs: - run: ${{ steps.scope.outputs.run }} + run: ${{ steps.queue.outputs.run || steps.scope.outputs.run }} steps: + - uses: actions/checkout@v4 + if: github.event_name == 'merge_group' + with: + persist-credentials: false + fetch-depth: 2 + filter: blob:none + sparse-checkout: | + .github/workflows + scripts/ci + - name: Merge queue scope + id: queue + if: github.event_name == 'merge_group' + env: + BASE: ${{ github.event.merge_group.base_sha }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py scope --workflow .github/workflows/picker-smoke.yml --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" - name: Scope id: scope + if: github.event_name != 'merge_group' env: GH_TOKEN: ${{ github.token }} EVENT: ${{ github.event_name }} diff --git a/.github/workflows/recording-reliability.yml b/.github/workflows/recording-reliability.yml index d7be8f0c13..bda5a21685 100644 --- a/.github/workflows/recording-reliability.yml +++ b/.github/workflows/recording-reliability.yml @@ -1,6 +1,7 @@ name: Recording Reliability on: + merge_group: workflow_dispatch: pull_request: paths: @@ -30,7 +31,34 @@ concurrency: cancel-in-progress: true jobs: + # GitHub ignores paths filters in the merge queue, so the queue run applies + # this workflow's pull_request paths to the queued change itself. + scope: + name: Scope + if: github.event_name == 'merge_group' + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + run: ${{ steps.queue.outputs.run }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 2 + filter: blob:none + sparse-checkout: | + .github/workflows + scripts/ci + - name: Merge queue scope + id: queue + env: + BASE: ${{ github.event.merge_group.base_sha }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py scope --workflow .github/workflows/recording-reliability.yml --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" + recording-contract: + needs: scope + if: ${{ !cancelled() && needs.scope.outputs.run != 'false' }} name: Recording ownership, preservation, and recovery runs-on: ubuntu-24.04 timeout-minutes: 15 diff --git a/.github/workflows/status.yml b/.github/workflows/status.yml index 0877f46d83..eeadf53dad 100644 --- a/.github/workflows/status.yml +++ b/.github/workflows/status.yml @@ -1,6 +1,7 @@ name: Status app on: + merge_group: pull_request: paths: - "apps/status/**" @@ -17,7 +18,34 @@ permissions: contents: read jobs: + # GitHub ignores paths filters in the merge queue, so the queue run applies + # this workflow's pull_request paths to the queued change itself. + scope: + name: Scope + if: github.event_name == 'merge_group' + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + run: ${{ steps.queue.outputs.run }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 2 + filter: blob:none + sparse-checkout: | + .github/workflows + scripts/ci + - name: Merge queue scope + id: queue + env: + BASE: ${{ github.event.merge_group.base_sha }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py scope --workflow .github/workflows/status.yml --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" + verify: + needs: scope + if: ${{ !cancelled() && needs.scope.outputs.run != 'false' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/sync-tests.yml b/.github/workflows/sync-tests.yml index c2176eca72..b299ac6548 100644 --- a/.github/workflows/sync-tests.yml +++ b/.github/workflows/sync-tests.yml @@ -12,6 +12,7 @@ name: A/V Sync Tests # Findings are published as a job-summary table and a JSON artifact per OS. on: + merge_group: workflow_dispatch: schedule: - cron: "0 5 * * *" @@ -45,6 +46,31 @@ permissions: contents: read jobs: + # GitHub ignores paths filters in the merge queue, so the queue run applies + # this workflow's pull_request paths to the queued change itself. + scope: + name: Scope + if: github.event_name == 'merge_group' + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + run: ${{ steps.queue.outputs.run }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 2 + filter: blob:none + sparse-checkout: | + .github/workflows + scripts/ci + - name: Merge queue scope + id: queue + env: + BASE: ${{ github.event.merge_group.base_sha }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py scope --workflow .github/workflows/sync-tests.yml --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" + # Each runner's steps are split into parts that run side by side, so every # step keeps its environment. One job per runner took about 25 minutes, most # of it these steps one after another: the timestamp tests (5 to 12 min), the @@ -56,6 +82,8 @@ jobs: # 44154886695). Blacksmith Macs have no hardware video encoder, so the # encoder crates also run on one hosted Mac, which has VideoToolbox's. sync-tests: + needs: scope + if: ${{ !cancelled() && needs.scope.outputs.run != 'false' }} name: sync-tests (${{ matrix.runner }}, ${{ matrix.parts }}) strategy: fail-fast: false diff --git a/.github/workflows/test-self-hosting.yml b/.github/workflows/test-self-hosting.yml index 3f5255fda9..8e0619813b 100644 --- a/.github/workflows/test-self-hosting.yml +++ b/.github/workflows/test-self-hosting.yml @@ -1,6 +1,7 @@ name: "Test Self-Hosting" on: + merge_group: workflow_dispatch: push: branches: @@ -20,7 +21,34 @@ on: - ".github/workflows/test-self-hosting.yml" jobs: + # GitHub ignores paths filters in the merge queue, so the queue run applies + # this workflow's pull_request paths to the queued change itself. + scope: + name: Scope + if: github.event_name == 'merge_group' + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + run: ${{ steps.queue.outputs.run }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 2 + filter: blob:none + sparse-checkout: | + .github/workflows + scripts/ci + - name: Merge queue scope + id: queue + env: + BASE: ${{ github.event.merge_group.base_sha }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py scope --workflow .github/workflows/test-self-hosting.yml --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" + test-docker-compose: + needs: scope + if: ${{ !cancelled() && needs.scope.outputs.run != 'false' }} name: Test Docker Compose Self-Hosting runs-on: ubuntu-latest timeout-minutes: 15 diff --git a/.github/workflows/validate-migration-journal.yml b/.github/workflows/validate-migration-journal.yml index 300da0a8c1..098d83ecb2 100644 --- a/.github/workflows/validate-migration-journal.yml +++ b/.github/workflows/validate-migration-journal.yml @@ -1,12 +1,40 @@ name: Validate Migrations on: + merge_group: pull_request: paths: - "packages/database/migrations/**" jobs: + # GitHub ignores paths filters in the merge queue, so the queue run applies + # this workflow's pull_request paths to the queued change itself. + scope: + name: Scope + if: github.event_name == 'merge_group' + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + run: ${{ steps.queue.outputs.run }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 2 + filter: blob:none + sparse-checkout: | + .github/workflows + scripts/ci + - name: Merge queue scope + id: queue + env: + BASE: ${{ github.event.merge_group.base_sha }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py scope --workflow .github/workflows/validate-migration-journal.yml --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" + validate-journal: + needs: scope + if: ${{ !cancelled() && needs.scope.outputs.run != 'false' }} runs-on: ubuntu-latest steps: - name: Checkout PR branch diff --git a/.github/workflows/verify-recording.yml b/.github/workflows/verify-recording.yml index 3ee34a96b9..a6b878242f 100644 --- a/.github/workflows/verify-recording.yml +++ b/.github/workflows/verify-recording.yml @@ -59,6 +59,7 @@ on: - cron: "41 3 * * *" pull_request: types: [opened, synchronize, reopened, ready_for_review] + merge_group: push: branches: - ci/verify-ci @@ -95,6 +96,8 @@ jobs: PR: ${{ github.event.pull_request.number }} DRAFT: ${{ github.event.pull_request.draft }} BEFORE: ${{ github.event.before }} + MQ_BASE: ${{ github.event.merge_group.base_sha }} + MQ_HEAD: ${{ github.event.merge_group.head_sha }} INPUT_TIER: ${{ inputs.tier }} INPUT_RUNNERS: ${{ inputs.runners }} INPUT_CASE: ${{ inputs.case_filter }} @@ -120,6 +123,9 @@ jobs: if [[ "$EVENT" == pull_request ]]; then gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$files" max=3000 + elif [[ "$EVENT" == merge_group ]]; then + python3 scripts/ci/merge_queue.py files --base "$MQ_BASE" --head "$MQ_HEAD" > "$files" + max=3000 elif [[ "$BEFORE" =~ ^0+$ ]]; then : > "$files"; echo Cargo.toml >> "$files" max=300 @@ -420,13 +426,13 @@ jobs: # Without the manifest the next restore cannot tell which sources the # cached build came from (scripts/verify/mtime.py). - name: Record sources of the cached build - if: github.event_name != 'pull_request' && matrix.shard == 1 && steps.setup.outputs.build-cache-hit != 'true' + if: github.event_name != 'pull_request' && github.event_name != 'merge_group' && matrix.shard == 1 && steps.setup.outputs.build-cache-hit != 'true' shell: bash run: | "$(command -v python3 || command -v python)" scripts/verify/mtime.py record target/ci-verify/verify-src-manifest.txt - name: Save build cache - if: github.event_name != 'pull_request' && matrix.shard == 1 && steps.setup.outputs.build-cache-hit != 'true' + if: github.event_name != 'pull_request' && github.event_name != 'merge_group' && matrix.shard == 1 && steps.setup.outputs.build-cache-hit != 'true' uses: actions/cache/save@v4 with: path: | diff --git a/.github/workflows/windows-recording-encoder.yml b/.github/workflows/windows-recording-encoder.yml index e12e993a5f..a07386c8ae 100644 --- a/.github/workflows/windows-recording-encoder.yml +++ b/.github/workflows/windows-recording-encoder.yml @@ -5,6 +5,7 @@ name: Windows recording encoder # Media Foundation self-test that runs in a child process because Intel's # transform faults on odd frame sizes. on: + merge_group: pull_request: paths: - "crates/enc-ffmpeg/**" @@ -34,7 +35,34 @@ concurrency: cancel-in-progress: true jobs: + # GitHub ignores paths filters in the merge queue, so the queue run applies + # this workflow's pull_request paths to the queued change itself. + scope: + name: Scope + if: github.event_name == 'merge_group' + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + run: ${{ steps.queue.outputs.run }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 2 + filter: blob:none + sparse-checkout: | + .github/workflows + scripts/ci + - name: Merge queue scope + id: queue + env: + BASE: ${{ github.event.merge_group.base_sha }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py scope --workflow .github/workflows/windows-recording-encoder.yml --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" + segmented-encoder: + needs: scope + if: ${{ !cancelled() && needs.scope.outputs.run != 'false' }} name: Segmented encoder (windows-2022) runs-on: windows-2022 timeout-minutes: 45 diff --git a/.github/workflows/windows-targets.yml b/.github/workflows/windows-targets.yml index ca235f3a56..0b42d189a6 100644 --- a/.github/workflows/windows-targets.yml +++ b/.github/workflows/windows-targets.yml @@ -1,6 +1,7 @@ name: Windows Target Discovery on: + merge_group: pull_request: paths: - "crates/scap-targets/**" @@ -20,7 +21,34 @@ concurrency: cancel-in-progress: true jobs: + # GitHub ignores paths filters in the merge queue, so the queue run applies + # this workflow's pull_request paths to the queued change itself. + scope: + name: Scope + if: github.event_name == 'merge_group' + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + run: ${{ steps.queue.outputs.run }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 2 + filter: blob:none + sparse-checkout: | + .github/workflows + scripts/ci + - name: Merge queue scope + id: queue + env: + BASE: ${{ github.event.merge_group.base_sha }} + HEAD: ${{ github.event.merge_group.head_sha }} + run: python3 scripts/ci/merge_queue.py scope --workflow .github/workflows/windows-targets.yml --base "$BASE" --head "$HEAD" >> "$GITHUB_OUTPUT" + windows-targets: + needs: scope + if: ${{ !cancelled() && needs.scope.outputs.run != 'false' }} name: Native Windows window discovery runs-on: windows-2022 timeout-minutes: 20 diff --git a/scripts/ci/merge_queue.py b/scripts/ci/merge_queue.py new file mode 100644 index 0000000000..0e8a631a35 --- /dev/null +++ b/scripts/ci/merge_queue.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Merge queue helpers for the GitHub Actions workflows. + +GitHub ignores `paths` filters for merge_group events, so a workflow that +runs in the merge queue decides its own scope with `scope`, from the same +`pull_request` paths filter it declares, applied to the queue commit's diff. +The "Merge queue" check (merge-queue.yml) runs `gate`, which waits for every +workflow that declares merge_group to finish on the queue commit. + +Commands: + files --base SHA --head SHA changed paths, one per line + scope --workflow FILE --base --head run=true|false for GITHUB_OUTPUT + expected workflow files that run in the queue + gate --head SHA wait for those workflows, fail on red + test self-test +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOWS = ROOT / ".github" / "workflows" +GATE_WORKFLOW = ".github/workflows/merge-queue.yml" +OK_CONCLUSIONS = {"success", "skipped"} + + +def glob_regex(pattern): + """GitHub's filter pattern syntax: `*` stops at `/`, `**` does not, `**/` + also matches no directory, `?` and `+` repeat the preceding character.""" + out = [] + i = 0 + while i < len(pattern): + c = pattern[i] + if c == "*": + if pattern.startswith("**", i): + i += 2 + if pattern.startswith("/", i): + out.append("(?:.*/)?") + i += 1 + else: + out.append(".*") + continue + out.append("[^/]*") + elif c in "?+" and out: + out.append(c) + elif c == "[": + end = pattern.find("]", i + 1) + if end == -1: + out.append(re.escape(c)) + else: + out.append(pattern[i : end + 1]) + i = end + else: + out.append(re.escape(c)) + i += 1 + return re.compile("".join(out) + r"\Z") + + +def matches(path, patterns): + hit = False + for pattern in patterns: + negate = pattern.startswith("!") + if glob_regex(pattern[1:] if negate else pattern).match(path): + hit = not negate + return hit + + +def _scalar(text): + text = text.split(" #", 1)[0].strip() + if len(text) >= 2 and text[0] == text[-1] and text[0] in "\"'": + return text[1:-1] + return text + + +def _inline_list(text): + text = text.strip() + if text.startswith("[") and text.endswith("]"): + return [_scalar(item) for item in text[1:-1].split(",") if item.strip()] + return None + + +def triggers(workflow): + """The workflow's `on:` events and their list-valued filters, read without + a YAML library (runner images do not all ship one). Every workflow in the + repository is checked against PyYAML by `test`.""" + lines = Path(workflow).read_text().splitlines() + events = {} + for index, line in enumerate(lines): + if not re.match(r"""^["']?on["']?:""", line): + continue + rest = line.split(":", 1)[1].strip() + if rest and not rest.startswith("#"): + inline = _inline_list(rest) + for name in inline if inline is not None else [_scalar(rest)]: + events[name] = {} + return events + event_indent = key_indent = None + event = key = None + for raw in lines[index + 1 :]: + if not raw.strip() or raw.lstrip().startswith("#"): + continue + indent = len(raw) - len(raw.lstrip()) + if indent == 0: + break + text = raw.strip() + if event_indent is None: + event_indent = indent + if indent == event_indent: + event = _scalar(text.split(":", 1)[0]) + events[event] = {} + key = key_indent = None + continue + if key_indent is None: + key_indent = indent + if indent == key_indent and not text.startswith("- "): + name, value = text.split(":", 1) + inline = _inline_list(value) + key = _scalar(name) if inline is not None or not value.strip() else None + if key is not None: + events[event][key] = inline if inline is not None else [] + if inline is not None: + key = None + elif indent > key_indent and key is not None: + if text.startswith("- "): + events[event][key].append(_scalar(text[2:])) + else: + del events[event][key] + key = None + return events + raise SystemExit(f"{workflow}: no on: block") + + +def run_git(*args, cwd=ROOT): + return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout + + +def changed_files(base, head, cwd=ROOT): + partial = subprocess.run(["git", "config", "remote.origin.promisor"], cwd=cwd, capture_output=True, text=True) + fetch = ["fetch", "--no-tags", "--depth=1"] + (["--filter=blob:none"] if partial.stdout.strip() == "true" else []) + for sha in (base, head): + if subprocess.run(["git", "cat-file", "-e", f"{sha}^{{commit}}"], cwd=cwd, capture_output=True).returncode: + run_git(*fetch, "origin", sha, cwd=cwd) + # -z: without it git quotes non-ASCII paths, which then match no filter. + return [path for path in run_git("diff", "--no-renames", "--name-only", "-z", base, head, cwd=cwd).split("\0") if path] + + +def cmd_files(args): + for path in changed_files(args.base, args.head): + print(path) + + +def cmd_scope(args): + workflow = ROOT / args.workflow + pull_request = triggers(workflow).get("pull_request", {}) + paths = pull_request.get("paths") + ignore = pull_request.get("paths-ignore") + summary = os.environ.get("GITHUB_STEP_SUMMARY") + try: + files = changed_files(args.base, args.head) + except subprocess.CalledProcessError as error: + print(f"::warning::could not diff {args.base}..{args.head} ({error.stderr.strip()}); running in full", file=sys.stderr) + print("run=true") + return + if paths: + hits = [path for path in files if matches(path, paths)] + elif ignore: + hits = [path for path in files if not matches(path, ignore)] + else: + hits = files or ["(no filter)"] + run = bool(hits) + note = ( + f"Merge queue: {len(files)} changed files, {len(hits)} in this workflow's pull_request paths; " + + ("running." if run else "nothing to run.") + ) + print(note, file=sys.stderr) + if summary: + with open(summary, "a") as handle: + handle.write(note + "\n") + for path in hits[:20]: + handle.write(f"- `{path}`\n") + print(f"run={'true' if run else 'false'}") + + +def queue_workflows(): + out = [] + for workflow in sorted([*WORKFLOWS.glob("*.yml"), *WORKFLOWS.glob("*.yaml")]): + rel = workflow.relative_to(ROOT).as_posix() + if rel != GATE_WORKFLOW and "merge_group" in triggers(workflow): + out.append(rel) + return out + + +def cmd_expected(_args): + for path in queue_workflows(): + print(path) + + +class Api: + """GitHub REST reads with ETags: a 304 does not count against the token's + hourly limit, and most polls of a long queue run change nothing.""" + + def __init__(self, token): + self.token = token + self.cache = {} + + def get(self, url): + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self.token}", + "X-GitHub-Api-Version": "2022-11-28", + } + if url in self.cache: + headers["If-None-Match"] = self.cache[url][0] + try: + with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=30) as response: + body = (json.load(response), response.headers.get("Link", "")) + if response.headers.get("ETag"): + self.cache[url] = (response.headers["ETag"], body) + return body + except urllib.error.HTTPError as error: + if error.code == 304 and url in self.cache: + return self.cache[url][1] + raise + + def pages(self, url, key): + items = [] + while url: + body, link = self.get(url) + items.extend(body.get(key, [])) + url = next((part.split(";")[0].strip(" <>") for part in link.split(",") if 'rel="next"' in part), None) + return items + + +FAILED_JOBS = {"failure", "cancelled", "timed_out", "action_required", "startup_failure"} + + +def gate_verdict(expected, runs, checks=()): + """(state, lines): state is pass, fail or wait. A failed job fails the gate + at once, before the rest of its workflow finishes.""" + latest = {} + for run in runs: + path = run.get("path", "").split("@", 1)[0] + if path in expected and (path not in latest or run["id"] > latest[path]["id"]): + latest[path] = run + suites = {run.get("check_suite_id"): path for path, run in latest.items()} + failed_jobs = {} + for check in checks: + path = suites.get((check.get("check_suite") or {}).get("id")) + if path and check.get("conclusion") in FAILED_JOBS: + failed_jobs.setdefault(path, []).append(f"{check['name']} {check['conclusion']}") + lines, failed, waiting = [], [], [] + for path in expected: + run = latest.get(path) + if run is None: + waiting.append(path) + lines.append(f"{path}: not started") + elif path in failed_jobs: + failed.append(path) + lines.append(f"{path}: {', '.join(failed_jobs[path])} {run['html_url']}") + elif run["status"] != "completed": + waiting.append(path) + lines.append(f"{path}: {run['status']} {run['html_url']}") + else: + lines.append(f"{path}: {run['conclusion']} {run['html_url']}") + if run["conclusion"] not in OK_CONCLUSIONS: + failed.append(path) + if failed: + return "fail", lines + return ("wait" if waiting else "pass"), lines + + +def cmd_gate(args): + api = Api(os.environ["GITHUB_TOKEN"]) + repo = os.environ["GITHUB_REPOSITORY"] + api_url = os.environ.get("GITHUB_API_URL", "https://api.github.com") + expected = queue_workflows() + print("Waiting for:\n " + "\n ".join(expected), flush=True) + runs_url = f"{api_url}/repos/{repo}/actions/runs?head_sha={args.head}&event=merge_group&per_page=100" + checks_url = f"{api_url}/repos/{repo}/commits/{args.head}/check-runs?filter=latest&per_page=100" + started = time.monotonic() + delay = args.first_poll + lines = [] + while True: + time.sleep(delay) + delay = args.poll + try: + runs = api.pages(runs_url, "workflow_runs") + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error: + print(f"::warning::listing runs failed ({error}); retrying", flush=True) + continue + try: + checks = api.pages(checks_url, "check_runs") + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error: + # Without job results the gate still fails, once the failing workflow finishes. + print(f"::warning::reading check runs failed ({error}); judging by workflow conclusions", flush=True) + checks = [] + state, lines = gate_verdict(expected, runs, checks) + elapsed = time.monotonic() - started + pending = [line for line in lines if not line.split(": ", 1)[1].startswith(("success", "skipped"))] + print(f"[{int(elapsed // 60)}m] {state}: " + ("; ".join(pending) or "all green"), flush=True) + if state == "pass": + break + if state == "fail": + write_gate_summary("failed", lines) + print("::error::a workflow failed on the merge queue commit (see the list above)") + return 1 + missing = [line for line in lines if line.endswith(": not started")] + if missing and elapsed > args.start_grace * 60: + write_gate_summary("failed", lines) + print(f"::error::{len(missing)} queue workflows never started on {args.head}") + return 1 + write_gate_summary("passed", lines) + print("every merge queue workflow passed") + return 0 + + +def write_gate_summary(verdict, lines): + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a") as handle: + handle.write(f"## Merge queue {verdict}\n\n") + for line in lines: + handle.write(f"- {line}\n") + + +def cmd_test(_args): + cases = [ + ("apps/desktop-gpui/**", "apps/desktop-gpui/src/main.rs", True), + ("apps/desktop-gpui/**", "apps/desktop-gpui", False), + ("apps/web/lib/infra-**", "apps/web/lib/infra-billing/x.ts", True), + ("apps/web/lib/infra-**", "apps/web/lib/infra.ts", False), + ("scripts/gpui-bench-*", "scripts/gpui-bench-run.py", True), + ("scripts/gpui-bench-*", "scripts/gpui-bench-x/y.py", False), + ("packages/web-*/**", "packages/web-backend/src/a.ts", True), + ("packages/web-*/**", "packages/webx/a.ts", False), + ("apps/web/app/(infra)/**", "apps/web/app/(infra)/page.tsx", True), + ("apps/web/app/(infra)/**", "apps/web/app/infra/page.tsx", False), + ("apps/web/lib/space-video-order*", "apps/web/lib/space-video-order.ts", True), + ("**/README.md", "README.md", True), + ("**/README.md", "docs/a/README.md", True), + ("docs/**/*.md", "docs/a.md", True), + ("*.md", "docs/a.md", False), + ("bun.lock", "bun.lock", True), + ("bun.lock", "apps/bun.lock", False), + ("Cargo.lock", "Cargo.lock", True), + (".github/workflows/ci.yml", ".github/workflows/ci.yml", True), + (".github/workflows/ci.yml", ".github/workflows/ciXyml", False), + ] + for pattern, path, want in cases: + got = matches(path, [pattern]) + assert got == want, f"{pattern!r} vs {path!r}: {got}" + assert matches("a/b.rs", ["a/**", "!a/*.rs"]) is False + assert matches("a/b.rs", ["a/**", "!a/*.rs", "a/b.rs"]) is True + + try: + import yaml + except ImportError: + yaml = None + checked = 0 + for workflow in sorted(WORKFLOWS.glob("*.yml")): + mine = triggers(workflow) + if yaml is None: + continue + data = yaml.safe_load(workflow.read_text()) + on = data.get("on", data.get(True)) + if isinstance(on, str): + on = {on: None} + elif isinstance(on, list): + on = {name: None for name in on} + assert set(mine) == set(on), f"{workflow.name}: events {sorted(mine)} != {sorted(on)}" + for event, config in on.items(): + if not isinstance(config, dict): + continue + for key in ("paths", "paths-ignore", "branches"): + theirs = (config or {}).get(key) + if theirs is not None: + assert mine[event].get(key) == theirs, f"{workflow.name} {event}.{key}: {mine[event].get(key)} != {theirs}" + checked += 1 + + with tempfile.TemporaryDirectory() as repo: + env = {"GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"} + git = lambda *a: subprocess.run(["git", *a], cwd=repo, check=True, capture_output=True, env={**os.environ, **env}) + git("init", "-q") + Path(repo, "a.txt").write_text("a") + git("add", "-A"); git("commit", "-q", "-m", "base") + base = run_git("rev-parse", "HEAD", cwd=repo).strip() + Path(repo, "apps/status/public").mkdir(parents=True) + Path(repo, "apps/status/public/caf\u00e9 x.svg").write_text("b") + Path(repo, "a.txt").rename(Path(repo, "b.txt")) + git("add", "-A"); git("commit", "-q", "-m", "head") + head = run_git("rev-parse", "HEAD", cwd=repo).strip() + got = sorted(changed_files(base, head, cwd=repo)) + assert got == ["a.txt", "apps/status/public/caf\u00e9 x.svg", "b.txt"], got + assert matches(got[1], ["apps/status/**"]) + + runs = [ + {"id": 1, "path": ".github/workflows/a.yml", "status": "completed", "conclusion": "failure", "html_url": "u1"}, + {"id": 2, "path": ".github/workflows/a.yml", "status": "completed", "conclusion": "success", "html_url": "u2"}, + {"id": 3, "path": ".github/workflows/b.yml", "status": "in_progress", "conclusion": None, "html_url": "u3"}, + ] + assert gate_verdict([".github/workflows/a.yml"], runs)[0] == "pass" + assert gate_verdict([".github/workflows/a.yml", ".github/workflows/b.yml"], runs)[0] == "wait" + assert gate_verdict([".github/workflows/c.yml"], runs)[0] == "wait" + runs[2].update(status="completed", conclusion="cancelled") + assert gate_verdict([".github/workflows/a.yml", ".github/workflows/b.yml"], runs)[0] == "fail" + runs[2]["conclusion"] = "skipped" + assert gate_verdict([".github/workflows/b.yml"], runs)[0] == "pass" + runs[1].update(status="in_progress", conclusion=None, check_suite_id=20) + checks = [ + {"name": "Clippy", "conclusion": "failure", "check_suite": {"id": 20}}, + {"name": "Merge queue", "conclusion": "failure", "check_suite": {"id": 99}}, + ] + state, lines = gate_verdict([".github/workflows/a.yml"], runs, checks) + assert state == "fail" and "Clippy failure" in lines[0], lines + assert gate_verdict([".github/workflows/a.yml"], runs, checks[1:])[0] == "wait" + print(f"ok ({len(cases)} patterns, {checked} workflows cross-checked, {len(queue_workflows())} run in the queue)") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + files = sub.add_parser("files") + files.add_argument("--base", required=True) + files.add_argument("--head", required=True) + scope = sub.add_parser("scope") + scope.add_argument("--workflow", required=True) + scope.add_argument("--base", required=True) + scope.add_argument("--head", required=True) + sub.add_parser("expected") + gate = sub.add_parser("gate") + gate.add_argument("--head", required=True) + gate.add_argument("--first-poll", type=float, default=30) + gate.add_argument("--poll", type=float, default=60) + gate.add_argument("--start-grace", type=float, default=20, help="minutes a queue workflow may take to start") + sub.add_parser("test") + args = parser.parse_args() + handler = { + "files": cmd_files, + "scope": cmd_scope, + "expected": cmd_expected, + "gate": cmd_gate, + "test": cmd_test, + }[args.command] + sys.exit(handler(args) or 0) + + +if __name__ == "__main__": + main()