WIP: [CI] Prototype a merge queue for main

.asf.yaml

@@ -28,8 +28,37 @@ github:
     discussions: true
 
   protected_branches:
-    # protect main against force push
-    main: {}
+    main:
+      required_status_checks:
+        # The queue tests the combined commit against current main.
+        strict: false
+        contexts:
+          - Merge Queue
+
+  rulesets:
+    - name: Merge Queue
+      target: branch
+      enforcement: active
+      bypass_actors:
+        # apache/root (ASF Infra) can recover a blocked queue.
+        - actor_id: 118420
+          actor_type: Team
+          bypass_mode: always
+      conditions:
+        ref_name:
+          include: [refs/heads/main]
+          exclude: []
+      rules:
+        - type: merge_queue
+          parameters:
+            merge_method: SQUASH
+            grouping_strategy: ALLGREEN
+            # Start with one candidate at a time to bound runner demand.
+            max_entries_to_build: 1
+            min_entries_to_merge: 1
+            max_entries_to_merge: 1
+            min_entries_to_merge_wait_minutes: 0
+            check_response_timeout_minutes: 360
 
   collaborators:
     - alinaliBQ

.github/workflows/README.md

@@ -0,0 +1,52 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Merge queue
+
+`merge_queue.yml` supplies the required **Merge Queue** check for `main`.
+On pull requests it permits entry into the queue without repeating the existing
+PR workflows. On `merge_group`, it runs every regular language and tooling
+workflow against the candidate containing current `main` and the queued change.
+Every called workflow must succeed; failure, cancellation, and skipped suites
+block merging. Extra, packaging, and release workflows retain their separate
+triggers and are not part of this gate.
+
+The `.asf.yaml` configuration activates the queue with one candidate build at a
+time and a six-hour check timeout, including runner scheduling. It leaves the
+existing PR and post-merge CI in place. This first step adds validation before
+merge; it does not reduce CI usage. Queue branch pushes are excluded so they
+cannot start duplicate builds.
+
+Use GitHub's merge queue UI to enqueue a reviewed PR. `dev/merge_arrow_pr.py`
+uses the direct merge API and cannot enqueue or wait for a queued merge. Close
+the associated issue and set its milestone after the queued PR actually merges.
+Maintenance branches retain their existing merge procedure.
+
+To exercise the same build suite on a fork with the workflow installed:
+
+```shell
+gh workflow run merge_queue.yml --repo OWNER/arrow --ref BRANCH
+```
+
+GitHub requires an organization-owned repository for a native merge queue.
+A manual fork run checks the suite and result aggregation, but does not test
+queue ordering, stale-candidate replacement, or automatic merging. Validate
+those behaviors and runner availability before enabling the proposed rules in
+Apache Arrow. Queue settings follow the
+[GitHub merge queue documentation](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue).

.github/workflows/archery.yml

@@ -18,22 +18,28 @@
 name: Archery & Crossbow
 
 on:
+  workflow_call:
   push:
     branches:
       - '**'
       - '!dependabot/**'
+      - '!gh-readonly-queue/**'
     tags:
       - '**'
     paths:
       - '.dockerignore'
-      - '.github/workflows/archery.yml'
+      - '.asf.yaml'
+      - '.github/workflows/**'
+      - 'dev/test_merge_queue.py'
       - 'dev/archery/**'
       - 'dev/tasks/**'
       - 'compose.yaml'
   pull_request:
     paths:
       - '.dockerignore'
-      - '.github/workflows/archery.yml'
+      - '.asf.yaml'
+      - '.github/workflows/**'
+      - 'dev/test_merge_queue.py'
       - 'dev/archery/**'
       - 'dev/tasks/**'
       - 'compose.yaml'
@@ -43,7 +49,7 @@ env:
   ARCHERY_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
 
 concurrency:
-  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}-archery
   cancel-in-progress: true
 
 permissions:
@@ -77,7 +83,7 @@ jobs:
           pip install -r dev/archery/requirements-test.txt
       - name: Archery Unittests
         working-directory: dev/archery
-        run: pytest -v archery
+        run: pytest -v archery ../test_merge_queue.py
       - name: Archery Docker Validation
         run: archery docker check-config
       - name: Crossbow Check Config

.github/workflows/cpp.yml

@@ -18,10 +18,12 @@
 name: C++
 
 on:
+  workflow_call:
   push:
     branches:
       - '**'
       - '!dependabot/**'
+      - '!gh-readonly-queue/**'
     tags:
       - '**'
     paths:
@@ -61,7 +63,7 @@ on:
       - 'testing'
 
 concurrency:
-  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}-cpp
   cancel-in-progress: true
 
 permissions:

.github/workflows/cpp_extra.yml

@@ -22,6 +22,7 @@ on:
     branches:
       - '**'
       - '!dependabot/**'
+      - '!gh-readonly-queue/**'
     paths:
       - '.dockerignore'
       - '.github/workflows/check_labels.yml'

.github/workflows/cuda_extra.yml

@@ -22,6 +22,7 @@ on:
     branches:
       - '**'
       - '!dependabot/**'
+      - '!gh-readonly-queue/**'
     paths:
       - '**/*cuda*'
       - '**/*gpu*'

.github/workflows/dev.yml

@@ -18,17 +18,19 @@
 name: Dev
 
 on:
+  workflow_call:
   # always trigger except Dependabot "push"
   push:
     branches:
       - '**'
       - '!dependabot/**'
+      - '!gh-readonly-queue/**'
     tags:
       - '**'
   pull_request:
 
 concurrency:
-  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}-dev
   cancel-in-progress: true
 
 permissions:

.github/workflows/docs.yml

@@ -18,10 +18,12 @@
 name: Docs
 
 on:
+  workflow_call:
   push:
     branches:
       - '**'
       - '!dependabot/**'
+      - '!gh-readonly-queue/**'
     tags:
       - '**'
   pull_request:

.github/workflows/docs_light.yml

@@ -18,6 +18,7 @@
 name: Docs
 
 on:
+  workflow_call:
   pull_request:
     paths:
       - '.dockerignore'

.github/workflows/integration.yml

@@ -18,10 +18,12 @@
 name: Integration
 
 on:
+  workflow_call:
   push:
     branches:
       - '**'
       - '!dependabot/**'
+      - '!gh-readonly-queue/**'
     tags:
       - '**'
     paths:
@@ -47,7 +49,7 @@ on:
       - 'ruby/red-arrow-format/**'
 
 concurrency:
-  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}-integration
   cancel-in-progress: true
 
 permissions:

.github/workflows/matlab.yml

@@ -18,10 +18,12 @@
 name: MATLAB
 
 on:
+  workflow_call:
   push:
     branches:
       - '**'
       - '!dependabot/**'
+      - '!gh-readonly-queue/**'
     tags:
       - '**'
     paths:
@@ -37,7 +39,7 @@ on:
       - 'cpp/src/arrow/**'
 
 concurrency:
-  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}-matlab
   cancel-in-progress: true
 
 permissions:

.github/workflows/merge_queue.yml

@@ -0,0 +1,122 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Merge Queue CI
+
+on:
+  pull_request:
+    branches: [main]
+  merge_group:
+    types: [checks_requested]
+    branches: [main]
+  workflow_dispatch:
+
+concurrency:
+  group: merge-queue-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+
+  archery:
+    if: github.event_name != 'pull_request'
+    uses: ./.github/workflows/archery.yml
+    secrets: inherit
+
+  cpp:
+    if: github.event_name != 'pull_request'
+    uses: ./.github/workflows/cpp.yml
+    secrets: inherit
+
+  dev:
+    if: github.event_name != 'pull_request'
+    uses: ./.github/workflows/dev.yml
+    secrets: inherit
+
+  docs:
+    if: github.event_name != 'pull_request'
+    uses: ./.github/workflows/docs.yml
+    secrets: inherit
+
+  docs_light:
+    if: github.event_name != 'pull_request'
+    uses: ./.github/workflows/docs_light.yml
+    secrets: inherit
+
+  integration:
+    if: github.event_name != 'pull_request'
+    uses: ./.github/workflows/integration.yml
+    secrets: inherit
+
+  matlab:
+    if: github.event_name != 'pull_request'
+    uses: ./.github/workflows/matlab.yml
+    secrets: inherit
+
+  python:
+    if: github.event_name != 'pull_request'
+    uses: ./.github/workflows/python.yml
+    secrets: inherit
+
+  r:
+    if: github.event_name != 'pull_request'
+    uses: ./.github/workflows/r.yml
+    secrets: inherit
+
+  ruby:
+    if: github.event_name != 'pull_request'
+    uses: ./.github/workflows/ruby.yml
+    secrets: inherit
+    permissions:
+      actions: read
+      contents: read
+      # The existing Windows job caches vcpkg packages in GitHub Packages.
+      packages: write
+
+  required:
+    name: Merge Queue
+    if: always()
+    needs: [archery, cpp, dev, docs, docs_light, integration, matlab, python, r, ruby]
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions: {}
+    steps:
+      - name: Check every suite result
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          NEEDS: ${{ toJSON(needs) }}
+        shell: python
+        run: |
+          import json
+          import os
+          import sys
+
+          event = os.environ["EVENT_NAME"]
+          results = json.loads(os.environ["NEEDS"])
+          # PRs can enter the queue; the combined commit must pass every suite.
+          if event == "pull_request":
+              expected = "skipped"
+          elif event in ("merge_group", "workflow_dispatch"):
+              expected = "success"
+          else:
+              sys.exit(f"Unexpected event: {event}")
+          print(json.dumps(results, indent=2))
+          if not results or any(job["result"] != expected for job in results.values()):
+              sys.exit(f"Every suite must report {expected}")

.github/workflows/package_linux.yml

@@ -22,6 +22,7 @@ on:
     branches:
       - '**'
       - '!dependabot/**'
+      - '!gh-readonly-queue/**'
       - '!release-*'
     paths:
       - '.github/workflows/check_labels.yml'