ci: harden workflows, add region fallback & /gcbrun, and remove legacy CI files

.env.example

@@ -56,7 +56,13 @@ GOOGLE_APPLICATION_CREDENTIALS="/path/to/your/service-account-key.json"
 # Custom Dataproc API endpoint (uncomment if needed)
 # GOOGLE_CLOUD_DATAPROC_API_ENDPOINT="your-region-dataproc.googleapis.com"
 
-# Subnet URI for Managed Spark Connect (full resource name format)
-# Example: projects/your-project-id/regions/us-central1/subnetworks/your-subnet-name
+# Subnet name or URI for Managed Spark Connect (optional; defaults to "default" in tests)
+# Accepts a short subnet name (e.g. "default") or full resource URI:
+# MANAGED_SPARK_CONNECT_SUBNET="default"
 # MANAGED_SPARK_CONNECT_SUBNET="projects/your-project-id/regions/us-central1/subnetworks/your-subnet-name"
 
+# Comma-separated fallback regions for integration tests when a short subnet name is used
+# and the primary region encounters a 503 stockout (defaults to us-central1,us-east1,us-west1)
+# GOOGLE_CLOUD_FALLBACK_REGIONS="us-central1,us-east1,us-west1"
+
+

.kokoro/trampoline.sh → .github/dependabot.yml

@@ -1,28 +1,24 @@
-#!/bin/bash
-# Copyright 2024 Google LLC
+# Copyright 2026 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -eo pipefail
-
-# Always run the cleanup script, regardless of the success of bouncing into
-# the container.
-function cleanup() {
-    chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh
-    ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh
-    echo "cleanup";
-}
-trap cleanup EXIT
-
-$(dirname $0)/populate-secrets.sh # Secret Manager secrets.
-python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py"
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      github-actions:
+        patterns:
+          - "*"

.github/workflows/integration-tests.yaml

@@ -16,34 +16,105 @@
 #
 # Required GitHub Secrets:
 #   - GCP_SA_KEY: Service account JSON key (project_id and client_email extracted automatically)
-#   - GCP_REGION: Google Cloud Region (optional, defaults to us-central1)
-#   - GCP_SUBNET: Managed Spark subnet name or URI (optional, defaults to "default")
 #
-# See INTEGRATION_TESTS.md for setup instructions.
+# Optional GitHub Repository Variables (vars.*):
+#   - GCP_REGION: Google Cloud Region (defaults to us-central1)
+#   - GCP_SUBNET: Managed Spark subnet name or URI (defaults to "default")
+#
+# Fork PRs can be tested by a repository maintainer commenting `/gcbrun` on the PR.
+# See DEVELOPING.md for local setup instructions.
 
 name: Integration Tests
 on:
+  push:
+    branches: [ main ]
   pull_request:
     branches: [ main ]
+  issue_comment:
+    types: [ created ]
   workflow_dispatch:
 
 permissions:
   contents: read
+  pull-requests: read
+  statuses: write
 
 jobs:
   integration-test:
     name: Run integration tests
     runs-on: ubuntu-latest
+    timeout-minutes: 25
+
+    # Run on push to main, manual dispatch, non-Dependabot same-repo pull requests,
+    # or maintainer /gcbrun comments on PRs.
+    if: >-
+      github.event_name == 'push' ||
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') ||
+      (github.event_name == 'issue_comment' && github.event.issue.pull_request != null && contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) && (github.event.comment.body == '/gcbrun' || startsWith(github.event.comment.body, '/gcbrun ') || startsWith(github.event.comment.body, '/gcbrun\r') || startsWith(github.event.comment.body, '/gcbrun\n')))
 
-    # Only run integration tests if secrets are available
-    if: github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number || github.ref }}
+      cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'issue_comment' }}
 
     steps:
+    - name: Authorize /gcbrun and resolve PR head SHA
+      if: github.event_name == 'issue_comment'
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
+        COMMENT_BODY: ${{ github.event.comment.body }}
+        COMMENT_CREATED_AT: ${{ github.event.comment.created_at }}
+        PR_NUMBER: ${{ github.event.issue.number }}
+        RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      run: |
+        PERMISSION=$(gh api "repos/${GITHUB_REPOSITORY}/collaborators/${COMMENT_AUTHOR}/permission" --jq '.permission')
+        if [[ "$PERMISSION" != "admin" && "$PERMISSION" != "maintain" && "$PERMISSION" != "write" ]]; then
+          echo "::error::User ${COMMENT_AUTHOR} (permission: ${PERMISSION}) is not authorized to trigger /gcbrun."
+          exit 1
+        fi
+        PR_JSON=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")
+        PR_STATE=$(echo "$PR_JSON" | jq -r '.state')
+        if [[ "$PR_STATE" != "open" ]]; then
+          echo "::error::PR #${PR_NUMBER} is not open (state: ${PR_STATE})."
+          exit 1
+        fi
+        PR_HEAD_SHA=$(echo "$PR_JSON" | jq -r '.head.sha')
+        REQUESTED_SHA=$(echo "$COMMENT_BODY" | head -n1 | tr -d '\r' | sed -E 's|^/gcbrun[[:space:]]*||; s|[[:space:]]+$||')
+        if [[ -n "$REQUESTED_SHA" && "$PR_HEAD_SHA" != "$REQUESTED_SHA"* ]]; then
+          echo "::error::PR head (${PR_HEAD_SHA}) does not match requested /gcbrun SHA (${REQUESTED_SHA})."
+          exit 1
+        fi
+        SUITE_CREATED_AT=$(gh api "repos/${GITHUB_REPOSITORY}/commits/${PR_HEAD_SHA}/check-suites" --jq '[.check_suites[].created_at] | min // empty' || true)
+        if [[ -n "$SUITE_CREATED_AT" && "$SUITE_CREATED_AT" > "$COMMENT_CREATED_AT" ]]; then
+          echo "::error::PR head commit ${PR_HEAD_SHA} was pushed at ${SUITE_CREATED_AT}, after /gcbrun comment at ${COMMENT_CREATED_AT}."
+          exit 1
+        fi
+        echo "PR_HEAD_SHA=${PR_HEAD_SHA}" >> "$GITHUB_ENV"
+        gh api "repos/${GITHUB_REPOSITORY}/statuses/${PR_HEAD_SHA}" \
+          -f state="pending" \
+          -f target_url="${RUN_URL}" \
+          -f description="Integration tests running via /gcbrun" \
+          -f context="Integration Tests / Run integration tests"
+
     - name: Checkout code
       uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       with:
         persist-credentials: false
 
+    - name: Checkout PR head commit for /gcbrun
+      if: github.event_name == 'issue_comment'
+      env:
+        PR_NUMBER: ${{ github.event.issue.number }}
+      run: |
+        git fetch origin "refs/pull/${PR_NUMBER}/head"
+        FETCH_SHA=$(git rev-parse FETCH_HEAD)
+        if [[ "$FETCH_SHA" != "$PR_HEAD_SHA" ]]; then
+          echo "::error::Fetched PR head (${FETCH_SHA}) does not match authorized SHA (${PR_HEAD_SHA}). Re-run /gcbrun after reviewing the new commit."
+          exit 1
+        fi
+        git checkout --detach "${PR_HEAD_SHA}"
+
     - name: Setup Python
       uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
       with:
@@ -86,12 +157,32 @@ jobs:
     - name: Run integration tests
       env:
         CI: "true"
-        # Extracted from service account JSON in the previous step
+        # Extracted from service account JSON in the previous step (kept secret)
         GOOGLE_CLOUD_PROJECT: ${{ env.PROJECT_ID }}
         MANAGED_SPARK_CONNECT_SERVICE_ACCOUNT: ${{ env.SA_EMAIL }}
-        # Infrastructure-specific secrets
-        GOOGLE_CLOUD_REGION: ${{ secrets.GCP_REGION || 'us-central1' }}
-        MANAGED_SPARK_CONNECT_SUBNET: ${{ secrets.GCP_SUBNET }}
+        # Non-secret infrastructure variables (falling back to secrets during transition)
+        GOOGLE_CLOUD_REGION: ${{ vars.GCP_REGION || secrets.GCP_REGION || 'us-central1' }}
+        MANAGED_SPARK_CONNECT_SUBNET: ${{ vars.GCP_SUBNET || secrets.GCP_SUBNET }}
         MANAGED_SPARK_CONNECT_AUTH_TYPE: "SERVICE_ACCOUNT"
       run: |
-        python -m pytest tests/integration/ -v --tb=short -x
+        python -m pytest tests/integration/ -v --tb=short -x
+
+    - name: Report /gcbrun commit status
+      if: always() && github.event_name == 'issue_comment' && env.PR_HEAD_SHA != ''
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        JOB_STATUS: ${{ job.status }}
+        RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      run: |
+        if [[ "$JOB_STATUS" == "success" ]]; then
+          STATE="success"
+          DESC="Integration tests passed via /gcbrun"
+        else
+          STATE="failure"
+          DESC="Integration tests ${JOB_STATUS} via /gcbrun"
+        fi
+        gh api "repos/${GITHUB_REPOSITORY}/statuses/${PR_HEAD_SHA}" \
+          -f state="${STATE}" \
+          -f target_url="${RUN_URL}" \
+          -f description="${DESC}" \
+          -f context="Integration Tests / Run integration tests"

.github/workflows/lint-readme.yaml

@@ -21,12 +21,22 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   lint-readme:
     name: lint-readme
     runs-on: ubuntu-latest
+    timeout-minutes: 5
 
     steps:
+    - name: Checkout code
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      with:
+        persist-credentials: false
+
     - name: Setup Python
       uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
       with:
@@ -35,9 +45,6 @@ jobs:
     - name: Set up uv
       uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
 
-    - name: Checkout code
-      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-
     - name: Install dependencies
       run: uv pip install --system build twine
 

.github/workflows/pyink.yaml

@@ -21,12 +21,22 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   pyink:
     name: Run pyink
     runs-on: ubuntu-latest
+    timeout-minutes: 5
 
     steps:
+    - name: Checkout code
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      with:
+        persist-credentials: false
+
     - name: Setup Python
       uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
       with:
@@ -36,10 +46,7 @@ jobs:
       uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
 
     - name: Install pyink
-      run: uv pip install --system pyink
-
-    - name: Checkout code
-      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      run: uv pip install --system 'pyink~=24.0'
 
     - name: Run pyink check
-      run: pyink . --check
+      run: pyink . --check

.github/workflows/tests.yaml

@@ -14,17 +14,24 @@
 
 name: Unit Tests
 on:
+  push:
+    branches: [ main ]
   pull_request:
     branches: [ main ]
   workflow_dispatch:
 
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   test:
     name: Run unit tests ([${{ matrix.spark-extra }}])
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     strategy:
       fail-fast: false
       matrix:
@@ -35,6 +42,8 @@ jobs:
     steps:
     - name: Checkout code
       uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      with:
+        persist-credentials: false
 
     - name: Setup Python
       uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
@@ -75,4 +84,4 @@ jobs:
       if: matrix.spark-extra == 'spark'
       run: python -m pytest tests/local/ -v --tb=short
       env:
-        JAVA_HOME: ${{ env.JAVA_HOME_17_X64 }}
+        JAVA_HOME: ${{ env.JAVA_HOME_17_X64 }}