From 1d9ab92ab462ab1436613a2ab4a1736240d41b36 Mon Sep 17 00:00:00 2001 From: austincalvelage Date: Wed, 4 Nov 2026 14:41:18 -0700 Subject: [PATCH 1/8] ci(repo): skip integration suites that a pull request doesn't affect A planning job runs turbo query affected once per pull request, comparing the merge commit with its base, and each integration job skips its setup and tests when its suite isn't affected. Every job still reports, so required checks pass. Changes under .github/, merge queue runs, and any planning failure run every suite. turbo.json enables affectedUsingTaskInputs so files outside a task's inputs, like changesets, don't mark every root task as affected. --- .changeset/integration-skip-unaffected.md | 2 + .github/workflows/ci.yml | 120 ++++++++++++++++++++-- turbo.json | 3 + 3 files changed, 115 insertions(+), 10 deletions(-) create mode 100644 .changeset/integration-skip-unaffected.md diff --git a/.changeset/integration-skip-unaffected.md b/.changeset/integration-skip-unaffected.md new file mode 100644 index 00000000000..a845151cc84 --- /dev/null +++ b/.changeset/integration-skip-unaffected.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1772dfea2ac..b5ec1785b96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -509,9 +509,79 @@ jobs: retention-days: 7 if-no-files-found: warn - integration-tests: - needs: [check-permissions, build-packages] + integration-plan: + needs: [check-permissions] if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + name: Integration Plan + permissions: + contents: read + runs-on: 'blacksmith-8vcpu-ubuntu-2204' + defaults: + run: + shell: bash + timeout-minutes: ${{ vars.TIMEOUT_MINUTES_NORMAL && fromJSON(vars.TIMEOUT_MINUTES_NORMAL) || 10 }} + outputs: + affected: ${{ steps.plan.outputs.affected }} + known: ${{ steps.plan.outputs.known }} + + steps: + - name: Checkout Repo + uses: actions/checkout@v4.4.0 + with: + persist-credentials: false + fetch-depth: 2 + fetch-tags: false + filter: 'blob:none' + show-progress: false + + - name: Setup + uses: $/.github/actions/init-blacksmith + with: + cache-enabled: true + turbo-signature: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }} + turbo-team: ${{ vars.TURBO_TEAM }} + turbo-token: ${{ secrets.TURBO_TOKEN }} + + - name: Find affected integration suites + id: plan + env: + EVENT_NAME: ${{ github.event_name }} + run: | + mapfile -t TASKS < <(jq -r '.tasks | keys[] | select(startswith("//#test:integration:")) | ltrimstr("//#")' turbo.json) + if [ "${#TASKS[@]}" -eq 0 ]; then + echo "::error::Found no integration tasks in turbo.json, so every integration suite runs." + exit 1 + fi + echo "known=$(printf '%s\n' "${TASKS[@]#test:integration:}" | jq -Rnc '[inputs]')" >> "$GITHUB_OUTPUT" + + if [ "$EVENT_NAME" != "pull_request" ]; then + echo "Running every integration suite for $EVENT_NAME." + echo "affected=all" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if git diff --name-only HEAD^1 HEAD | grep -q '^\.github/'; then + echo "This PR changes .github/, so every integration suite runs." + echo "affected=all" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^1 --head HEAD --tasks "${TASKS[@]}"); then + echo "::error::turbo query affected failed, so every integration suite runs." + exit 1 + fi + TURBO_JSON=$(printf '%s\n' "$TURBO_RAW" | awk '/^\{/{found=1} found') + if ! AFFECTED=$(jq -ec '[.data.affectedTasks.items[] | select(.package.name == "//" and (.name | startswith("test:integration:"))) | .name | ltrimstr("test:integration:")]' <<< "$TURBO_JSON"); then + echo "::error::turbo query affected returned unexpected output, so every integration suite runs." + printf '%s\n' "$TURBO_RAW" + exit 1 + fi + echo "affected=$AFFECTED" >> "$GITHUB_OUTPUT" + echo "Affected integration suites: $AFFECTED" + + integration-tests: + needs: [check-permissions, build-packages, integration-plan] + if: ${{ !cancelled() && needs.check-permissions.result == 'success' && needs.build-packages.result == 'success' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} name: Integration Tests (${{ matrix.test-name }}, ${{ matrix.test-project }}${{ matrix.next-version && format(', {0}', matrix.next-version) || '' }}) permissions: contents: read @@ -571,7 +641,28 @@ jobs: next-version: '16' steps: + - name: Check whether this suite is affected + id: affected + env: + AFFECTED: ${{ needs.integration-plan.outputs.affected }} + KNOWN: ${{ needs.integration-plan.outputs.known }} + PLAN_RESULT: ${{ needs.integration-plan.result }} + SUITE: ${{ matrix.test-name }} + run: | + if [ "$PLAN_RESULT" != "success" ] || [ -z "$AFFECTED" ] || [ "$AFFECTED" = "all" ]; then + echo "run=true" >> "$GITHUB_OUTPUT" + elif [[ "$KNOWN" != *"\"$SUITE\""* ]]; then + echo "::warning::$SUITE is not an integration task in turbo.json, so it runs and the task validation reports it." + echo "run=true" >> "$GITHUB_OUTPUT" + elif [[ "$AFFECTED" == *"\"$SUITE\""* ]]; then + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "::notice title=Integration suite skipped::Nothing the $SUITE suite uses changed in this PR." + echo "run=false" >> "$GITHUB_OUTPUT" + fi + - name: Checkout Repo + if: steps.affected.outputs.run == 'true' uses: actions/checkout@v4.4.0 with: persist-credentials: false @@ -581,6 +672,7 @@ jobs: show-progress: false - name: Setup + if: steps.affected.outputs.run == 'true' id: config uses: $/.github/actions/init-blacksmith with: @@ -591,6 +683,7 @@ jobs: playwright-enabled: true - name: Verify jq is installed + if: steps.affected.outputs.run == 'true' shell: bash run: | if ! command -v jq &> /dev/null; then @@ -600,6 +693,7 @@ jobs: jq --version - name: Validate turbo task + if: steps.affected.outputs.run == 'true' env: E2E_APP_CLERK_JS_DIR: ${{runner.temp}} E2E_APP_CLERK_UI_DIR: ${{runner.temp}} @@ -609,10 +703,8 @@ jobs: E2E_PROJECT: ${{ matrix.test-project }} INTEGRATION_INSTANCE_KEYS: ${{ secrets.INTEGRATION_INSTANCE_KEYS }} run: | - # Validate the turbo task exists. Turbo's --affected flag is a no-op for - # root-level tasks (//#) — they are always returned as affected. The previous - # --affected check only served to silently skip tests when the task was missing - # from turbo.json, hiding real configuration errors. + # Validate the turbo task exists, so a missing or misnamed task fails here + # instead of silently running nothing. TASK_NAME="test:integration:${{ matrix.test-name }}" TURBO_STDERR=$(mktemp) if ! TURBO_RAW=$(pnpm turbo run "$TASK_NAME" --dry=json 2>"$TURBO_STDERR"); then @@ -639,12 +731,15 @@ jobs: echo "Task '$TASK_NAME' validated ($TASK_COUNT tasks in graph)" - name: Build packages + if: steps.affected.outputs.run == 'true' run: pnpm turbo build $TURBO_ARGS --only - name: Publish to local registry + if: steps.affected.outputs.run == 'true' run: pkglab pub --force - name: Install @clerk/clerk-js in os temp + if: steps.affected.outputs.run == 'true' working-directory: ${{runner.temp}} run: | mkdir clerk-js && cd clerk-js @@ -653,6 +748,7 @@ jobs: pkglab add @clerk/clerk-js - name: Install @clerk/ui in os temp + if: steps.affected.outputs.run == 'true' working-directory: ${{runner.temp}} run: | mkdir clerk-ui && cd clerk-ui @@ -661,10 +757,11 @@ jobs: pkglab add @clerk/ui - name: Copy components @clerk/astro - if: ${{ matrix.test-name == 'astro' }} + if: ${{ steps.affected.outputs.run == 'true' && (matrix.test-name == 'astro') }} run: cd packages/astro && pnpm copy:components - name: Write all ENV certificates to files in integration/certs + if: steps.affected.outputs.run == 'true' uses: actions/github-script@v7.1.0 env: INTEGRATION_CERTS: '${{secrets.INTEGRATION_CERTS}}' @@ -682,11 +779,12 @@ jobs: } - name: LS certs + if: steps.affected.outputs.run == 'true' working-directory: ./integration/certs run: ls -la && pwd - name: Ensure Xvfb is installed - if: ${{ matrix.test-name == 'electron' }} + if: ${{ steps.affected.outputs.run == 'true' && (matrix.test-name == 'electron') }} run: | if ! command -v xvfb-run &> /dev/null; then sudo apt-get update @@ -694,6 +792,7 @@ jobs: fi - name: Configure test user cleanup + if: steps.affected.outputs.run == 'true' run: echo "INTEGRATION_TEST_RUN_KEY=${RUN_ID}-${RUN_ATTEMPT}-${TEST_NAME}-${TEST_PROJECT}-${NEXT_VERSION:-default}" >> "$GITHUB_ENV" env: RUN_ID: ${{ github.run_id }} @@ -703,6 +802,7 @@ jobs: NEXT_VERSION: ${{ matrix.next-version }} - name: Run Integration Tests + if: steps.affected.outputs.run == 'true' id: integration-tests timeout-minutes: 25 run: | @@ -726,7 +826,7 @@ jobs: VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} - name: Sanitize artifact name - if: ${{ cancelled() || failure() }} + if: ${{ steps.affected.outputs.run == 'true' && (cancelled() || failure()) }} id: sanitize run: | SANITIZED="${TEST_NAME//:/-}" @@ -735,7 +835,7 @@ jobs: TEST_NAME: ${{ matrix.test-name }} - name: Upload test-results - if: ${{ cancelled() || failure() }} + if: ${{ steps.affected.outputs.run == 'true' && (cancelled() || failure()) }} uses: actions/upload-artifact@v4.6.2 with: name: playwright-traces-${{ github.run_id }}-${{ github.run_attempt }}-${{ steps.sanitize.outputs.artifact-suffix }}${{ matrix.next-version && format('-next{0}', matrix.next-version) || '' }} diff --git a/turbo.json b/turbo.json index 461564fd311..99b3832521a 100644 --- a/turbo.json +++ b/turbo.json @@ -4,6 +4,9 @@ "signature": true }, "ui": "tui", + "futureFlags": { + "affectedUsingTaskInputs": true + }, "globalDependencies": [ ".github/.cache-version", ".npmrc", From ead5c061ef9bb9547f59942d0ec8f3a4f670d06e Mon Sep 17 00:00:00 2001 From: austincalvelage Date: Wed, 4 Nov 2026 15:34:22 -0700 Subject: [PATCH 2/8] chore(repo): temporarily compare the integration plan with the previous commit --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5ec1785b96..7918963e001 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -529,7 +529,7 @@ jobs: uses: actions/checkout@v4.4.0 with: persist-credentials: false - fetch-depth: 2 + fetch-depth: 3 fetch-tags: false filter: 'blob:none' show-progress: false @@ -560,13 +560,13 @@ jobs: exit 0 fi - if git diff --name-only HEAD^1 HEAD | grep -q '^\.github/'; then + if git diff --name-only HEAD^2^1 HEAD^2 | grep -q '^\.github/'; then echo "This PR changes .github/, so every integration suite runs." echo "affected=all" >> "$GITHUB_OUTPUT" exit 0 fi - if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^1 --head HEAD --tasks "${TASKS[@]}"); then + if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^2^1 --head HEAD^2 --tasks "${TASKS[@]}"); then echo "::error::turbo query affected failed, so every integration suite runs." exit 1 fi From 299d63b58b85e86978caa9d8df504a408fc471a1 Mon Sep 17 00:00:00 2001 From: austincalvelage Date: Wed, 4 Nov 2026 15:34:22 -0700 Subject: [PATCH 3/8] chore(repo): temporary expo-only change to prove skipped integration jobs --- packages/expo/src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/expo/src/index.ts b/packages/expo/src/index.ts index aa9223501c0..d62e1a28456 100644 --- a/packages/expo/src/index.ts +++ b/packages/expo/src/index.ts @@ -22,3 +22,5 @@ export * from './trusted-devices'; setErrorThrowerOptions({ packageName: PACKAGE_NAME }); export type { TokenCache } from './cache/types'; + +// CI cache probe, reverted in this PR From 88fb3c6697b9b0df9e7d10cb8097909e50d57b2b Mon Sep 17 00:00:00 2001 From: austincalvelage Date: Wed, 4 Nov 2026 15:38:09 -0700 Subject: [PATCH 4/8] chore(repo): revert temporary skip probe changes --- .github/workflows/ci.yml | 6 +++--- packages/expo/src/index.ts | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7918963e001..b5ec1785b96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -529,7 +529,7 @@ jobs: uses: actions/checkout@v4.4.0 with: persist-credentials: false - fetch-depth: 3 + fetch-depth: 2 fetch-tags: false filter: 'blob:none' show-progress: false @@ -560,13 +560,13 @@ jobs: exit 0 fi - if git diff --name-only HEAD^2^1 HEAD^2 | grep -q '^\.github/'; then + if git diff --name-only HEAD^1 HEAD | grep -q '^\.github/'; then echo "This PR changes .github/, so every integration suite runs." echo "affected=all" >> "$GITHUB_OUTPUT" exit 0 fi - if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^2^1 --head HEAD^2 --tasks "${TASKS[@]}"); then + if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^1 --head HEAD --tasks "${TASKS[@]}"); then echo "::error::turbo query affected failed, so every integration suite runs." exit 1 fi diff --git a/packages/expo/src/index.ts b/packages/expo/src/index.ts index d62e1a28456..aa9223501c0 100644 --- a/packages/expo/src/index.ts +++ b/packages/expo/src/index.ts @@ -22,5 +22,3 @@ export * from './trusted-devices'; setErrorThrowerOptions({ packageName: PACKAGE_NAME }); export type { TokenCache } from './cache/types'; - -// CI cache probe, reverted in this PR From 17e0056191d0df54c35ac79a7eb65b4c8c2dfe8d Mon Sep 17 00:00:00 2001 From: austincalvelage Date: Wed, 4 Nov 2026 15:44:58 -0700 Subject: [PATCH 5/8] chore(repo): temporarily compare the integration plan with the previous commit --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5ec1785b96..7918963e001 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -529,7 +529,7 @@ jobs: uses: actions/checkout@v4.4.0 with: persist-credentials: false - fetch-depth: 2 + fetch-depth: 3 fetch-tags: false filter: 'blob:none' show-progress: false @@ -560,13 +560,13 @@ jobs: exit 0 fi - if git diff --name-only HEAD^1 HEAD | grep -q '^\.github/'; then + if git diff --name-only HEAD^2^1 HEAD^2 | grep -q '^\.github/'; then echo "This PR changes .github/, so every integration suite runs." echo "affected=all" >> "$GITHUB_OUTPUT" exit 0 fi - if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^1 --head HEAD --tasks "${TASKS[@]}"); then + if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^2^1 --head HEAD^2 --tasks "${TASKS[@]}"); then echo "::error::turbo query affected failed, so every integration suite runs." exit 1 fi From f3d4c97ba0842f9354bcc35e37f0edfb2cb232ae Mon Sep 17 00:00:00 2001 From: austincalvelage Date: Wed, 4 Nov 2026 15:44:58 -0700 Subject: [PATCH 6/8] chore(repo): temporary mosaic-only change to prove skipped integration jobs --- packages/mosaic/src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/mosaic/src/index.ts b/packages/mosaic/src/index.ts index a64f3111ea2..da0e8742943 100644 --- a/packages/mosaic/src/index.ts +++ b/packages/mosaic/src/index.ts @@ -14,3 +14,5 @@ export type { MosaicProfile } from './features/profile-pages/use-mosaic-profile' export { MosaicProvider } from './mosaic-provider'; export type { PortalContainer, PortalProviderProps } from './primitives/portal'; export { PortalProvider } from './primitives/portal'; + +// CI cache probe, reverted in this PR From ebecee2a3bf2dc4739a1e521d468687205a9a0ca Mon Sep 17 00:00:00 2001 From: austincalvelage Date: Wed, 4 Nov 2026 15:52:14 -0700 Subject: [PATCH 7/8] chore(repo): revert temporary skip probe changes --- .github/workflows/ci.yml | 6 +++--- packages/mosaic/src/index.ts | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7918963e001..b5ec1785b96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -529,7 +529,7 @@ jobs: uses: actions/checkout@v4.4.0 with: persist-credentials: false - fetch-depth: 3 + fetch-depth: 2 fetch-tags: false filter: 'blob:none' show-progress: false @@ -560,13 +560,13 @@ jobs: exit 0 fi - if git diff --name-only HEAD^2^1 HEAD^2 | grep -q '^\.github/'; then + if git diff --name-only HEAD^1 HEAD | grep -q '^\.github/'; then echo "This PR changes .github/, so every integration suite runs." echo "affected=all" >> "$GITHUB_OUTPUT" exit 0 fi - if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^2^1 --head HEAD^2 --tasks "${TASKS[@]}"); then + if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^1 --head HEAD --tasks "${TASKS[@]}"); then echo "::error::turbo query affected failed, so every integration suite runs." exit 1 fi diff --git a/packages/mosaic/src/index.ts b/packages/mosaic/src/index.ts index da0e8742943..a64f3111ea2 100644 --- a/packages/mosaic/src/index.ts +++ b/packages/mosaic/src/index.ts @@ -14,5 +14,3 @@ export type { MosaicProfile } from './features/profile-pages/use-mosaic-profile' export { MosaicProvider } from './mosaic-provider'; export type { PortalContainer, PortalProviderProps } from './primitives/portal'; export { PortalProvider } from './primitives/portal'; - -// CI cache probe, reverted in this PR From aa3e90bde7ac04b2dd61f2680e2453fa995cf452 Mon Sep 17 00:00:00 2001 From: austincalvelage Date: Thu, 5 Nov 2026 10:58:03 -0700 Subject: [PATCH 8/8] ci(repo): detect .github changes without a pipe Under pipefail, grep -q exiting on an early .github/ match sent SIGPIPE to git diff, so the check came out false on large PRs and suites were skipped. Replaying the last 60 commits that touched .github/, it missed a 470-file one. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5ec1785b96..278dbe96d5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -560,7 +560,7 @@ jobs: exit 0 fi - if git diff --name-only HEAD^1 HEAD | grep -q '^\.github/'; then + if ! git diff --quiet HEAD^1 HEAD -- .github; then echo "This PR changes .github/, so every integration suite runs." echo "affected=all" >> "$GITHUB_OUTPUT" exit 0