From be11fd09a7210a47580cc0e6ec9d2b02556f69dd Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Tue, 20 Oct 2026 12:05:15 +0200 Subject: [PATCH] ci: Run optional E2E tests in a separate workflow Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yml | 222 ++++++++++++---------- .github/workflows/optional-e2e.yml | 160 ++++++++++++++++ dev-packages/test-utils/src/cloudflare.ts | 6 +- 3 files changed, 284 insertions(+), 104 deletions(-) create mode 100644 .github/workflows/optional-e2e.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7502ea53b9ac..811664bcba51 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1185,8 +1185,10 @@ jobs: retention-days: 7 # - We skip optional tests on release branches + # - The tests run in optional-e2e.yml, so their failures do not mark the PR as failed. This job only waits for + # them and reports failures in a PR comment. job_optional_e2e_tests: - name: E2E ${{ matrix.label || matrix.test-application }} Test (optional) + name: E2E Tests (optional) # We only run E2E tests for non-fork PRs because the E2E tests require secrets to work and they can't be accessed from forks # We need to add the `always()` check here because the previous step has this as well :( # See: https://github.com/actions/runner/issues/2205 @@ -1196,110 +1198,126 @@ jobs: github.event.pull_request.head.repo.full_name == github.repository) && github.actor != 'dependabot[bot]' needs: [job_get_metadata, job_build, job_build_tarballs] runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 60 + permissions: + actions: write + pull-requests: write env: - E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }} - E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }} - # Needed because some apps expect a certain prefix - NEXT_PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }} - PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }} - REACT_APP_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }} - E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks' - E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests' - E2E_OPENROUTER_API_KEY: ${{ secrets.E2E_OPENROUTER_API_KEY }} - # Only this app talks to OpenAI directly (OpenRouter has no Decisions route), so only it gets the key. - E2E_OPENAI_DECISIONS_API_KEY: - ${{ matrix.test-application == 'node-openai-decisions-send-to-sentry' && secrets.E2E_OPENAI_DECISIONS_API_KEY || - '' }} - # Used by test apps that deploy a real Cloudflare Worker, e.g. cloudflare-workers-send-to-sentry - CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} - CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - # The `runtime` of a runtime variant, see dev-packages/e2e-tests/README.md - RUNTIME: ${{ matrix.runtime }} - # Without the root node_modules, `pnpm run` would first install the whole workspace. - pnpm_config_verify_deps_before_run: 'false' - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.job_build.outputs.e2e-matrix-optional) }} - + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + CORRELATION_ID: ${{ github.run_id }}-${{ github.run_attempt }} steps: - - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) - uses: actions/checkout@v7 - with: - ref: ${{ env.HEAD_COMMIT }} - - name: Set up Node - uses: actions/setup-node@v7 - with: - node-version-file: 'dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}/package.json' - - name: Set up Bun - if: - matrix.test-application == 'node-exports-test-app' || contains(matrix.test-application, 'bun') || - matrix.runtime == 'bun' - uses: oven-sh/setup-bun@v2 - with: - bun-version: '1.3.14' - - name: Set up Deno - if: contains(matrix.test-application, 'deno') || matrix.runtime == 'deno' - uses: denoland/setup-deno@v2.0.5 - with: - deno-version: ${{ matrix.deno-version || 'v2.8.3' }} - - name: Install pnpm - uses: ./.github/actions/install-pnpm - - - name: Restore tarball artifacts - uses: actions/download-artifact@v8 - with: - name: build-tarball-output - path: ${{ env.TARBALL_ARTIFACT_DOWNLOAD_PATH }} - - - name: Restore test-utils tarball artifact - uses: actions/download-artifact@v8 - with: - name: build-test-utils-tarball-output - path: dev-packages/test-utils - - - name: Prepare E2E tests - run: pnpm run test:prepare - working-directory: dev-packages/e2e-tests - - - name: Validate test setup - run: pnpm run test:validate - working-directory: dev-packages/e2e-tests - - - name: Copy to temp - run: - pnpm run ci:copy-to-temp ./test-applications/${{ matrix.test-application }} ${{ runner.temp - }}/test-application - working-directory: dev-packages/e2e-tests - - - name: Add pnpm overrides - run: - pnpm run ci:pnpm-overrides ${{ runner.temp }}/test-application ${{ github.workspace - }}/dev-packages/e2e-tests/packed - working-directory: dev-packages/e2e-tests - - # Installs Playwright while the app builds. Keep it right before the build: the runner sets up a background - # composite action's steps from the job's shared `github` context, which fails with "Collection was modified" if - # a foreground step starts at that moment. Starting next to `download-artifact` also made it fail to read the event file. - - name: Install Playwright - id: install-playwright - background: true - uses: ./.github/actions/install-playwright - with: - browsers: chromium - cwd: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }} - - - name: Build E2E app - working-directory: ${{ runner.temp }}/test-application - timeout-minutes: 7 - run: ${{ matrix.build-command || 'pnpm test:build' }} - - - wait: [install-playwright] + - name: Trigger optional E2E tests + env: + MATRIX: ${{ needs.job_build.outputs.e2e-matrix-optional }} + PR_NUMBER: ${{ github.event.pull_request.number }} + DISPATCH_REF: ${{ github.head_ref || github.ref_name }} + run: | + # A dispatch by file name only works when the file is on the default branch, a dispatch by ID always works. + workflow_id=$(gh api "repos/$GH_REPO/actions/workflows" --paginate \ + --jq '.workflows[] | select(.path == ".github/workflows/optional-e2e.yml") | .id') + echo "WORKFLOW_ID=$workflow_id" >> "$GITHUB_ENV" + + gh workflow run "$workflow_id" --ref "$DISPATCH_REF" \ + -f commit="$HEAD_COMMIT" \ + -f build_run_id="$GITHUB_RUN_ID" \ + -f matrix="$MATRIX" \ + -f pr_number="$PR_NUMBER" \ + -f correlation_id="$CORRELATION_ID" + + - name: Wait for optional E2E tests + id: wait + run: | + for _ in $(seq 1 30); do + run_id=$(gh run list --workflow "$WORKFLOW_ID" --event workflow_dispatch --limit 20 \ + --json databaseId,displayTitle \ + --jq ".[] | select(.displayTitle | contains(\"$CORRELATION_ID\")) | .databaseId" | head -1) + [ -n "$run_id" ] && break + sleep 2 + done + [ -z "$run_id" ] && echo "::error::Optional E2E run not found" && exit 1 + run_url="$GITHUB_SERVER_URL/$GH_REPO/actions/runs/$run_id" + echo "Optional E2E run: $run_url" + + until [ "$(gh run view "$run_id" --json status --jq .status)" = completed ]; do + sleep 30 + done + + conclusion=$(gh run view "$run_id" --json conclusion --jq .conclusion) + echo "Optional E2E tests finished with: $conclusion" + [ "$conclusion" != success ] && echo "::warning::Optional E2E tests $conclusion: $run_url" + echo "run_id=$run_id" >> "$GITHUB_OUTPUT" + echo "url=$run_url" >> "$GITHUB_OUTPUT" + exit 0 + + # Keeps one comment per PR up to date, so new pushes do not add more comments. The comment + # is only created when a check fails. + - name: Report result in PR comment + if: github.event_name == 'pull_request' + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + RUN_ID: ${{ steps.wait.outputs.run_id }} + RUN_URL: ${{ steps.wait.outputs.url }} + run: | + gh run view "$RUN_ID" --json jobs > jobs.json + + rows() { + jq -r --arg filter "$1" ' + def dur: ((.completedAt | fromdate) - (.startedAt | fromdate)) as $s + | if $s >= 60 then "\($s / 60 | floor)m" else "\($s)s" end; + def icon: {success: "✅", failure: "❌", cancelled: "⛔", timed_out: "⏱️"}[.conclusion] // "❔"; + def status: + if .conclusion == "success" then "Successful in \(dur)" + elif .conclusion == "cancelled" then "Cancelled after \(dur)" + else "Failing after \(dur)" end; + .jobs[] + | select(.conclusion != "skipped") + | select(if $filter == "success" then .conclusion == "success" else .conclusion != "success" end) + | "| \(icon) | CI: Optional E2E Tests / \(.name) | \(status) | [Details](\(.url)) |"' jobs.json + } + failing=$(rows failing) + successful=$(rows success) + counts=$(jq -r '[.jobs[] | select(.conclusion != "skipped")] + | [(map(select(.conclusion != "success")) | length | select(. > 0) | "\(.) failing"), + (map(select(.conclusion == "success")) | length | select(. > 0) | "\(.) successful")] + | join(", ")' jobs.json) + header=$'| | Check | Status | |\n|---|---|---|---|' + + marker='' + comment_id=$(gh api "repos/$GH_REPO/issues/$PR_NUMBER/comments" --paginate \ + --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | contains(\"$marker\"))) | .id" | tail -1) + if [ -z "$failing" ] && [ -z "$comment_id" ]; then + exit 0 + fi - - name: Run E2E test - working-directory: ${{ runner.temp }}/test-application - timeout-minutes: 10 - run: ${{ matrix.assert-command || 'pnpm test:assert' }} + { + echo "$marker" + echo '### Optional E2E tests' + echo + if [ -n "$failing" ]; then + echo "Some optional checks were not successful. They do not block the merge. $counts. [See the run]($RUN_URL)." + echo + echo "$header" + echo "$failing" + else + echo "All optional checks were successful. $counts. [See the run]($RUN_URL)." + fi + if [ -n "$successful" ]; then + echo + echo '
Successful checks' + echo + echo "$header" + echo "$successful" + echo + echo '
' + fi + } > comment.md + + if [ -n "$comment_id" ]; then + gh api -X PATCH "repos/$GH_REPO/issues/comments/$comment_id" -F body=@comment.md > /dev/null + else + gh pr comment "$PR_NUMBER" --body-file comment.md + fi job_required_jobs_passed: name: All required jobs passed or were skipped diff --git a/.github/workflows/optional-e2e.yml b/.github/workflows/optional-e2e.yml new file mode 100644 index 000000000000..53dd4929dacb --- /dev/null +++ b/.github/workflows/optional-e2e.yml @@ -0,0 +1,160 @@ +name: 'CI: Optional E2E Tests' +run-name: Optional E2E tests for ${{ inputs.commit }} (${{ inputs.correlation_id }}) + +# Dispatched by `job_optional_e2e_tests` in build.yml. This is a separate workflow because GitHub +# does not count failed checks of a dispatched run in the PR status, so these failures do not mark +# the PR as failed. +on: + # Only here so that GitHub registers this workflow before it is on the default branch, which a + # dispatch needs. The job is skipped for this event. Remove this trigger after the merge. + pull_request: + workflow_dispatch: + inputs: + commit: + description: The commit that the tarballs were built from + required: true + build_run_id: + description: The build.yml run that uploaded the tarball artifacts + required: true + matrix: + description: The optional E2E test matrix as JSON + required: true + pr_number: + description: The PR that the run is for, if there is one + required: false + correlation_id: + description: Lets the dispatching job find this run + required: true + +# Like build.yml: a new push to a PR cancels the old run, other runs are never cancelled. +concurrency: + group: ${{ github.workflow }}-${{ inputs.pr_number || github.run_id }} + cancel-in-progress: true + +env: + HEAD_COMMIT: ${{ inputs.commit }} + + # See build.yml + TARBALL_ARTIFACT_DOWNLOAD_PATH: ${{ github.workspace }}/packages + DISABLE_V8_COMPILE_CACHE: '1' + +jobs: + job_optional_e2e_tests: + name: E2E ${{ matrix.label || matrix.test-application }} Test (optional) + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + # To download the artifacts of the build.yml run + actions: read + env: + E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }} + E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }} + # Needed because some apps expect a certain prefix + NEXT_PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }} + PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }} + REACT_APP_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }} + E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks' + E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests' + E2E_OPENROUTER_API_KEY: ${{ secrets.E2E_OPENROUTER_API_KEY }} + # Only this app talks to OpenAI directly (OpenRouter has no Decisions route), so only it gets the key. + E2E_OPENAI_DECISIONS_API_KEY: + ${{ matrix.test-application == 'node-openai-decisions-send-to-sentry' && secrets.E2E_OPENAI_DECISIONS_API_KEY || + '' }} + # Used by test apps that deploy a real Cloudflare Worker, e.g. cloudflare-workers-send-to-sentry + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + # Keeps the Worker name at `-pr-`, which the cleanup-e2e-workers workflow deletes + E2E_PR_NUMBER: ${{ inputs.pr_number }} + # The `runtime` of a runtime variant, see dev-packages/e2e-tests/README.md + RUNTIME: ${{ matrix.runtime }} + # Without the root node_modules, `pnpm run` would first install the whole workspace. + pnpm_config_verify_deps_before_run: 'false' + strategy: + fail-fast: false + # The fallback only keeps the expression valid for the skipped `pull_request` run. + matrix: ${{ fromJson(inputs.matrix || '{"include":[{}]}') }} + + steps: + - name: Check out commit ${{ env.HEAD_COMMIT }} + uses: actions/checkout@v7 + with: + ref: ${{ env.HEAD_COMMIT }} + - name: Set up Node + uses: actions/setup-node@v7 + with: + node-version-file: 'dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}/package.json' + - name: Set up Bun + if: + matrix.test-application == 'node-exports-test-app' || contains(matrix.test-application, 'bun') || + matrix.runtime == 'bun' + uses: oven-sh/setup-bun@v2 + with: + bun-version: '1.3.14' + - name: Set up Deno + if: contains(matrix.test-application, 'deno') || matrix.runtime == 'deno' + uses: denoland/setup-deno@v2.0.5 + with: + deno-version: ${{ matrix.deno-version || 'v2.8.3' }} + - name: Install pnpm + uses: ./.github/actions/install-pnpm + + - name: Restore tarball artifacts + uses: actions/download-artifact@v8 + with: + name: build-tarball-output + path: ${{ env.TARBALL_ARTIFACT_DOWNLOAD_PATH }} + run-id: ${{ inputs.build_run_id }} + github-token: ${{ github.token }} + + - name: Restore test-utils tarball artifact + uses: actions/download-artifact@v8 + with: + name: build-test-utils-tarball-output + path: dev-packages/test-utils + run-id: ${{ inputs.build_run_id }} + github-token: ${{ github.token }} + + - name: Prepare E2E tests + run: pnpm run test:prepare + working-directory: dev-packages/e2e-tests + + - name: Validate test setup + run: pnpm run test:validate + working-directory: dev-packages/e2e-tests + + - name: Copy to temp + run: + pnpm run ci:copy-to-temp ./test-applications/${{ matrix.test-application }} ${{ runner.temp + }}/test-application + working-directory: dev-packages/e2e-tests + + - name: Add pnpm overrides + run: + pnpm run ci:pnpm-overrides ${{ runner.temp }}/test-application ${{ github.workspace + }}/dev-packages/e2e-tests/packed + working-directory: dev-packages/e2e-tests + + # Installs Playwright while the app builds. Keep it right before the build: the runner sets up a background + # composite action's steps from the job's shared `github` context, which fails with "Collection was modified" if + # a foreground step starts at that moment. Starting next to `download-artifact` also made it fail to read the event file. + - name: Install Playwright + id: install-playwright + background: true + uses: ./.github/actions/install-playwright + with: + browsers: chromium + cwd: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }} + + - name: Build E2E app + working-directory: ${{ runner.temp }}/test-application + timeout-minutes: 7 + run: ${{ matrix.build-command || 'pnpm test:build' }} + + - wait: [install-playwright] + + - name: Run E2E test + working-directory: ${{ runner.temp }}/test-application + timeout-minutes: 10 + run: ${{ matrix.assert-command || 'pnpm test:assert' }} diff --git a/dev-packages/test-utils/src/cloudflare.ts b/dev-packages/test-utils/src/cloudflare.ts index 2e21a316d789..f50908f03dc4 100644 --- a/dev-packages/test-utils/src/cloudflare.ts +++ b/dev-packages/test-utils/src/cloudflare.ts @@ -36,9 +36,11 @@ function getWorkerName(workerPrefix: string): string { return `${workerPrefix}-local-${randomBytes(3).toString('hex')}`; } - const { GITHUB_EVENT_NAME, GITHUB_REF_NAME = '' } = process.env; + const { GITHUB_EVENT_NAME, GITHUB_REF_NAME = '', E2E_PR_NUMBER } = process.env; + // The optional E2E workflow is dispatched on the PR branch, so its ref has no PR number. const prNumber = - GITHUB_EVENT_NAME === 'pull_request' ? GITHUB_REF_NAME.split('/')[0] : /\/pr-(\d+)-/.exec(GITHUB_REF_NAME)?.[1]; + E2E_PR_NUMBER || + (GITHUB_EVENT_NAME === 'pull_request' ? GITHUB_REF_NAME.split('/')[0] : /\/pr-(\d+)-/.exec(GITHUB_REF_NAME)?.[1]); const ref = prNumber ? `pr-${prNumber}` : GITHUB_REF_NAME; // Worker names allow lowercase alphanumerics and dashes only, up to 63 characters. const slug = ref.toLowerCase().replace(/[^a-z0-9]+/g, '-');