Skip to content

ci: Run optional E2E tests in a separate workflow #25548

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
andreiborza wants to merge 1 commit into
base: develop
Choose a base branch
from
Draft

ci: Run optional E2E tests in a separate workflow #25548

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
222 changes: 120 additions & 102 deletions .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1185,8 +1185,10 @@ jobs:
retention-days: 7

# - We skip optional tests on release branches
# - The tests run in optional-e2e.yml, so their failures do not mark the PR as failed. This job only waits for
# them and reports failures in a PR comment.
job_optional_e2e_tests:
name: E2E ${{ matrix.label || matrix.test-application }} Test (optional)
name: E2E Tests (optional)
# We only run E2E tests for non-fork PRs because the E2E tests require secrets to work and they can't be accessed from forks
# We need to add the `always()` check here because the previous step has this as well :(
# See: https://github.com/actions/runner/issues/2205
Expand All @@ -1196,110 +1198,126 @@ jobs:
github.event.pull_request.head.repo.full_name == github.repository) && github.actor != 'dependabot[bot]'
needs: [job_get_metadata, job_build, job_build_tarballs]
runs-on: ubuntu-24.04
timeout-minutes: 15
timeout-minutes: 60
permissions:
actions: write
pull-requests: write
env:
E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
# Needed because some apps expect a certain prefix
NEXT_PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
REACT_APP_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests'
E2E_OPENROUTER_API_KEY: ${{ secrets.E2E_OPENROUTER_API_KEY }}
# Only this app talks to OpenAI directly (OpenRouter has no Decisions route), so only it gets the key.
E2E_OPENAI_DECISIONS_API_KEY:
${{ matrix.test-application == 'node-openai-decisions-send-to-sentry' && secrets.E2E_OPENAI_DECISIONS_API_KEY ||
'' }}
# Used by test apps that deploy a real Cloudflare Worker, e.g. cloudflare-workers-send-to-sentry
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
# The `runtime` of a runtime variant, see dev-packages/e2e-tests/README.md
RUNTIME: ${{ matrix.runtime }}
# Without the root node_modules, `pnpm run` would first install the whole workspace.
pnpm_config_verify_deps_before_run: 'false'
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.job_build.outputs.e2e-matrix-optional) }}

GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
CORRELATION_ID: ${{ github.run_id }}-${{ github.run_attempt }}
steps:
- name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
uses: actions/checkout@v7
with:
ref: ${{ env.HEAD_COMMIT }}
- name: Set up Node
uses: actions/setup-node@v7
with:
node-version-file: 'dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}/package.json'
- name: Set up Bun
if:
matrix.test-application == 'node-exports-test-app' || contains(matrix.test-application, 'bun') ||
matrix.runtime == 'bun'
uses: oven-sh/setup-bun@v2
with:
bun-version: '1.3.14'
- name: Set up Deno
if: contains(matrix.test-application, 'deno') || matrix.runtime == 'deno'
uses: denoland/setup-deno@v2.0.5
with:
deno-version: ${{ matrix.deno-version || 'v2.8.3' }}
- name: Install pnpm
uses: ./.github/actions/install-pnpm

- name: Restore tarball artifacts
uses: actions/download-artifact@v8
with:
name: build-tarball-output
path: ${{ env.TARBALL_ARTIFACT_DOWNLOAD_PATH }}

- name: Restore test-utils tarball artifact
uses: actions/download-artifact@v8
with:
name: build-test-utils-tarball-output
path: dev-packages/test-utils

- name: Prepare E2E tests
run: pnpm run test:prepare
working-directory: dev-packages/e2e-tests

- name: Validate test setup
run: pnpm run test:validate
working-directory: dev-packages/e2e-tests

- name: Copy to temp
run:
pnpm run ci:copy-to-temp ./test-applications/${{ matrix.test-application }} ${{ runner.temp
}}/test-application
working-directory: dev-packages/e2e-tests

- name: Add pnpm overrides
run:
pnpm run ci:pnpm-overrides ${{ runner.temp }}/test-application ${{ github.workspace
}}/dev-packages/e2e-tests/packed
working-directory: dev-packages/e2e-tests

# Installs Playwright while the app builds. Keep it right before the build: the runner sets up a background
# composite action's steps from the job's shared `github` context, which fails with "Collection was modified" if
# a foreground step starts at that moment. Starting next to `download-artifact` also made it fail to read the event file.
- name: Install Playwright
id: install-playwright
background: true
uses: ./.github/actions/install-playwright
with:
browsers: chromium
cwd: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}

- name: Build E2E app
working-directory: ${{ runner.temp }}/test-application
timeout-minutes: 7
run: ${{ matrix.build-command || 'pnpm test:build' }}

- wait: [install-playwright]
- name: Trigger optional E2E tests

@mydea mydea Oct 20, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no strong feeling, but maybe this could be a JS-based in-repo action (like size limit etc)? would maybe be a bit easier to read/adjust in the future?

env:
MATRIX: ${{ needs.job_build.outputs.e2e-matrix-optional }}
PR_NUMBER: ${{ github.event.pull_request.number }}
DISPATCH_REF: ${{ github.head_ref || github.ref_name }}
run: |
# A dispatch by file name only works when the file is on the default branch, a dispatch by ID always works.
workflow_id=$(gh api "repos/$GH_REPO/actions/workflows" --paginate \
--jq '.workflows[] | select(.path == ".github/workflows/optional-e2e.yml") | .id')
echo "WORKFLOW_ID=$workflow_id" >> "$GITHUB_ENV"

gh workflow run "$workflow_id" --ref "$DISPATCH_REF" \
-f commit="$HEAD_COMMIT" \
-f build_run_id="$GITHUB_RUN_ID" \
-f matrix="$MATRIX" \
-f pr_number="$PR_NUMBER" \
-f correlation_id="$CORRELATION_ID"

- name: Wait for optional E2E tests
id: wait
run: |
for _ in $(seq 1 30); do
run_id=$(gh run list --workflow "$WORKFLOW_ID" --event workflow_dispatch --limit 20 \
--json databaseId,displayTitle \
--jq ".[] | select(.displayTitle | contains(\"$CORRELATION_ID\")) | .databaseId" | head -1)
[ -n "$run_id" ] && break
sleep 2
done
[ -z "$run_id" ] && echo "::error::Optional E2E run not found" && exit 1
run_url="$GITHUB_SERVER_URL/$GH_REPO/actions/runs/$run_id"
echo "Optional E2E run: $run_url"

until [ "$(gh run view "$run_id" --json status --jq .status)" = completed ]; do
sleep 30
done

conclusion=$(gh run view "$run_id" --json conclusion --jq .conclusion)
echo "Optional E2E tests finished with: $conclusion"
[ "$conclusion" != success ] && echo "::warning::Optional E2E tests $conclusion: $run_url"
echo "run_id=$run_id" >> "$GITHUB_OUTPUT"
echo "url=$run_url" >> "$GITHUB_OUTPUT"
exit 0

# Keeps one comment per PR up to date, so new pushes do not add more comments. The comment
# is only created when a check fails.
- name: Report result in PR comment
if: github.event_name == 'pull_request'
env:
PR_NUMBER: ${{ github.event.pull_request.number }}
RUN_ID: ${{ steps.wait.outputs.run_id }}
RUN_URL: ${{ steps.wait.outputs.url }}
run: |
gh run view "$RUN_ID" --json jobs > jobs.json

rows() {
jq -r --arg filter "$1" '
def dur: ((.completedAt | fromdate) - (.startedAt | fromdate)) as $s
| if $s >= 60 then "\($s / 60 | floor)m" else "\($s)s" end;
def icon: {success: "✅", failure: "❌", cancelled: "⛔", timed_out: "⏱️"}[.conclusion] // "❔";
def status:
if .conclusion == "success" then "Successful in \(dur)"
elif .conclusion == "cancelled" then "Cancelled after \(dur)"
else "Failing after \(dur)" end;
.jobs[]
| select(.conclusion != "skipped")
| select(if $filter == "success" then .conclusion == "success" else .conclusion != "success" end)
| "| \(icon) | CI: Optional E2E Tests / \(.name) | \(status) | [Details](\(.url)) |"' jobs.json
}
failing=$(rows failing)
successful=$(rows success)
counts=$(jq -r '[.jobs[] | select(.conclusion != "skipped")]
| [(map(select(.conclusion != "success")) | length | select(. > 0) | "\(.) failing"),
(map(select(.conclusion == "success")) | length | select(. > 0) | "\(.) successful")]
| join(", ")' jobs.json)
header=$'| | Check | Status | |\n|---|---|---|---|'

marker='<!-- optional-e2e-report -->'
comment_id=$(gh api "repos/$GH_REPO/issues/$PR_NUMBER/comments" --paginate \
--jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | contains(\"$marker\"))) | .id" | tail -1)
if [ -z "$failing" ] && [ -z "$comment_id" ]; then
exit 0
fi

- name: Run E2E test
working-directory: ${{ runner.temp }}/test-application
timeout-minutes: 10
run: ${{ matrix.assert-command || 'pnpm test:assert' }}
{
echo "$marker"
echo '### Optional E2E tests'
echo
if [ -n "$failing" ]; then
echo "Some optional checks were not successful. They do not block the merge. $counts. [See the run]($RUN_URL)."
echo
echo "$header"
echo "$failing"
else
echo "All optional checks were successful. $counts. [See the run]($RUN_URL)."
fi
if [ -n "$successful" ]; then
echo
echo '<details><summary>Successful checks</summary>'
echo
echo "$header"
echo "$successful"
echo
echo '</details>'
fi
} > comment.md

if [ -n "$comment_id" ]; then
gh api -X PATCH "repos/$GH_REPO/issues/comments/$comment_id" -F body=@comment.md > /dev/null
else
gh pr comment "$PR_NUMBER" --body-file comment.md
fi

job_required_jobs_passed:
name: All required jobs passed or were skipped
Expand Down
Loading
Loading