From 6538ab6206bc630a8dc959605df55c17e59a21c8 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:59:00 +0000 Subject: [PATCH 01/11] ci(e2e): Run short E2E jobs on 2 vCPU runners 189 of the 231 required E2E jobs finish in about 42 s (p50) on the 4 vCPU runner, so they pay for cores they don't use. They now run on the small profile. The 20 apps whose jobs take longer than about 90 s set `sentryTest.runner` to keep the 4 vCPU profile. Co-Authored-By: Joshua Li <14209781+joshuarli@users.noreply.github.com> --- .github/workflows/build.yml | 5 +++-- dev-packages/e2e-tests/README.md | 4 ++++ dev-packages/e2e-tests/lib/getTestMatrix.mjs | 8 ++++---- .../test-applications/aws-serverless-layer/package.json | 1 + .../test-applications/browser-bfcache/package.json | 3 +++ .../test-applications/nextjs-15-basepath/package.json | 3 +++ .../test-applications/nextjs-15-intl/package.json | 3 +++ .../e2e-tests/test-applications/nextjs-15-t3/package.json | 3 +++ .../e2e-tests/test-applications/nextjs-15/package.json | 1 + .../nextjs-16-cacheComponents/package.json | 1 + .../test-applications/nextjs-16-static/package.json | 3 +++ .../nextjs-16-streaming-cacheComponents/package.json | 1 + .../test-applications/nextjs-16-streaming/package.json | 1 + .../test-applications/nextjs-16-tunnel/package.json | 1 + .../e2e-tests/test-applications/nextjs-16/package.json | 1 + .../test-applications/nextjs-app-dir/package.json | 1 + .../test-applications/nextjs-pages-dir/package.json | 1 + .../e2e-tests/test-applications/nuxt-3-min/package.json | 3 +++ .../test-applications/nuxt-4-static/package.json | 1 + .../e2e-tests/test-applications/nuxt-4/package.json | 1 + .../test-applications/react-router-6/package.json | 3 +++ .../react-router-7-lazy-routes/package.json | 3 +++ .../react-router-8-framework/package.json | 1 + 23 files changed, 47 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 23377c9c06cc..8e65611d4c50 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1083,9 +1083,10 @@ jobs: if: always() && needs.job_build_tarballs.result == 'success' && needs.job_build.outputs.e2e-matrix !='{"include":[]}' needs: [job_get_metadata, job_build, job_build_layer, job_build_tarballs] + # Most apps finish in under a minute on 4 vCPUs, so they run on 2. Apps that take longer set `sentryTest.runner`. runs-on: - namespace-profile-sentry-javascript;overrides.cache-tag=sentry-javascript-e2e${{ github.event_name == - 'pull_request' && '-pr' || '' }} + ${{ matrix.runner || 'namespace-profile-sentry-javascript-small' }};overrides.cache-tag=sentry-javascript-e2e${{ + github.event_name == 'pull_request' && '-pr' || '' }} timeout-minutes: 15 env: # We just use a dummy DSN here, only send to the tunnel anyhow diff --git a/dev-packages/e2e-tests/README.md b/dev-packages/e2e-tests/README.md index 971ba3057c10..d71ba9b1b8a3 100644 --- a/dev-packages/e2e-tests/README.md +++ b/dev-packages/e2e-tests/README.md @@ -280,6 +280,10 @@ For example, if something is changed in the browser package, only E2E test apps You can add additional information about the test (e.g. canary versions, optional in CI) by adding `sentryTest` in the `package.json` of a test application. +The required E2E jobs run on 2 vCPUs (`namespace-profile-sentry-javascript-small`). If an app's job takes longer than +about 90 seconds, set `"sentryTest": { "runner": "namespace-profile-sentry-javascript" }` to run it and its variants on +4 vCPUs. + If you add Sentry dependencies to your test application, set the dependency versions to `latest || *` in order for it to work with both regular and prerelease versions: diff --git a/dev-packages/e2e-tests/lib/getTestMatrix.mjs b/dev-packages/e2e-tests/lib/getTestMatrix.mjs index bae7c3262787..cec2cb530828 100644 --- a/dev-packages/e2e-tests/lib/getTestMatrix.mjs +++ b/dev-packages/e2e-tests/lib/getTestMatrix.mjs @@ -72,6 +72,8 @@ function addIncludesForTestApp(testApp, includes, { optionalMode }) { const shouldSkip = packageJson.sentryTest?.skip || false; const isOptional = packageJson.sentryTest?.optional || false; const variants = (optionalMode ? packageJson.sentryTest?.optionalVariants : packageJson.sentryTest?.variants) || []; + const runner = packageJson.sentryTest?.runner; + const baseInclude = { 'test-application': testApp, ...(runner ? { runner } : {}) }; if (shouldSkip) { return; @@ -79,9 +81,7 @@ function addIncludesForTestApp(testApp, includes, { optionalMode }) { // Add the basic test-application itself, if it is in the current mode if (optionalMode === isOptional) { - includes.push({ - 'test-application': testApp, - }); + includes.push(baseInclude); } variants.forEach(variant => { @@ -94,7 +94,7 @@ function addIncludesForTestApp(testApp, includes, { optionalMode }) { // Don't leak the `skip` flag into the matrix include. const { skip: _skip, ...variantConfig } = variant; includes.push({ - 'test-application': testApp, + ...baseInclude, ...variantConfig, }); }); diff --git a/dev-packages/e2e-tests/test-applications/aws-serverless-layer/package.json b/dev-packages/e2e-tests/test-applications/aws-serverless-layer/package.json index 66acf6b2afeb..f24cd95e12a0 100644 --- a/dev-packages/e2e-tests/test-applications/aws-serverless-layer/package.json +++ b/dev-packages/e2e-tests/test-applications/aws-serverless-layer/package.json @@ -24,6 +24,7 @@ "extends": "../../package.json" }, "sentryTest": { + "runner": "namespace-profile-sentry-javascript", "variants": [ { "build-command": "NODE_VERSION=22 pnpm test:build", diff --git a/dev-packages/e2e-tests/test-applications/browser-bfcache/package.json b/dev-packages/e2e-tests/test-applications/browser-bfcache/package.json index 1db45bd6f42e..e1c004757888 100644 --- a/dev-packages/e2e-tests/test-applications/browser-bfcache/package.json +++ b/dev-packages/e2e-tests/test-applications/browser-bfcache/package.json @@ -23,5 +23,8 @@ }, "volta": { "extends": "../../package.json" + }, + "sentryTest": { + "runner": "namespace-profile-sentry-javascript" } } diff --git a/dev-packages/e2e-tests/test-applications/nextjs-15-basepath/package.json b/dev-packages/e2e-tests/test-applications/nextjs-15-basepath/package.json index 318d05a6185f..379b4038a5e9 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-15-basepath/package.json +++ b/dev-packages/e2e-tests/test-applications/nextjs-15-basepath/package.json @@ -26,5 +26,8 @@ }, "volta": { "extends": "../../package.json" + }, + "sentryTest": { + "runner": "namespace-profile-sentry-javascript" } } diff --git a/dev-packages/e2e-tests/test-applications/nextjs-15-intl/package.json b/dev-packages/e2e-tests/test-applications/nextjs-15-intl/package.json index 998a4976ad3a..15294b246cbc 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-15-intl/package.json +++ b/dev-packages/e2e-tests/test-applications/nextjs-15-intl/package.json @@ -27,5 +27,8 @@ }, "volta": { "extends": "../../package.json" + }, + "sentryTest": { + "runner": "namespace-profile-sentry-javascript" } } diff --git a/dev-packages/e2e-tests/test-applications/nextjs-15-t3/package.json b/dev-packages/e2e-tests/test-applications/nextjs-15-t3/package.json index db01367495e8..2bbf3cdb8dcb 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-15-t3/package.json +++ b/dev-packages/e2e-tests/test-applications/nextjs-15-t3/package.json @@ -46,5 +46,8 @@ }, "volta": { "extends": "../../package.json" + }, + "sentryTest": { + "runner": "namespace-profile-sentry-javascript" } } diff --git a/dev-packages/e2e-tests/test-applications/nextjs-15/package.json b/dev-packages/e2e-tests/test-applications/nextjs-15/package.json index 30da1c0e8747..7f18b33c2879 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-15/package.json +++ b/dev-packages/e2e-tests/test-applications/nextjs-15/package.json @@ -34,6 +34,7 @@ "extends": "../../package.json" }, "sentryTest": { + "runner": "namespace-profile-sentry-javascript", "optionalVariants": [ { "build-command": "pnpm test:build-latest", diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cacheComponents/package.json b/dev-packages/e2e-tests/test-applications/nextjs-16-cacheComponents/package.json index 855491173d4f..e90155de6c2c 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cacheComponents/package.json +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cacheComponents/package.json @@ -46,6 +46,7 @@ "extends": "../../package.json" }, "sentryTest": { + "runner": "namespace-profile-sentry-javascript", "variants": [ { "build-command": "pnpm test:build-webpack", diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-static/package.json b/dev-packages/e2e-tests/test-applications/nextjs-16-static/package.json index 50f57e8ca472..b2b626143a71 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-static/package.json +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-static/package.json @@ -42,5 +42,8 @@ }, "volta": { "extends": "../../package.json" + }, + "sentryTest": { + "runner": "namespace-profile-sentry-javascript" } } diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-streaming-cacheComponents/package.json b/dev-packages/e2e-tests/test-applications/nextjs-16-streaming-cacheComponents/package.json index 3e6d5d60e233..231afe799973 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-streaming-cacheComponents/package.json +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-streaming-cacheComponents/package.json @@ -46,6 +46,7 @@ "extends": "../../package.json" }, "sentryTest": { + "runner": "namespace-profile-sentry-javascript", "variants": [ { "build-command": "pnpm test:build-webpack", diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-streaming/package.json b/dev-packages/e2e-tests/test-applications/nextjs-16-streaming/package.json index 39da00835495..af35c23fcc8d 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-streaming/package.json +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-streaming/package.json @@ -44,6 +44,7 @@ "extends": "../../package.json" }, "sentryTest": { + "runner": "namespace-profile-sentry-javascript", "variants": [ { "build-command": "pnpm test:build-webpack", diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-tunnel/package.json b/dev-packages/e2e-tests/test-applications/nextjs-16-tunnel/package.json index adbc49ec455c..53c5adcc5761 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-tunnel/package.json +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-tunnel/package.json @@ -54,6 +54,7 @@ "extends": "../../package.json" }, "sentryTest": { + "runner": "namespace-profile-sentry-javascript", "variants": [ { "build-command": "pnpm test:build-webpack", diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/package.json b/dev-packages/e2e-tests/test-applications/nextjs-16/package.json index 25ec54f931d0..34b8d7f62dc9 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/package.json +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/package.json @@ -67,6 +67,7 @@ "extends": "../../package.json" }, "sentryTest": { + "runner": "namespace-profile-sentry-javascript", "variants": [ { "build-command": "pnpm test:build-webpack", diff --git a/dev-packages/e2e-tests/test-applications/nextjs-app-dir/package.json b/dev-packages/e2e-tests/test-applications/nextjs-app-dir/package.json index 43cea1078cf2..1fd3800e0d2e 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-app-dir/package.json +++ b/dev-packages/e2e-tests/test-applications/nextjs-app-dir/package.json @@ -32,6 +32,7 @@ "extends": "../../package.json" }, "sentryTest": { + "runner": "namespace-profile-sentry-javascript", "variants": [ { "build-command": "pnpm test:build-15", diff --git a/dev-packages/e2e-tests/test-applications/nextjs-pages-dir/package.json b/dev-packages/e2e-tests/test-applications/nextjs-pages-dir/package.json index fb3aeaf3399f..44591f3a2ec0 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-pages-dir/package.json +++ b/dev-packages/e2e-tests/test-applications/nextjs-pages-dir/package.json @@ -33,6 +33,7 @@ "extends": "../../package.json" }, "sentryTest": { + "runner": "namespace-profile-sentry-javascript", "variants": [ { "build-command": "pnpm test:build-15", diff --git a/dev-packages/e2e-tests/test-applications/nuxt-3-min/package.json b/dev-packages/e2e-tests/test-applications/nuxt-3-min/package.json index b5299208a860..06fe7b58519a 100644 --- a/dev-packages/e2e-tests/test-applications/nuxt-3-min/package.json +++ b/dev-packages/e2e-tests/test-applications/nuxt-3-min/package.json @@ -27,5 +27,8 @@ }, "volta": { "extends": "../../package.json" + }, + "sentryTest": { + "runner": "namespace-profile-sentry-javascript" } } diff --git a/dev-packages/e2e-tests/test-applications/nuxt-4-static/package.json b/dev-packages/e2e-tests/test-applications/nuxt-4-static/package.json index ee355572b388..798eef35e989 100644 --- a/dev-packages/e2e-tests/test-applications/nuxt-4-static/package.json +++ b/dev-packages/e2e-tests/test-applications/nuxt-4-static/package.json @@ -33,6 +33,7 @@ "extends": "../../package.json" }, "sentryTest": { + "runner": "namespace-profile-sentry-javascript", "variants": [ { "build-command": "E2E_TEST_OTEL_SETUP=true pnpm test:build", diff --git a/dev-packages/e2e-tests/test-applications/nuxt-4/package.json b/dev-packages/e2e-tests/test-applications/nuxt-4/package.json index 92fa2a41d87b..bf2941ed1992 100644 --- a/dev-packages/e2e-tests/test-applications/nuxt-4/package.json +++ b/dev-packages/e2e-tests/test-applications/nuxt-4/package.json @@ -35,6 +35,7 @@ "extends": "../../package.json" }, "sentryTest": { + "runner": "namespace-profile-sentry-javascript", "variants": [ { "build-command": "E2E_TEST_OTEL_SETUP=true pnpm test:build", diff --git a/dev-packages/e2e-tests/test-applications/react-router-6/package.json b/dev-packages/e2e-tests/test-applications/react-router-6/package.json index 2ee798930dc4..dd022fa72381 100644 --- a/dev-packages/e2e-tests/test-applications/react-router-6/package.json +++ b/dev-packages/e2e-tests/test-applications/react-router-6/package.json @@ -50,5 +50,8 @@ }, "volta": { "extends": "../../package.json" + }, + "sentryTest": { + "runner": "namespace-profile-sentry-javascript" } } diff --git a/dev-packages/e2e-tests/test-applications/react-router-7-lazy-routes/package.json b/dev-packages/e2e-tests/test-applications/react-router-7-lazy-routes/package.json index 561d8a362f29..605ff3df45a9 100644 --- a/dev-packages/e2e-tests/test-applications/react-router-7-lazy-routes/package.json +++ b/dev-packages/e2e-tests/test-applications/react-router-7-lazy-routes/package.json @@ -48,5 +48,8 @@ }, "volta": { "extends": "../../package.json" + }, + "sentryTest": { + "runner": "namespace-profile-sentry-javascript" } } diff --git a/dev-packages/e2e-tests/test-applications/react-router-8-framework/package.json b/dev-packages/e2e-tests/test-applications/react-router-8-framework/package.json index de5503c7626a..bc4dc0092188 100644 --- a/dev-packages/e2e-tests/test-applications/react-router-8-framework/package.json +++ b/dev-packages/e2e-tests/test-applications/react-router-8-framework/package.json @@ -69,6 +69,7 @@ "extends": "../../package.json" }, "sentryTest": { + "runner": "namespace-profile-sentry-javascript", "variants": [ { "build-command": "pnpm test:build-latest", From 71a2ea7174458167914821ab16e0d0e386157647 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:28:27 -0700 Subject: [PATCH 02/11] ci: Cut CI cost and fix the most common flakes --- .../actions/install-dependencies/action.yml | 1 + .github/actions/install-playwright/action.yml | 7 ++ .github/workflows/build.yml | 101 ++++++------------ .../workflows/enforce-license-compliance.yml | 6 ++ .../aws-serverless-layer/tests/layer.test.ts | 5 +- .../scripts/use-ts-5_0.js | 16 --- .../src/coreHandlers/handleClick.ts | 9 +- .../unit/coreHandlers/handleClick.test.ts | 32 ++++++ 8 files changed, 84 insertions(+), 93 deletions(-) delete mode 100644 dev-packages/node-integration-tests/scripts/use-ts-5_0.js diff --git a/.github/actions/install-dependencies/action.yml b/.github/actions/install-dependencies/action.yml index 27971a484f63..2a4b331fd3f6 100644 --- a/.github/actions/install-dependencies/action.yml +++ b/.github/actions/install-dependencies/action.yml @@ -11,6 +11,7 @@ runs: echo 'paths<> "$GITHUB_OUTPUT" shell: bash diff --git a/.github/actions/install-playwright/action.yml b/.github/actions/install-playwright/action.yml index bd84a13e9107..378e77666093 100644 --- a/.github/actions/install-playwright/action.yml +++ b/.github/actions/install-playwright/action.yml @@ -34,6 +34,13 @@ runs: playwright apt + # `--with-deps` names packages that the runner image already has (glib, mesa, nss, ...), and `apt-get install` would + # upgrade them to the latest Ubuntu update. That is half of the download on a cold cache, and the image versions work. + - name: Keep installed apt packages + if: steps.playwright-version.outputs.version != '' + run: echo 'APT::Get::Upgrade "false";' | sudo tee /etc/apt/apt.conf.d/99-no-upgrade > /dev/null + shell: bash + # Downloads only the browsers and apt packages that are not on the cache volume yet. `npx` runs outside the repo, # so it can't pick up a `playwright` that a parallel `pnpm install` is still linking. - name: Install Playwright diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8e65611d4c50..d59dc8f590a4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -343,6 +343,14 @@ jobs: run: pnpm run lint:es-compatibility - name: Lint types run: pnpm run lint:types + - name: Check for circular dependencies + run: pnpm run circularDepCheck + - name: Check for dts files that reference stuff in the temporary build folder + run: | + if grep -r --include "*.d.ts" --exclude-dir ".nx" 'import("@sentry(-internal)?/[^/]*/build' .; then + echo "Found illegal TypeScript import statement." + exit 1 + fi - name: Check Playwright versions run: | version=$(node -p "require('./dev-packages/test-utils/package.json').devDependencies['@playwright/test']") @@ -406,27 +414,6 @@ jobs: - name: Check file formatting run: pnpm run format:check - job_circular_dep_check: - name: Circular Dependency Check - needs: [job_get_metadata, job_build] - timeout-minutes: 10 - runs-on: - namespace-profile-sentry-javascript;overrides.cache-tag=sentry-javascript${{ github.event_name == 'pull_request' - && '-pr' || '' }} - steps: - - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) - uses: actions/checkout@v7 - with: - ref: ${{ env.HEAD_COMMIT }} - - name: Set up Node - uses: actions/setup-node@v7 - with: - node-version-file: 'package.json' - - name: Restore caches - uses: ./.github/actions/restore-cache - - name: Run madge - run: pnpm run circularDepCheck - job_artifacts: name: Upload Artifacts needs: [job_get_metadata, job_build, job_build_layer, job_build_bundles, job_build_tarballs] @@ -772,35 +759,8 @@ jobs: overwrite: true retention-days: 7 - job_check_for_faulty_dts: - name: Check for faulty .d.ts files - needs: [job_get_metadata, job_build] - runs-on: - namespace-profile-sentry-javascript;overrides.cache-tag=sentry-javascript${{ github.event_name == 'pull_request' - && '-pr' || '' }} - timeout-minutes: 5 - steps: - - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) - uses: actions/checkout@v7 - with: - ref: ${{ env.HEAD_COMMIT }} - - name: Set up Node - uses: actions/setup-node@v7 - with: - node-version-file: 'package.json' - - name: Restore caches - uses: ./.github/actions/restore-cache - - name: Check for dts files that reference stuff in the temporary build folder - run: | - if grep -r --include "*.d.ts" --exclude-dir ".nx" 'import("@sentry(-internal)?/[^/]*/build' .; then - echo "Found illegal TypeScript import statement." - exit 1 - fi - job_node_integration_tests: - name: - Node (${{ matrix.node }})${{ (matrix.typescript && format(' (TS {0})', matrix.typescript)) || '' }} Integration - Tests (${{ matrix.shard }}/3) + name: Node (${{ matrix.node }}) Integration Tests (${{ matrix.shard }}/3) needs: [job_get_metadata, job_build] if: needs.job_build.outputs.changed_node_integration == 'true' || github.event_name != 'pull_request' runs-on: @@ -812,19 +772,6 @@ jobs: matrix: node: [20.19, 22, 24, 26] shard: [1, 2, 3] - typescript: - - false - include: - # Only check typescript for latest version (to streamline CI) - - node: 24 - typescript: '5.0' - shard: 1 - - node: 24 - typescript: '5.0' - shard: 2 - - node: 24 - typescript: '5.0' - shard: 3 steps: - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) uses: actions/checkout@v7 @@ -850,11 +797,6 @@ jobs: - name: Restore caches uses: ./.github/actions/restore-cache - - name: Overwrite typescript version - if: matrix.typescript == '5.0' - run: node ./scripts/use-ts-5_0.js - working-directory: dev-packages/node-integration-tests - - name: Run integration tests working-directory: dev-packages/node-integration-tests run: pnpm run test --shard=${{ matrix.shard }}/3 @@ -1116,9 +1058,10 @@ jobs: runtime: ${{ matrix.runtime }} deno-version: ${{ matrix.deno-version }} - # Installs Playwright while the app builds. Keep it right before the build: the runner sets up a background - # composite action's steps from the job's shared `github` context, which fails with "Collection was modified" if - # a foreground step starts at that moment. Starting next to `download-artifact` also made it fail to read the event file. + # Installs Playwright while the app installs and builds. Keep it right before the install: the runner sets up a + # background composite action's steps from the job's shared `github` context, which fails with "Collection was + # modified" if a foreground step starts at that moment. Starting next to `download-artifact` also made it fail to + # read the event file. - name: Install Playwright id: install-playwright background: true @@ -1127,6 +1070,23 @@ jobs: browsers: chromium cwd: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }} + # Every build command runs `pnpm install`, so installing first makes that a no-op. Some downloads from the npm + # registry crawl at tens of kB/s for the whole 7 minutes of the build step, and pnpm does not give up on a slow + # download. Another attempt reuses what is already in the store and opens new connections. + # The hoisted variant installs with another node linker, so installing first would only redo the install. + - name: Install E2E app dependencies + if: ${{ !contains(matrix.build-command, 'hoisted') }} + working-directory: ${{ runner.temp }}/test-application + timeout-minutes: 7 + run: | + for attempt in 1 2; do + timeout 150 pnpm install && exit 0 + echo "::warning::pnpm install attempt $attempt failed or did not finish in 150 s, retrying" + done + pnpm install + env: + SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }} + - name: Build E2E app working-directory: ${{ runner.temp }}/test-application timeout-minutes: 7 @@ -1217,7 +1177,6 @@ jobs: job_lint, job_check_lockfile, job_check_format, - job_circular_dep_check, job_size_check, ] # Always run this, even if a dependent job failed diff --git a/.github/workflows/enforce-license-compliance.yml b/.github/workflows/enforce-license-compliance.yml index e748668b92db..5d44ccb3497e 100644 --- a/.github/workflows/enforce-license-compliance.yml +++ b/.github/workflows/enforce-license-compliance.yml @@ -17,6 +17,12 @@ on: - v9 - v8 +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: enforce-license-compliance: runs-on: namespace-profile-sentry-javascript diff --git a/dev-packages/e2e-tests/test-applications/aws-serverless-layer/tests/layer.test.ts b/dev-packages/e2e-tests/test-applications/aws-serverless-layer/tests/layer.test.ts index 538b533967e0..8e47ac4255a4 100644 --- a/dev-packages/e2e-tests/test-applications/aws-serverless-layer/tests/layer.test.ts +++ b/dev-packages/e2e-tests/test-applications/aws-serverless-layer/tests/layer.test.ts @@ -337,7 +337,8 @@ test.describe('Lambda layer', () => { const transactionEvent = await transactionEventPromise; expect(transactionEvent.contexts?.trace?.status).toBe('ok'); - expect(transactionEvent.timestamp! - transactionEvent.start_timestamp!).toBeGreaterThanOrEqual(0.2); + // Node timers can fire up to a millisecond early, so the 200 ms callback delay can measure as 199.x ms + expect(transactionEvent.timestamp! - transactionEvent.start_timestamp!).toBeGreaterThanOrEqual(0.19); }); test('sets error status when a callback-style handler calls back with an error after it returned', async ({ @@ -360,7 +361,7 @@ test.describe('Lambda layer', () => { const [errorEvent, transactionEvent] = await Promise.all([errorEventPromise, transactionEventPromise]); expect(transactionEvent.contexts?.trace?.status).toBe('internal_error'); - expect(transactionEvent.timestamp! - transactionEvent.start_timestamp!).toBeGreaterThanOrEqual(0.2); + expect(transactionEvent.timestamp! - transactionEvent.start_timestamp!).toBeGreaterThanOrEqual(0.19); expect(errorEvent.contexts?.trace?.trace_id).toBe(transactionEvent.contexts?.trace?.trace_id); }); diff --git a/dev-packages/node-integration-tests/scripts/use-ts-5_0.js b/dev-packages/node-integration-tests/scripts/use-ts-5_0.js deleted file mode 100644 index 281bb4e9dcb7..000000000000 --- a/dev-packages/node-integration-tests/scripts/use-ts-5_0.js +++ /dev/null @@ -1,16 +0,0 @@ -/* eslint-disable no-console */ -const { execSync } = require('child_process'); -const { join } = require('path'); - -const cwd = join(__dirname, '../../..'); - -const tsVersion = '5.0.4'; - -console.log(`Installing typescript@${tsVersion}, and @types/node@18...`); - -execSync(`pnpm add --save-dev --workspace-root typescript@${tsVersion} @types/node@^18`, { - stdio: 'inherit', - cwd, -}); - -console.log('TypeScript version updated successfully.'); diff --git a/packages/replay-internal/src/coreHandlers/handleClick.ts b/packages/replay-internal/src/coreHandlers/handleClick.ts index f4576c3a7ca7..c61278494646 100644 --- a/packages/replay-internal/src/coreHandlers/handleClick.ts +++ b/packages/replay-internal/src/coreHandlers/handleClick.ts @@ -183,10 +183,10 @@ export class ClickDetector implements ReplayClickDetector { const now = nowInSeconds(); this._clicks.forEach(click => { - if (!click.mutationAfter && this._lastMutation) { + if (click.mutationAfter === undefined && this._lastMutation) { click.mutationAfter = click.timestamp <= this._lastMutation ? this._lastMutation - click.timestamp : undefined; } - if (!click.scrollAfter && this._lastScroll) { + if (click.scrollAfter === undefined && this._lastScroll) { click.scrollAfter = click.timestamp <= this._lastScroll ? this._lastScroll - click.timestamp : undefined; } @@ -215,8 +215,9 @@ export class ClickDetector implements ReplayClickDetector { /** Generate matching breadcrumb(s) for the click. */ private _generateBreadcrumbs(click: Click): void { const replay = this._replay; - const hadScroll = click.scrollAfter && click.scrollAfter <= this._scrollTimeout; - const hadMutation = click.mutationAfter && click.mutationAfter <= this._threshold; + // A mutation or scroll in the same millisecond as the click gives `0`, which still counts + const hadScroll = click.scrollAfter !== undefined && click.scrollAfter <= this._scrollTimeout; + const hadMutation = click.mutationAfter !== undefined && click.mutationAfter <= this._threshold; const isSlowClick = !hadScroll && !hadMutation; const { clickCount, clickBreadcrumb } = click; diff --git a/packages/replay-internal/test/unit/coreHandlers/handleClick.test.ts b/packages/replay-internal/test/unit/coreHandlers/handleClick.test.ts index 66beddf6ab19..7a51c5ca3e5b 100644 --- a/packages/replay-internal/test/unit/coreHandlers/handleClick.test.ts +++ b/packages/replay-internal/test/unit/coreHandlers/handleClick.test.ts @@ -321,6 +321,22 @@ describe('Unit | coreHandlers | handleClick', () => { expect(mockAddBreadcrumbEvent).toHaveBeenCalledTimes(0); }); + test('it does not consider clicks with a mutation in the same millisecond as slow click', async () => { + const breadcrumb: Breadcrumb = { + timestamp: BASE_TIMESTAMP / 1000, + data: { + nodeId: 1, + }, + }; + const node = document.createElement('button'); + detector.handleClick(breadcrumb, node); + detector.registerMutation(BASE_TIMESTAMP); + + vi.advanceTimersByTime(3_500); + + expect(mockAddBreadcrumbEvent).toHaveBeenCalledTimes(0); + }); + test('it considers clicks with mutation after threshold as slow click', async () => { const breadcrumb: Breadcrumb = { timestamp: BASE_TIMESTAMP / 1000, @@ -429,6 +445,22 @@ describe('Unit | coreHandlers | handleClick', () => { ); }); + test('it does not consider clicks with a scroll in the same millisecond as slow click', async () => { + const breadcrumb: Breadcrumb = { + timestamp: BASE_TIMESTAMP / 1000, + data: { + nodeId: 1, + }, + }; + const node = document.createElement('button'); + detector.handleClick(breadcrumb, node); + detector.registerScroll(BASE_TIMESTAMP); + + vi.advanceTimersByTime(3_500); + + expect(mockAddBreadcrumbEvent).toHaveBeenCalledTimes(0); + }); + test('it does not consider clicks with scroll before threshold as slow click', async () => { const breadcrumb: Breadcrumb = { timestamp: BASE_TIMESTAMP / 1000, From e905c5127b343e1d6aafdc875d21fb32dcb7d34c Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:55:26 +0000 Subject: [PATCH 03/11] ci: Retry flaky Playwright tests, batch short E2E apps and warm the E2E caches Every sampled first-attempt CI run failed at least one job, and CI never retried a single test. Playwright now retries a failed test once in CI. A test that passes on retry no longer fails the job, but it is listed in the job summary, and the weekly flaky test report now also counts it, so flakes stay visible. The flaky test detector keeps `--retries=0`. The required E2E apps on the 2 vCPU runner now run four per job, one after another, which cuts their jobs from 189 to 48. Setup, Playwright and the runner queue are paid once per four apps. Every app and variant still runs, a failed app does not stop the others, and the job fails if any app failed. Apps that set `sentryTest.runner` keep a job of their own. The E2E jobs share one cache volume, where the last job to finish overwrites what the others saved. A nightly job installs every test app into the pnpm store on both E2E cache tags and prunes packages no app uses, so later jobs fork a complete store that stays under the size limit. Co-Authored-By: Joshua Li <14209781+joshuarli@users.noreply.github.com> --- .github/actions/build-e2e-app/action.yml | 68 +++++++ .github/actions/prepare-e2e-app/action.yml | 59 ++++++ .github/actions/prepare-e2e-test/action.yml | 95 ++-------- .github/actions/setup-e2e-tests/action.yml | 77 ++++++++ .github/actions/test-e2e-app/action.yml | 43 +++++ .github/workflows/build.yml | 178 +++++++++++++----- .github/workflows/warm-e2e-cache.yml | 99 ++++++++++ .../browser-integration-tests/.gitignore | 1 + .../playwright.config.ts | 8 +- .../scripts/detectFlakyTests.ts | 2 +- dev-packages/e2e-tests/README.md | 10 +- .../e2e-tests/lib/checkJobResults.mjs | 12 ++ dev-packages/e2e-tests/lib/getTestMatrix.mjs | 69 ++++++- dev-packages/e2e-tests/lib/warmCache.mjs | 89 +++++++++ .../test-utils/src/playwright-config.ts | 8 +- scripts/ci-report-flaky-tests.mjs | 85 +++++++++ scripts/ci-report-flaky-tests.test.ts | 89 +++++++++ scripts/weekly-flaky-tests.mjs | 70 ++++--- scripts/weekly-flaky-tests.test.ts | 107 +++++++++++ 19 files changed, 1004 insertions(+), 165 deletions(-) create mode 100644 .github/actions/build-e2e-app/action.yml create mode 100644 .github/actions/prepare-e2e-app/action.yml create mode 100644 .github/actions/setup-e2e-tests/action.yml create mode 100644 .github/actions/test-e2e-app/action.yml create mode 100644 .github/workflows/warm-e2e-cache.yml create mode 100644 dev-packages/e2e-tests/lib/checkJobResults.mjs create mode 100644 dev-packages/e2e-tests/lib/warmCache.mjs create mode 100644 scripts/ci-report-flaky-tests.mjs create mode 100644 scripts/ci-report-flaky-tests.test.ts create mode 100644 scripts/weekly-flaky-tests.test.ts diff --git a/.github/actions/build-e2e-app/action.yml b/.github/actions/build-e2e-app/action.yml new file mode 100644 index 000000000000..db5b4124391e --- /dev/null +++ b/.github/actions/build-e2e-app/action.yml @@ -0,0 +1,68 @@ +name: 'Build E2E test application' +description: + 'Prepares, installs and builds one application of an E2E job in `runner.temp/e2e/`. Takes an entry of the `apps` + list that `ci:build-matrix --batch-size` generates.' +inputs: + app: + description: + 'The app as JSON: `test-application`, `dir` and the optional `runtime`, `deno-version`, `build-command`.' + required: true + clean-up: + description: 'Whether to stop the servers and containers that an earlier app of the job left behind.' + default: 'false' + +runs: + using: 'composite' + steps: + # Playwright stops the web servers of an app and the apps stop their containers in their global teardown. This only + # matters when an app timed out before that, and its servers would block the ports of the next app. + - name: Stop leftovers of the previous app + if: inputs.clean-up == 'true' + shell: bash + run: | + for proc in /proc/[0-9]*; do + if [[ "$(readlink "$proc/cwd" 2>/dev/null)" == "$RUNNER_TEMP/e2e/"* ]]; then + echo "Stopping $(tr '\0' ' ' < "$proc/cmdline" 2>/dev/null)" + kill "${proc#/proc/}" 2>/dev/null || true + fi + done + containers=$(docker ps -aq 2>/dev/null || true) + if [ -n "$containers" ]; then + echo "Removing containers: $containers" + docker rm -f $containers > /dev/null || true + fi + + - name: Prepare E2E test application + uses: ./.github/actions/prepare-e2e-app + with: + test-application: ${{ fromJSON(inputs.app)['test-application'] }} + dir: ${{ runner.temp }}/e2e/${{ fromJSON(inputs.app).dir }} + runtime: ${{ fromJSON(inputs.app).runtime }} + deno-version: ${{ fromJSON(inputs.app)['deno-version'] }} + + # Every build command runs `pnpm install`, so installing first makes that a no-op. Some downloads from the npm + # registry crawl at tens of kB/s for the whole 7 minutes of the build step, and pnpm does not give up on a slow + # download. Another attempt reuses what is already in the store and opens new connections. + # The hoisted variant installs with another node linker, so installing first would only redo the install. + - name: Install E2E app dependencies + if: ${{ !contains(fromJSON(inputs.app)['build-command'], 'hoisted') }} + shell: bash + working-directory: ${{ runner.temp }}/e2e/${{ fromJSON(inputs.app).dir }} + run: | + for attempt in 1 2; do + timeout 150 pnpm install && exit 0 + echo "::warning::pnpm install attempt $attempt failed or did not finish in 150 s, retrying" + done + timeout 300 pnpm install + env: + SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }} + RUNTIME: ${{ fromJSON(inputs.app).runtime }} + + - name: Build E2E app + shell: bash + working-directory: ${{ runner.temp }}/e2e/${{ fromJSON(inputs.app).dir }} + run: timeout 420 bash -c "$BUILD_COMMAND" + env: + BUILD_COMMAND: ${{ fromJSON(inputs.app)['build-command'] || 'pnpm test:build' }} + SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }} + RUNTIME: ${{ fromJSON(inputs.app).runtime }} diff --git a/.github/actions/prepare-e2e-app/action.yml b/.github/actions/prepare-e2e-app/action.yml new file mode 100644 index 000000000000..bfb7d4b5b67e --- /dev/null +++ b/.github/actions/prepare-e2e-app/action.yml @@ -0,0 +1,59 @@ +name: 'Prepare E2E test application' +description: + 'Sets up the runtimes for an E2E test application and copies it to `dir` with pnpm overrides pointing at the tarballs. + Run `setup-e2e-tests` first.' +inputs: + test-application: + description: 'The name of the test application in dev-packages/e2e-tests/test-applications.' + required: true + dir: + description: 'The directory to copy the application to.' + required: true + runtime: + description: 'The `runtime` of a runtime variant, see dev-packages/e2e-tests/README.md.' + default: '' + deno-version: + description: 'The Deno version to install, if the application needs Deno.' + default: 'v2.8.3' + +runs: + using: 'composite' + steps: + - name: Set up Node + uses: actions/setup-node@v7 + with: + node-version-file: 'dev-packages/e2e-tests/test-applications/${{ inputs.test-application }}/package.json' + - name: Set up Bun + if: + inputs.test-application == 'node-exports-test-app' || contains(inputs.test-application, 'bun') || inputs.runtime + == 'bun' + uses: oven-sh/setup-bun@v2 + with: + bun-version: '1.3.14' + - name: Set up AWS SAM + if: contains(inputs.test-application, 'aws-serverless') + uses: aws-actions/setup-sam@v3 + with: + use-installer: true + token: ${{ github.token }} + - name: Set up Deno + if: contains(inputs.test-application, 'deno') || inputs.runtime == 'deno' + uses: denoland/setup-deno@v2.0.5 + with: + deno-version: ${{ inputs.deno-version || 'v2.8.3' }} + + - name: Copy to temp + run: pnpm run ci:copy-to-temp "./test-applications/$TEST_APPLICATION" "$APP_DIR" + shell: bash + working-directory: dev-packages/e2e-tests + env: + TEST_APPLICATION: ${{ inputs.test-application }} + APP_DIR: ${{ inputs.dir }} + RUNTIME: ${{ inputs.runtime }} + + - name: Add pnpm overrides + run: pnpm run ci:pnpm-overrides "$APP_DIR" "$GITHUB_WORKSPACE/dev-packages/e2e-tests/packed" + shell: bash + working-directory: dev-packages/e2e-tests + env: + APP_DIR: ${{ inputs.dir }} diff --git a/.github/actions/prepare-e2e-test/action.yml b/.github/actions/prepare-e2e-test/action.yml index 3f5b37b41506..c71bce363237 100644 --- a/.github/actions/prepare-e2e-test/action.yml +++ b/.github/actions/prepare-e2e-test/action.yml @@ -1,7 +1,7 @@ name: 'Prepare E2E test application' description: 'Sets up the runtimes for an E2E test application, restores the built tarballs and copies the application to - `runner.temp/test-application` with pnpm overrides pointing at the tarballs.' + `runner.temp/test-application` with pnpm overrides pointing at the tarballs. For a job that runs one application.' inputs: test-application: description: 'The name of the test application in dev-packages/e2e-tests/test-applications.' @@ -22,89 +22,18 @@ inputs: runs: using: 'composite' steps: - - name: Set up Node - uses: actions/setup-node@v7 + - name: Set up E2E tests + uses: ./.github/actions/setup-e2e-tests with: - node-version-file: 'dev-packages/e2e-tests/test-applications/${{ inputs.test-application }}/package.json' - - name: Set up Bun - if: - inputs.test-application == 'node-exports-test-app' || contains(inputs.test-application, 'bun') || inputs.runtime - == 'bun' - uses: oven-sh/setup-bun@v2 - with: - bun-version: '1.3.14' - - name: Set up AWS SAM - if: contains(inputs.test-application, 'aws-serverless') - uses: aws-actions/setup-sam@v3 - with: - use-installer: true - token: ${{ github.token }} - - name: Set up Deno - if: contains(inputs.test-application, 'deno') || inputs.runtime == 'deno' - uses: denoland/setup-deno@v2.0.5 - with: - deno-version: ${{ inputs.deno-version || 'v2.8.3' }} - # Keeps the pnpm store on the cache volume for the `pnpm install` of the test application - - name: Mount pnpm store - uses: ./.github/actions/mount-pnpm-store - - name: Install pnpm - uses: ./.github/actions/install-pnpm - - # upload-artifact strips the path through the first `*`, so the tarballs are stored as /*.tgz and have to be - # downloaded into packages/. - - name: Restore tarball artifacts - uses: actions/download-artifact@v8 - with: - name: build-tarball-output - path: ${{ github.workspace }}/packages - run-id: ${{ inputs.artifacts-run-id || github.run_id }} - github-token: ${{ inputs.github-token }} - - - name: Restore test-utils tarball artifact - uses: actions/download-artifact@v8 - with: - name: build-test-utils-tarball-output - path: dev-packages/test-utils - run-id: ${{ inputs.artifacts-run-id || github.run_id }} + aws-serverless-layer: ${{ contains(inputs.test-application, 'aws-serverless') }} + aws-serverless-build: ${{ inputs.test-application == 'aws-serverless' }} + artifacts-run-id: ${{ inputs.artifacts-run-id }} github-token: ${{ inputs.github-token }} - - name: Restore build layer artifacts - if: contains(inputs.test-application, 'aws-serverless') - uses: actions/download-artifact@v8 + - name: Prepare E2E test application + uses: ./.github/actions/prepare-e2e-app with: - name: build-layer-output - path: ${{ github.workspace }}/packages/aws-serverless/build/aws/dist-serverless - run-id: ${{ inputs.artifacts-run-id || github.run_id }} - github-token: ${{ inputs.github-token }} - - # The Lambda functions install the built workspace packages directly (see `src/stack.ts`). - - name: Restore build artifacts - if: inputs.test-application == 'aws-serverless' - uses: actions/download-artifact@v8 - with: - name: build-output - run-id: ${{ inputs.artifacts-run-id || github.run_id }} - github-token: ${{ inputs.github-token }} - - - name: Prepare E2E tests - run: pnpm run test:prepare - shell: bash - working-directory: dev-packages/e2e-tests - - - name: Validate E2E test setup - run: pnpm run test:validate - shell: bash - working-directory: dev-packages/e2e-tests - - - name: Copy to temp - run: - pnpm run ci:copy-to-temp ./test-applications/${{ inputs.test-application }} ${{ runner.temp }}/test-application - shell: bash - working-directory: dev-packages/e2e-tests - - - name: Add pnpm overrides - run: - pnpm run ci:pnpm-overrides ${{ runner.temp }}/test-application ${{ github.workspace - }}/dev-packages/e2e-tests/packed - shell: bash - working-directory: dev-packages/e2e-tests + test-application: ${{ inputs.test-application }} + dir: ${{ runner.temp }}/test-application + runtime: ${{ inputs.runtime }} + deno-version: ${{ inputs.deno-version }} diff --git a/.github/actions/setup-e2e-tests/action.yml b/.github/actions/setup-e2e-tests/action.yml new file mode 100644 index 000000000000..a7bcfed3c041 --- /dev/null +++ b/.github/actions/setup-e2e-tests/action.yml @@ -0,0 +1,77 @@ +name: 'Set up E2E tests' +description: + 'Restores the built tarballs and prepares the E2E test setup once per job. Run `prepare-e2e-app` afterwards for each + test application of the job.' +inputs: + aws-serverless-layer: + description: 'Whether to restore the Lambda layer, which the `aws-serverless*` apps need.' + default: 'false' + aws-serverless-build: + description: 'Whether to restore the build output, which the `aws-serverless` app needs.' + default: 'false' + artifacts-run-id: + description: 'The workflow run that uploaded the tarball artifacts. Defaults to the current run.' + default: '' + github-token: + description: 'Token to download artifacts from `artifacts-run-id`. Only needed when that is set.' + default: '' + +runs: + using: 'composite' + steps: + # `prepare-e2e-app` switches to the Node version of each app + - name: Set up Node + uses: actions/setup-node@v7 + with: + node-version-file: 'package.json' + # Keeps the pnpm store on the cache volume for the `pnpm install` of the test application + - name: Mount pnpm store + uses: ./.github/actions/mount-pnpm-store + - name: Install pnpm + uses: ./.github/actions/install-pnpm + + # upload-artifact strips the path through the first `*`, so the tarballs are stored as /*.tgz and have to be + # downloaded into packages/. + - name: Restore tarball artifacts + uses: actions/download-artifact@v8 + with: + name: build-tarball-output + path: ${{ github.workspace }}/packages + run-id: ${{ inputs.artifacts-run-id || github.run_id }} + github-token: ${{ inputs.github-token }} + + - name: Restore test-utils tarball artifact + uses: actions/download-artifact@v8 + with: + name: build-test-utils-tarball-output + path: dev-packages/test-utils + run-id: ${{ inputs.artifacts-run-id || github.run_id }} + github-token: ${{ inputs.github-token }} + + - name: Restore build layer artifacts + if: inputs.aws-serverless-layer == 'true' + uses: actions/download-artifact@v8 + with: + name: build-layer-output + path: ${{ github.workspace }}/packages/aws-serverless/build/aws/dist-serverless + run-id: ${{ inputs.artifacts-run-id || github.run_id }} + github-token: ${{ inputs.github-token }} + + # The Lambda functions install the built workspace packages directly (see `src/stack.ts`). + - name: Restore build artifacts + if: inputs.aws-serverless-build == 'true' + uses: actions/download-artifact@v8 + with: + name: build-output + run-id: ${{ inputs.artifacts-run-id || github.run_id }} + github-token: ${{ inputs.github-token }} + + - name: Prepare E2E tests + run: pnpm run test:prepare + shell: bash + working-directory: dev-packages/e2e-tests + + - name: Validate E2E test setup + run: pnpm run test:validate + shell: bash + working-directory: dev-packages/e2e-tests diff --git a/.github/actions/test-e2e-app/action.yml b/.github/actions/test-e2e-app/action.yml new file mode 100644 index 000000000000..2d8eb3a67665 --- /dev/null +++ b/.github/actions/test-e2e-app/action.yml @@ -0,0 +1,43 @@ +name: 'Run E2E test application' +description: + 'Runs the tests of one application that `build-e2e-app` built, and uploads its Playwright traces if they fail. Takes + an entry of the `apps` list that `ci:build-matrix --batch-size` generates.' +inputs: + app: + description: 'The app as JSON: `dir` and the optional `runtime` and `assert-command`.' + required: true + +runs: + using: 'composite' + steps: + - name: Run E2E test + id: test + shell: bash + working-directory: ${{ runner.temp }}/e2e/${{ fromJSON(inputs.app).dir }} + run: | + status=0 + timeout 600 bash -c "$ASSERT_COMMAND" || status=$? + echo "status=$status" >> "$GITHUB_OUTPUT" + env: + ASSERT_COMMAND: ${{ fromJSON(inputs.app)['assert-command'] || 'pnpm test:assert' }} + SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }} + RUNTIME: ${{ fromJSON(inputs.app).runtime }} + + - name: Upload Playwright Traces + if: steps.test.outputs.status != '0' + uses: actions/upload-artifact@v7 + with: + name: playwright-traces-job_e2e_playwright_tests-${{ fromJSON(inputs.app).dir }} + path: ${{ runner.temp }}/e2e/${{ fromJSON(inputs.app).dir }}/test-results + overwrite: true + retention-days: 7 + + - name: Fail if the tests failed + if: steps.test.outputs.status != '0' + shell: bash + run: | + echo "::error::$LABEL failed with exit code $STATUS" + exit 1 + env: + LABEL: ${{ fromJSON(inputs.app).label }} + STATUS: ${{ steps.test.outputs.status }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d59dc8f590a4..711680037891 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -140,7 +140,7 @@ jobs: - name: Determine which test applications should be run id: matrix run: - pnpm --silent run ci:build-matrix --base=${{ (github.event_name == 'pull_request' && + pnpm --silent run ci:build-matrix --batch-size=4 --base=${{ (github.event_name == 'pull_request' && needs.job_get_metadata.outputs.changed_ci != 'true' && github.event.pull_request.base.sha) || '' }} >> $GITHUB_OUTPUT working-directory: dev-packages/e2e-tests @@ -688,6 +688,18 @@ jobs: pnpm run test:all${{ matrix.project && format(' --project={0}', matrix.project) || '' }}${{ matrix.shard && format(' --shard={0}/{1}', matrix.shard, matrix.shards) || '' }} + - &report_browser_flaky_tests + name: Report flaky tests + id: flaky + if: ${{ !cancelled() }} + run: node scripts/ci-report-flaky-tests.mjs dev-packages/browser-integration-tests/results.json + + # Only runs if a test passed on retry. scripts/weekly-flaky-tests.mjs looks for this step name. + - &flaky_tests_passed_on_retry + name: Flaky tests passed on retry + if: ${{ !cancelled() && steps.flaky.outputs.flaky == 'true' }} + run: echo "::warning::Some tests only passed on retry, see the job summary." + - name: Upload Playwright Traces uses: actions/upload-artifact@v7 if: failure() @@ -750,6 +762,9 @@ jobs: cd dev-packages/browser-integration-tests pnpm run test:loader + - *report_browser_flaky_tests + - *flaky_tests_passed_on_retry + - name: Upload Playwright Traces uses: actions/upload-artifact@v7 if: failure() @@ -1019,17 +1034,18 @@ jobs: overwrite: true job_e2e_tests: - name: E2E ${{ matrix.label || matrix.test-application }} Test + name: E2E ${{ matrix.label }} Test # We need to add the `always()` check here because the previous step has this as well :( # See: https://github.com/actions/runner/issues/2205 if: always() && needs.job_build_tarballs.result == 'success' && needs.job_build.outputs.e2e-matrix !='{"include":[]}' needs: [job_get_metadata, job_build, job_build_layer, job_build_tarballs] - # Most apps finish in under a minute on 4 vCPUs, so they run on 2. Apps that take longer set `sentryTest.runner`. + # Most apps finish in under a minute on 4 vCPUs, so they run on 2, four apps per job (see `--batch-size` in + # `getTestMatrix.mjs`). Apps that take longer set `sentryTest.runner` and get a job of their own. runs-on: ${{ matrix.runner || 'namespace-profile-sentry-javascript-small' }};overrides.cache-tag=sentry-javascript-e2e${{ github.event_name == 'pull_request' && '-pr' || '' }} - timeout-minutes: 15 + timeout-minutes: ${{ matrix.runner && 25 || 40 }} env: # We just use a dummy DSN here, only send to the tunnel anyhow E2E_TEST_DSN: 'https://username@domain/123' @@ -1039,8 +1055,6 @@ jobs: REACT_APP_E2E_TEST_DSN: 'https://username@domain/123' E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks' E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests' - # The `runtime` of a runtime variant, see dev-packages/e2e-tests/README.md - RUNTIME: ${{ matrix.runtime }} # Without the root node_modules, `pnpm run` would first install the whole workspace. pnpm_config_verify_deps_before_run: 'false' strategy: @@ -1051,66 +1065,126 @@ jobs: uses: actions/checkout@v7 with: ref: ${{ env.HEAD_COMMIT }} - - name: Prepare E2E test application - uses: ./.github/actions/prepare-e2e-test + - name: Set up E2E tests + uses: ./.github/actions/setup-e2e-tests with: - test-application: ${{ matrix.test-application }} - runtime: ${{ matrix.runtime }} - deno-version: ${{ matrix.deno-version }} + aws-serverless-layer: ${{ matrix.aws-serverless-layer }} + aws-serverless-build: ${{ matrix.aws-serverless-build }} - # Installs Playwright while the app installs and builds. Keep it right before the install: the runner sets up a - # background composite action's steps from the job's shared `github` context, which fails with "Collection was - # modified" if a foreground step starts at that moment. Starting next to `download-artifact` also made it fail to - # read the event file. + # Installs Playwright while the first app installs and builds. The apps of a job use the same Playwright version. + # Keep a `run` step right after it: the runner sets up a background composite action's steps from the job's shared + # `github` context, which fails with "Collection was modified" if a foreground step starts at that moment. Starting + # next to `download-artifact` also made it fail to read the event file. - name: Install Playwright id: install-playwright background: true uses: ./.github/actions/install-playwright with: browsers: chromium - cwd: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }} - - # Every build command runs `pnpm install`, so installing first makes that a no-op. Some downloads from the npm - # registry crawl at tens of kB/s for the whole 7 minutes of the build step, and pnpm does not give up on a slow - # download. Another attempt reuses what is already in the store and opens new connections. - # The hoisted variant installs with another node linker, so installing first would only redo the install. - - name: Install E2E app dependencies - if: ${{ !contains(matrix.build-command, 'hoisted') }} - working-directory: ${{ runner.temp }}/test-application - timeout-minutes: 7 - run: | - for attempt in 1 2; do - timeout 150 pnpm install && exit 0 - echo "::warning::pnpm install attempt $attempt failed or did not finish in 150 s, retrying" - done - pnpm install - env: - SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }} + cwd: dev-packages/e2e-tests/test-applications/${{ matrix.playwright-app }} - - name: Build E2E app - working-directory: ${{ runner.temp }}/test-application - timeout-minutes: 7 - run: ${{ matrix.build-command || 'pnpm test:build' }} + - name: List the apps of this job + run: echo "$APPS" env: - SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }} + APPS: ${{ toJSON(matrix.apps) }} + + # Each app runs in its own steps. A failed app does not stop the next one, and `Check E2E results` fails the job. + - name: Build ${{ matrix.apps[0].label }} + id: build-0 + continue-on-error: true + timeout-minutes: 15 + uses: ./.github/actions/build-e2e-app + with: + app: ${{ toJSON(matrix.apps[0]) }} - wait: [install-playwright] - - name: Run E2E test - working-directory: ${{ runner.temp }}/test-application - timeout-minutes: 10 - run: ${{ matrix.assert-command || 'pnpm test:assert' }} + - name: Test ${{ matrix.apps[0].label }} + id: test-0 + if: steps.build-0.outcome == 'success' + continue-on-error: true + timeout-minutes: 11 + uses: ./.github/actions/test-e2e-app + with: + app: ${{ toJSON(matrix.apps[0]) }} + + - name: Build ${{ matrix.apps[1].label }} + id: build-1 + if: matrix.apps[1] + continue-on-error: true + timeout-minutes: 15 + uses: ./.github/actions/build-e2e-app + with: + app: ${{ toJSON(matrix.apps[1]) }} + clean-up: true + + - name: Test ${{ matrix.apps[1].label }} + id: test-1 + if: steps.build-1.outcome == 'success' + continue-on-error: true + timeout-minutes: 11 + uses: ./.github/actions/test-e2e-app + with: + app: ${{ toJSON(matrix.apps[1]) }} + + - name: Build ${{ matrix.apps[2].label }} + id: build-2 + if: matrix.apps[2] + continue-on-error: true + timeout-minutes: 15 + uses: ./.github/actions/build-e2e-app + with: + app: ${{ toJSON(matrix.apps[2]) }} + clean-up: true + + - name: Test ${{ matrix.apps[2].label }} + id: test-2 + if: steps.build-2.outcome == 'success' + continue-on-error: true + timeout-minutes: 11 + uses: ./.github/actions/test-e2e-app + with: + app: ${{ toJSON(matrix.apps[2]) }} + + - name: Build ${{ matrix.apps[3].label }} + id: build-3 + if: matrix.apps[3] + continue-on-error: true + timeout-minutes: 15 + uses: ./.github/actions/build-e2e-app + with: + app: ${{ toJSON(matrix.apps[3]) }} + clean-up: true + + - name: Test ${{ matrix.apps[3].label }} + id: test-3 + if: steps.build-3.outcome == 'success' + continue-on-error: true + timeout-minutes: 11 + uses: ./.github/actions/test-e2e-app + with: + app: ${{ toJSON(matrix.apps[3]) }} + + - name: Report flaky tests + id: flaky + if: ${{ !cancelled() }} + run: node scripts/ci-report-flaky-tests.mjs "$RUNNER_TEMP"/e2e/*/results.json + + - *flaky_tests_passed_on_retry + + - name: Check E2E results + if: ${{ !cancelled() }} + run: node dev-packages/e2e-tests/lib/checkJobResults.mjs env: - SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }} - - - name: Upload Playwright Traces - uses: actions/upload-artifact@v7 - if: failure() - with: - name: playwright-traces-job_e2e_playwright_tests-${{ matrix.test-application}} - path: ${{ runner.temp }}/test-application/test-results - overwrite: true - retention-days: 7 + APPS: ${{ toJSON(matrix.apps) }} + BUILD_0: ${{ steps.build-0.outcome }} + TEST_0: ${{ steps.test-0.outcome }} + BUILD_1: ${{ steps.build-1.outcome }} + TEST_1: ${{ steps.test-1.outcome }} + BUILD_2: ${{ steps.build-2.outcome }} + TEST_2: ${{ steps.test-2.outcome }} + BUILD_3: ${{ steps.build-3.outcome }} + TEST_3: ${{ steps.test-3.outcome }} # - We skip optional tests on release branches # - The tests run in optional-e2e.yml, so their failures do not mark the PR as failed. This job only waits for diff --git a/.github/workflows/warm-e2e-cache.yml b/.github/workflows/warm-e2e-cache.yml new file mode 100644 index 000000000000..ba75b039c500 --- /dev/null +++ b/.github/workflows/warm-e2e-cache.yml @@ -0,0 +1,99 @@ +name: 'CI: Warm E2E caches' + +# The E2E jobs share one cache volume per tag, and the last job to finish overwrites what the others saved, so the pnpm +# store on it never holds the packages of every app. Downloads that the store misses are what crawls in the E2E builds. +# This job installs every test application into the store once, and later jobs fork the volume it commits. +# It also removes the packages no app uses anymore, so the volume stays under its size limit: Namespace does not clone +# a cache that outgrew it. + +on: + workflow_dispatch: + schedule: + - cron: '0 4 * * *' + +permissions: + contents: read + # To download the tarballs of the build.yml run + actions: read + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + job_find_build: + name: Find the latest develop build + runs-on: namespace-profile-sentry-javascript-small + timeout-minutes: 5 + outputs: + run-id: ${{ steps.build.outputs.run-id }} + steps: + - name: Find a develop build.yml run with tarballs + id: build + uses: actions/github-script@v9 + with: + retries: 3 + script: | + const { data } = await github.rest.actions.listWorkflowRuns({ + ...context.repo, + workflow_id: 'build.yml', + branch: 'develop', + event: 'push', + status: 'completed', + per_page: 30, + }); + for (const run of data.workflow_runs) { + const { data: artifacts } = await github.rest.actions.listWorkflowRunArtifacts({ + ...context.repo, + run_id: run.id, + per_page: 100, + }); + const names = new Set(artifacts.artifacts.filter(artifact => !artifact.expired).map(artifact => artifact.name)); + if (names.has('build-tarball-output') && names.has('build-test-utils-tarball-output')) { + core.info(`Using the tarballs of ${run.html_url}`); + core.setOutput('run-id', run.id); + return; + } + } + core.setFailed('None of the last 30 develop build.yml runs has tarball artifacts.'); + + job_warm_cache: + name: Warm ${{ matrix.cache-tag }} + needs: [job_find_build] + # The profile and tags of the E2E jobs in build.yml and optional-e2e.yml + runs-on: namespace-profile-sentry-javascript;overrides.cache-tag=${{ matrix.cache-tag }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + cache-tag: [sentry-javascript-e2e, sentry-javascript-e2e-pr] + steps: + # The test applications of this commit with the tarballs of a slightly older one. An app that needs a package the + # tarballs do not have yet fails to install and only gets a warning. + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up E2E tests + uses: ./.github/actions/setup-e2e-tests + with: + artifacts-run-id: ${{ needs.job_find_build.outputs.run-id }} + github-token: ${{ github.token }} + + - name: Install Playwright + uses: ./.github/actions/install-playwright + with: + browsers: chromium + cwd: dev-packages/test-utils + + - name: Install every test application + working-directory: dev-packages/e2e-tests + run: node lib/warmCache.mjs "$RUNNER_TEMP/warm" 4 + + - name: Remove packages that no test application uses + working-directory: ${{ runner.temp }} + run: | + echo "pnpm store before: $(du -sh "$PNPM_CONFIG_STORE_DIR" | cut -f1)" + pnpm store prune --yes + echo "pnpm store after: $(du -sh "$PNPM_CONFIG_STORE_DIR" | cut -f1)" + df -h /cache || true diff --git a/dev-packages/browser-integration-tests/.gitignore b/dev-packages/browser-integration-tests/.gitignore index b258dda7330c..bf71095ac0b2 100644 --- a/dev-packages/browser-integration-tests/.gitignore +++ b/dev-packages/browser-integration-tests/.gitignore @@ -1,3 +1,4 @@ test-results tmp *.junit.xml +results.json diff --git a/dev-packages/browser-integration-tests/playwright.config.ts b/dev-packages/browser-integration-tests/playwright.config.ts index 681de57d4e59..b5a96e94be0f 100644 --- a/dev-packages/browser-integration-tests/playwright.config.ts +++ b/dev-packages/browser-integration-tests/playwright.config.ts @@ -2,7 +2,9 @@ import type { PlaywrightTestConfig } from '@playwright/test'; import { devices } from '@playwright/test'; const config: PlaywrightTestConfig = { - retries: 0, + // A test that passes on retry is reported as flaky in the job summary and in the weekly flaky test report. + // `test:detect-flaky` passes `--retries=0`, so it still fails on the first flaky run. + retries: process.env.CI ? 1 : 0, // Run tests inside of a single file in parallel fullyParallel: true, // Use 3 workers on CI, else use defaults (based on available CPU cores) @@ -30,7 +32,9 @@ const config: PlaywrightTestConfig = { }, ], - reporter: process.env.CI ? [['list'], ['github'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list', + reporter: process.env.CI + ? [['list'], ['github'], ['junit', { outputFile: 'results.junit.xml' }], ['json', { outputFile: 'results.json' }]] + : 'list', globalSetup: require.resolve('./playwright.setup.ts'), globalTeardown: require.resolve('./playwright.teardown.ts'), diff --git a/dev-packages/browser-integration-tests/scripts/detectFlakyTests.ts b/dev-packages/browser-integration-tests/scripts/detectFlakyTests.ts index 21f666adeb0e..5ad3eaf1806a 100644 --- a/dev-packages/browser-integration-tests/scripts/detectFlakyTests.ts +++ b/dev-packages/browser-integration-tests/scripts/detectFlakyTests.ts @@ -48,7 +48,7 @@ ${changedPaths.join('\n')} const cp = childProcess.spawn( `npx playwright test ${ testPaths.length ? testPaths.join(' ') : './suites' - } --repeat-each ${repeatEachCount} --project=chromium`, + } --repeat-each ${repeatEachCount} --retries=0 --project=chromium`, { shell: true, cwd, stdio: 'inherit' }, ); diff --git a/dev-packages/e2e-tests/README.md b/dev-packages/e2e-tests/README.md index d71ba9b1b8a3..f6b6dfb3247d 100644 --- a/dev-packages/e2e-tests/README.md +++ b/dev-packages/e2e-tests/README.md @@ -280,9 +280,13 @@ For example, if something is changed in the browser package, only E2E test apps You can add additional information about the test (e.g. canary versions, optional in CI) by adding `sentryTest` in the `package.json` of a test application. -The required E2E jobs run on 2 vCPUs (`namespace-profile-sentry-javascript-small`). If an app's job takes longer than -about 90 seconds, set `"sentryTest": { "runner": "namespace-profile-sentry-javascript" }` to run it and its variants on -4 vCPUs. +The required E2E jobs run on 2 vCPUs (`namespace-profile-sentry-javascript-small`), four apps or variants per job, one +after another in `$RUNNER_TEMP/e2e/