diff --git a/.github/actions/build-e2e-app/action.yml b/.github/actions/build-e2e-app/action.yml new file mode 100644 index 000000000000..aa42271f8421 --- /dev/null +++ b/.github/actions/build-e2e-app/action.yml @@ -0,0 +1,95 @@ +name: 'Build E2E test application' +description: + 'Prepares, installs and builds one application of an E2E job in `runner.temp/e2e/`. Takes an entry of the `apps` + list that `ci:build-matrix --batch-size` generates.' +inputs: + app: + description: + 'The app as JSON: `test-application`, `dir` and the optional `runtime`, `deno-version`, `build-command`.' + required: true + previous-app: + description: + 'The app that ran before in the same job, as JSON. Its leftover processes and containers are removed first.' + default: '' + +# Isolation between the apps of a job: the install, build and test steps of an app run with `SENTRY_E2E_APP=`, +# which every process they start inherits, also servers that Playwright detaches. Before the next app starts, the +# processes with the previous app's marker are stopped, and the containers created since the previous app started are +# removed. This also covers an app whose build or test step timed out. The last app of a job needs no clean-up. +runs: + using: 'composite' + steps: + # Playwright stops the web servers of an app and the apps stop their containers in their global teardown. This only + # matters when an app timed out before that, and its servers would block the ports of the next app. + - name: Stop leftovers of the previous app + if: inputs.previous-app != '' + shell: bash + run: | + pids=() + for proc in /proc/[0-9]*; do + if tr '\0' '\n' < "$proc/environ" 2>/dev/null | grep -qxF "SENTRY_E2E_APP=$PREVIOUS_DIR"; then + pids+=("${proc#/proc/}") + echo "Stopping $(tr '\0' ' ' < "$proc/cmdline" 2>/dev/null)" + fi + done + if [ ${#pids[@]} -gt 0 ]; then + kill -TERM "${pids[@]}" 2>/dev/null || true + sleep 5 + kill -KILL "${pids[@]}" 2>/dev/null || true + fi + + before="$RUNNER_TEMP/e2e/$PREVIOUS_DIR.containers" + if [ -f "$before" ]; then + containers=$(comm -13 "$before" <(docker ps -aq --no-trunc 2>/dev/null | sort)) + if [ -n "$containers" ]; then + echo "Removing containers: $containers" + docker rm -f $containers > /dev/null || true + fi + fi + env: + PREVIOUS_DIR: ${{ inputs.previous-app && fromJSON(inputs.previous-app).dir || '' }} + + - name: Record the containers that exist before the app + shell: bash + run: | + mkdir -p "$RUNNER_TEMP/e2e" + docker ps -aq --no-trunc 2>/dev/null | sort > "$RUNNER_TEMP/e2e/$APP_DIR.containers" || true + env: + APP_DIR: ${{ fromJSON(inputs.app).dir }} + + - name: Prepare E2E test application + uses: ./.github/actions/prepare-e2e-app + with: + test-application: ${{ fromJSON(inputs.app)['test-application'] }} + dir: ${{ runner.temp }}/e2e/${{ fromJSON(inputs.app).dir }} + runtime: ${{ fromJSON(inputs.app).runtime }} + deno-version: ${{ fromJSON(inputs.app)['deno-version'] }} + + # Every build command runs `pnpm install`, so installing first makes that a no-op. Some downloads from the npm + # registry crawl at tens of kB/s for the whole 7 minutes of the build step, and pnpm does not give up on a slow + # download. Another attempt reuses what is already in the store and opens new connections. + # The hoisted variant installs with another node linker, so installing first would only redo the install. + - name: Install E2E app dependencies + if: ${{ !contains(fromJSON(inputs.app)['build-command'], 'hoisted') }} + shell: bash + working-directory: ${{ runner.temp }}/e2e/${{ fromJSON(inputs.app).dir }} + run: | + for attempt in 1 2; do + timeout 150 pnpm install && exit 0 + echo "::warning::pnpm install attempt $attempt failed or did not finish in 150 s, retrying" + done + timeout 300 pnpm install + env: + SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }} + SENTRY_E2E_APP: ${{ fromJSON(inputs.app).dir }} + RUNTIME: ${{ fromJSON(inputs.app).runtime }} + + - name: Build E2E app + shell: bash + working-directory: ${{ runner.temp }}/e2e/${{ fromJSON(inputs.app).dir }} + run: timeout 420 bash -c "$BUILD_COMMAND" + env: + BUILD_COMMAND: ${{ fromJSON(inputs.app)['build-command'] || 'pnpm test:build' }} + SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }} + SENTRY_E2E_APP: ${{ fromJSON(inputs.app).dir }} + RUNTIME: ${{ fromJSON(inputs.app).runtime }} diff --git a/.github/actions/install-dependencies/action.yml b/.github/actions/install-dependencies/action.yml index 2a4b331fd3f6..e021ac385e5b 100644 --- a/.github/actions/install-dependencies/action.yml +++ b/.github/actions/install-dependencies/action.yml @@ -58,7 +58,14 @@ runs: shell: bash # The cache volume can hold node_modules of another commit or branch. This is a no-op when they match - # pnpm-lock.yaml. + # pnpm-lock.yaml. On an empty cache volume, single downloads from the npm registry can crawl at kB/s for longer + # than the job's timeout, and pnpm does not give up on them. Another attempt reuses what is already in the store + # and opens new connections. - name: Install dependencies - run: pnpm install --frozen-lockfile + run: | + for attempt in 1 2; do + timeout 300 pnpm install --frozen-lockfile && exit 0 + echo "::warning::pnpm install attempt $attempt failed or did not finish in 300 s, retrying" + done + timeout 600 pnpm install --frozen-lockfile shell: bash diff --git a/.github/actions/install-playwright/action.yml b/.github/actions/install-playwright/action.yml index 378e77666093..5ea801184ed7 100644 --- a/.github/actions/install-playwright/action.yml +++ b/.github/actions/install-playwright/action.yml @@ -34,7 +34,7 @@ runs: playwright apt - # `--with-deps` names packages that the runner image already has (glib, mesa, nss, ...), and `apt-get install` would + # `install-deps` names packages that the runner image already has (glib, mesa, nss, ...), and `apt-get install` would # upgrade them to the latest Ubuntu update. That is half of the download on a cold cache, and the image versions work. - name: Keep installed apt packages if: steps.playwright-version.outputs.version != '' @@ -43,11 +43,40 @@ runs: # Downloads only the browsers and apt packages that are not on the cache volume yet. `npx` runs outside the repo, # so it can't pick up a `playwright` that a parallel `pnpm install` is still linking. + # A browser download can crawl for minutes, so an attempt that takes too long is retried. apt is not stopped + # midway, so it can't leave half-installed packages behind. + # On the runner image, the only packages `install-deps` installs for Chromium are fonts, and no test compares + # rendered pixels. On an empty cache volume, their download crawled for up to 10 minutes, so Chromium skips them + # unless a library is missing. WebKit and Firefox need libraries the image does not have. - name: Install Playwright if: steps.playwright-version.outputs.version != '' env: PLAYWRIGHT_BROWSERS: ${{ inputs.browsers || 'chromium webkit firefox' }} PLAYWRIGHT_VERSION: ${{ steps.playwright-version.outputs.version }} - run: npx -y "playwright@$PLAYWRIGHT_VERSION" install --with-deps $PLAYWRIGHT_BROWSERS + run: | + read -ra browsers <<< "$PLAYWRIGHT_BROWSERS" + playwright=(npx -y "playwright@$PLAYWRIGHT_VERSION") + installed=false + for attempt in 1 2; do + if timeout 300 "${playwright[@]}" install "${browsers[@]}"; then + installed=true + break + fi + echo "::warning::playwright install attempt $attempt failed or did not finish in 300 s, retrying" + done + if [[ "$installed" != true ]]; then + timeout 600 "${playwright[@]}" install "${browsers[@]}" + fi + + if [[ "${browsers[*]}" == chromium ]]; then + missing=$(find ~/.cache/ms-playwright -type f \( -name chrome -o -name chrome-headless-shell \) \ + -exec ldd {} + 2> /dev/null | grep 'not found' || true) + if [[ -z "$missing" ]]; then + exit 0 + fi + echo "Chromium is missing libraries, installing its system dependencies:" + echo "$missing" + fi + "${playwright[@]}" install-deps "${browsers[@]}" shell: bash working-directory: ${{ runner.temp }} diff --git a/.github/actions/install-pnpm/action.yml b/.github/actions/install-pnpm/action.yml index 434a1251edf5..d6703a204a86 100644 --- a/.github/actions/install-pnpm/action.yml +++ b/.github/actions/install-pnpm/action.yml @@ -4,5 +4,76 @@ description: 'Installs the pnpm version of `packageManager` in the root package. runs: using: 'composite' steps: - # Without a `version` input, the action reads the version from `packageManager`. With both, it fails. - - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + # `pnpm/action-setup` installs a bootstrap pnpm with npm, which then downloads the `packageManager` pnpm. Every job + # repeated both downloads, and neither gives up on a download that crawls. This keeps the standalone pnpm binary in + # the Actions cache, and aborts and retries a download that stays too slow. + - name: Get pnpm version + id: pnpm + shell: bash + run: | + version=$(node -p 'require("./package.json").packageManager.replace(/^pnpm@/, "").split("+")[0]') + case "$RUNNER_ARCH" in + X64) arch=x64 ;; + ARM64) arch=arm64 ;; + *) echo "::error::No standalone pnpm for runner architecture $RUNNER_ARCH" && exit 1 ;; + esac + { + echo "version=$version" + echo "package=@pnpm/exe.linux-$arch" + echo "key=pnpm-exe-linux-$arch-$version" + echo "dir=$HOME/.pnpm-exe/$version-linux-$arch" + } >> "$GITHUB_OUTPUT" + + - name: Restore pnpm from the Actions cache + id: cache + uses: actions/cache/restore@v6.1.0 + with: + path: ${{ steps.pnpm.outputs.dir }} + key: ${{ steps.pnpm.outputs.key }} + + - name: Download pnpm + if: steps.cache.outputs.cache-hit != 'true' + shell: bash + env: + PNPM_DIR: ${{ steps.pnpm.outputs.dir }} + PNPM_PACKAGE: ${{ steps.pnpm.outputs.package }} + PNPM_VERSION: ${{ steps.pnpm.outputs.version }} + run: | + fetch() { + curl --fail --silent --show-error --location --retry 4 --retry-all-errors --connect-timeout 10 "$@" + } + manifest=$(fetch "https://registry.npmjs.org/$PNPM_PACKAGE/$PNPM_VERSION") + tarball=$(node -p 'JSON.parse(process.argv[1]).dist.tarball' "$manifest") + integrity=$(node -p 'JSON.parse(process.argv[1]).dist.integrity' "$manifest") + # Aborts a transfer that stays under 256 kB/s for 15 s, so a retry can open a faster connection. If every + # connection is that slow, the last download takes whatever speed it gets. + if ! fetch --speed-limit 262144 --speed-time 15 --output "$RUNNER_TEMP/pnpm.tgz" "$tarball"; then + echo "::warning::Every download of $tarball was slower than 256 kB/s, downloading it without a speed limit" + fetch --max-time 600 --output "$RUNNER_TEMP/pnpm.tgz" "$tarball" + fi + actual="sha512-$(openssl dgst -sha512 -binary "$RUNNER_TEMP/pnpm.tgz" | base64 -w0)" + if [[ "$actual" != "$integrity" ]]; then + echo "::error::$tarball has integrity $actual, but the registry lists $integrity" + exit 1 + fi + mkdir -p "$PNPM_DIR" + tar -xzf "$RUNNER_TEMP/pnpm.tgz" -C "$PNPM_DIR" --strip-components=1 package/pnpm + rm "$RUNNER_TEMP/pnpm.tgz" + + # Saved right away rather than at the end of the job, so jobs that fail later still fill the cache. PRs restore the + # base branch's copy, so they don't save their own. + - name: Save pnpm to the Actions cache + if: steps.cache.outputs.cache-hit != 'true' && github.event_name != 'pull_request' + uses: actions/cache/save@v6.1.0 + with: + path: ${{ steps.pnpm.outputs.dir }} + key: ${{ steps.pnpm.outputs.key }} + + - name: Add pnpm to PATH + shell: bash + env: + PNPM_DIR: ${{ steps.pnpm.outputs.dir }} + run: | + echo "PNPM_HOME=$PNPM_DIR" >> "$GITHUB_ENV" + echo "$PNPM_DIR" >> "$GITHUB_PATH" + "$PNPM_DIR/pnpm" --version diff --git a/.github/actions/mount-pnpm-store/action.yml b/.github/actions/mount-pnpm-store/action.yml index fed4e40c1b73..9423cf30ba09 100644 --- a/.github/actions/mount-pnpm-store/action.yml +++ b/.github/actions/mount-pnpm-store/action.yml @@ -1,7 +1,5 @@ name: 'Mount pnpm store' -description: - 'Mounts the pnpm store, the pnpm and npm caches and the given paths from the Namespace cache volume. Use it before - `install-pnpm`, so `install-pnpm` takes pnpm from the cache volume.' +description: 'Mounts the pnpm store, the pnpm and npm caches and the given paths from the Namespace cache volume.' inputs: path: @@ -12,8 +10,7 @@ inputs: runs: using: 'composite' steps: - # Replaces the `pnpm` mode of nscloud-cache-action until it works without pnpm on PATH. The mode runs `pnpm store - # path`, so it only works after `install-pnpm`, when `pnpm self-update` has already downloaded pnpm. + # Replaces the `pnpm` mode of nscloud-cache-action, which runs `pnpm store path` and so needs pnpm on PATH. # The cache directory holds `lockfile-verified.jsonl`. With it, `pnpm install` skips the supply-chain check of a # lockfile that was already verified, which otherwise fetches the metadata of every package from the registry. - name: Set pnpm store and cache directories @@ -22,7 +19,7 @@ runs: echo "PNPM_CONFIG_CACHE_DIR=$HOME/.cache/pnpm" >> "$GITHUB_ENV" shell: bash - # The `npm` mode keeps the npm cache of the `npm ci` in `pnpm/action-setup`, which installs the bootstrap pnpm + # The `npm` mode keeps the npm cache of `npx` and of the `npm install`s in the integration tests - name: Mount cache paths uses: namespacelabs/nscloud-cache-action@1124a6f3ce44e5cf84cc22111530961f4d2a15f9 # v1.7.0 with: diff --git a/.github/actions/prepare-e2e-app/action.yml b/.github/actions/prepare-e2e-app/action.yml new file mode 100644 index 000000000000..bfb7d4b5b67e --- /dev/null +++ b/.github/actions/prepare-e2e-app/action.yml @@ -0,0 +1,59 @@ +name: 'Prepare E2E test application' +description: + 'Sets up the runtimes for an E2E test application and copies it to `dir` with pnpm overrides pointing at the tarballs. + Run `setup-e2e-tests` first.' +inputs: + test-application: + description: 'The name of the test application in dev-packages/e2e-tests/test-applications.' + required: true + dir: + description: 'The directory to copy the application to.' + required: true + runtime: + description: 'The `runtime` of a runtime variant, see dev-packages/e2e-tests/README.md.' + default: '' + deno-version: + description: 'The Deno version to install, if the application needs Deno.' + default: 'v2.8.3' + +runs: + using: 'composite' + steps: + - name: Set up Node + uses: actions/setup-node@v7 + with: + node-version-file: 'dev-packages/e2e-tests/test-applications/${{ inputs.test-application }}/package.json' + - name: Set up Bun + if: + inputs.test-application == 'node-exports-test-app' || contains(inputs.test-application, 'bun') || inputs.runtime + == 'bun' + uses: oven-sh/setup-bun@v2 + with: + bun-version: '1.3.14' + - name: Set up AWS SAM + if: contains(inputs.test-application, 'aws-serverless') + uses: aws-actions/setup-sam@v3 + with: + use-installer: true + token: ${{ github.token }} + - name: Set up Deno + if: contains(inputs.test-application, 'deno') || inputs.runtime == 'deno' + uses: denoland/setup-deno@v2.0.5 + with: + deno-version: ${{ inputs.deno-version || 'v2.8.3' }} + + - name: Copy to temp + run: pnpm run ci:copy-to-temp "./test-applications/$TEST_APPLICATION" "$APP_DIR" + shell: bash + working-directory: dev-packages/e2e-tests + env: + TEST_APPLICATION: ${{ inputs.test-application }} + APP_DIR: ${{ inputs.dir }} + RUNTIME: ${{ inputs.runtime }} + + - name: Add pnpm overrides + run: pnpm run ci:pnpm-overrides "$APP_DIR" "$GITHUB_WORKSPACE/dev-packages/e2e-tests/packed" + shell: bash + working-directory: dev-packages/e2e-tests + env: + APP_DIR: ${{ inputs.dir }} diff --git a/.github/actions/prepare-e2e-test/action.yml b/.github/actions/prepare-e2e-test/action.yml index 3f5b37b41506..c71bce363237 100644 --- a/.github/actions/prepare-e2e-test/action.yml +++ b/.github/actions/prepare-e2e-test/action.yml @@ -1,7 +1,7 @@ name: 'Prepare E2E test application' description: 'Sets up the runtimes for an E2E test application, restores the built tarballs and copies the application to - `runner.temp/test-application` with pnpm overrides pointing at the tarballs.' + `runner.temp/test-application` with pnpm overrides pointing at the tarballs. For a job that runs one application.' inputs: test-application: description: 'The name of the test application in dev-packages/e2e-tests/test-applications.' @@ -22,89 +22,18 @@ inputs: runs: using: 'composite' steps: - - name: Set up Node - uses: actions/setup-node@v7 + - name: Set up E2E tests + uses: ./.github/actions/setup-e2e-tests with: - node-version-file: 'dev-packages/e2e-tests/test-applications/${{ inputs.test-application }}/package.json' - - name: Set up Bun - if: - inputs.test-application == 'node-exports-test-app' || contains(inputs.test-application, 'bun') || inputs.runtime - == 'bun' - uses: oven-sh/setup-bun@v2 - with: - bun-version: '1.3.14' - - name: Set up AWS SAM - if: contains(inputs.test-application, 'aws-serverless') - uses: aws-actions/setup-sam@v3 - with: - use-installer: true - token: ${{ github.token }} - - name: Set up Deno - if: contains(inputs.test-application, 'deno') || inputs.runtime == 'deno' - uses: denoland/setup-deno@v2.0.5 - with: - deno-version: ${{ inputs.deno-version || 'v2.8.3' }} - # Keeps the pnpm store on the cache volume for the `pnpm install` of the test application - - name: Mount pnpm store - uses: ./.github/actions/mount-pnpm-store - - name: Install pnpm - uses: ./.github/actions/install-pnpm - - # upload-artifact strips the path through the first `*`, so the tarballs are stored as /*.tgz and have to be - # downloaded into packages/. - - name: Restore tarball artifacts - uses: actions/download-artifact@v8 - with: - name: build-tarball-output - path: ${{ github.workspace }}/packages - run-id: ${{ inputs.artifacts-run-id || github.run_id }} - github-token: ${{ inputs.github-token }} - - - name: Restore test-utils tarball artifact - uses: actions/download-artifact@v8 - with: - name: build-test-utils-tarball-output - path: dev-packages/test-utils - run-id: ${{ inputs.artifacts-run-id || github.run_id }} + aws-serverless-layer: ${{ contains(inputs.test-application, 'aws-serverless') }} + aws-serverless-build: ${{ inputs.test-application == 'aws-serverless' }} + artifacts-run-id: ${{ inputs.artifacts-run-id }} github-token: ${{ inputs.github-token }} - - name: Restore build layer artifacts - if: contains(inputs.test-application, 'aws-serverless') - uses: actions/download-artifact@v8 + - name: Prepare E2E test application + uses: ./.github/actions/prepare-e2e-app with: - name: build-layer-output - path: ${{ github.workspace }}/packages/aws-serverless/build/aws/dist-serverless - run-id: ${{ inputs.artifacts-run-id || github.run_id }} - github-token: ${{ inputs.github-token }} - - # The Lambda functions install the built workspace packages directly (see `src/stack.ts`). - - name: Restore build artifacts - if: inputs.test-application == 'aws-serverless' - uses: actions/download-artifact@v8 - with: - name: build-output - run-id: ${{ inputs.artifacts-run-id || github.run_id }} - github-token: ${{ inputs.github-token }} - - - name: Prepare E2E tests - run: pnpm run test:prepare - shell: bash - working-directory: dev-packages/e2e-tests - - - name: Validate E2E test setup - run: pnpm run test:validate - shell: bash - working-directory: dev-packages/e2e-tests - - - name: Copy to temp - run: - pnpm run ci:copy-to-temp ./test-applications/${{ inputs.test-application }} ${{ runner.temp }}/test-application - shell: bash - working-directory: dev-packages/e2e-tests - - - name: Add pnpm overrides - run: - pnpm run ci:pnpm-overrides ${{ runner.temp }}/test-application ${{ github.workspace - }}/dev-packages/e2e-tests/packed - shell: bash - working-directory: dev-packages/e2e-tests + test-application: ${{ inputs.test-application }} + dir: ${{ runner.temp }}/test-application + runtime: ${{ inputs.runtime }} + deno-version: ${{ inputs.deno-version }} diff --git a/.github/actions/setup-e2e-tests/action.yml b/.github/actions/setup-e2e-tests/action.yml new file mode 100644 index 000000000000..a7bcfed3c041 --- /dev/null +++ b/.github/actions/setup-e2e-tests/action.yml @@ -0,0 +1,77 @@ +name: 'Set up E2E tests' +description: + 'Restores the built tarballs and prepares the E2E test setup once per job. Run `prepare-e2e-app` afterwards for each + test application of the job.' +inputs: + aws-serverless-layer: + description: 'Whether to restore the Lambda layer, which the `aws-serverless*` apps need.' + default: 'false' + aws-serverless-build: + description: 'Whether to restore the build output, which the `aws-serverless` app needs.' + default: 'false' + artifacts-run-id: + description: 'The workflow run that uploaded the tarball artifacts. Defaults to the current run.' + default: '' + github-token: + description: 'Token to download artifacts from `artifacts-run-id`. Only needed when that is set.' + default: '' + +runs: + using: 'composite' + steps: + # `prepare-e2e-app` switches to the Node version of each app + - name: Set up Node + uses: actions/setup-node@v7 + with: + node-version-file: 'package.json' + # Keeps the pnpm store on the cache volume for the `pnpm install` of the test application + - name: Mount pnpm store + uses: ./.github/actions/mount-pnpm-store + - name: Install pnpm + uses: ./.github/actions/install-pnpm + + # upload-artifact strips the path through the first `*`, so the tarballs are stored as /*.tgz and have to be + # downloaded into packages/. + - name: Restore tarball artifacts + uses: actions/download-artifact@v8 + with: + name: build-tarball-output + path: ${{ github.workspace }}/packages + run-id: ${{ inputs.artifacts-run-id || github.run_id }} + github-token: ${{ inputs.github-token }} + + - name: Restore test-utils tarball artifact + uses: actions/download-artifact@v8 + with: + name: build-test-utils-tarball-output + path: dev-packages/test-utils + run-id: ${{ inputs.artifacts-run-id || github.run_id }} + github-token: ${{ inputs.github-token }} + + - name: Restore build layer artifacts + if: inputs.aws-serverless-layer == 'true' + uses: actions/download-artifact@v8 + with: + name: build-layer-output + path: ${{ github.workspace }}/packages/aws-serverless/build/aws/dist-serverless + run-id: ${{ inputs.artifacts-run-id || github.run_id }} + github-token: ${{ inputs.github-token }} + + # The Lambda functions install the built workspace packages directly (see `src/stack.ts`). + - name: Restore build artifacts + if: inputs.aws-serverless-build == 'true' + uses: actions/download-artifact@v8 + with: + name: build-output + run-id: ${{ inputs.artifacts-run-id || github.run_id }} + github-token: ${{ inputs.github-token }} + + - name: Prepare E2E tests + run: pnpm run test:prepare + shell: bash + working-directory: dev-packages/e2e-tests + + - name: Validate E2E test setup + run: pnpm run test:validate + shell: bash + working-directory: dev-packages/e2e-tests diff --git a/.github/actions/test-e2e-app/action.yml b/.github/actions/test-e2e-app/action.yml new file mode 100644 index 000000000000..716fa8282894 --- /dev/null +++ b/.github/actions/test-e2e-app/action.yml @@ -0,0 +1,50 @@ +name: 'Run E2E test application' +description: + 'Runs the tests of one application that `build-e2e-app` built, and uploads its Playwright traces if a test failed, + even if it passed on retry. Takes an entry of the `apps` list that `ci:build-matrix --batch-size` generates.' +inputs: + app: + description: 'The app as JSON: `dir` and the optional `runtime` and `assert-command`.' + required: true + +runs: + using: 'composite' + steps: + - name: Run E2E test + id: test + shell: bash + working-directory: ${{ runner.temp }}/e2e/${{ fromJSON(inputs.app).dir }} + run: | + status=0 + timeout 600 bash -c "$ASSERT_COMMAND" || status=$? + echo "status=$status" >> "$GITHUB_OUTPUT" + # The apps keep traces only of failed attempts (`retain-on-failure`) or of retries (`on-first-retry`), so traces + # after a passing run belong to a test that passed on retry. + if [ -n "$(find test-results -name 'trace.zip' -print -quit 2>/dev/null)" ]; then + echo "traces=true" >> "$GITHUB_OUTPUT" + fi + env: + ASSERT_COMMAND: ${{ fromJSON(inputs.app)['assert-command'] || 'pnpm test:assert' }} + SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }} + # See build-e2e-app + SENTRY_E2E_APP: ${{ fromJSON(inputs.app).dir }} + RUNTIME: ${{ fromJSON(inputs.app).runtime }} + + - name: Upload Playwright Traces + if: steps.test.outputs.status != '0' || steps.test.outputs.traces == 'true' + uses: actions/upload-artifact@v7 + with: + name: playwright-traces-job_e2e_playwright_tests-${{ fromJSON(inputs.app).dir }} + path: ${{ runner.temp }}/e2e/${{ fromJSON(inputs.app).dir }}/test-results + overwrite: true + retention-days: 7 + + - name: Fail if the tests failed + if: steps.test.outputs.status != '0' + shell: bash + run: | + echo "::error::$LABEL failed with exit code $STATUS" + exit 1 + env: + LABEL: ${{ fromJSON(inputs.app).label }} + STATUS: ${{ steps.test.outputs.status }} diff --git a/.github/scripts/nsc-report b/.github/scripts/nsc-report new file mode 100755 index 000000000000..ea2e26d16c8f --- /dev/null +++ b/.github/scripts/nsc-report @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 + +import argparse +import csv +import io +import json +import math +import subprocess +import sys + +from datetime import datetime, timedelta +from textwrap import wrap + +REPOSITORY = "getsentry/sentry-javascript" + + +def run_command(*command): + result = subprocess.run(command, capture_output=True, check=False, text=True) + if result.returncode: + error = result.stderr.strip() or result.stdout.strip() + raise RuntimeError(f"{command[0]} failed: {error}") + return result.stdout + + +# A named tuple keeps this valid before Python 3.14; ruff format strips the parentheses +# from an inline `except (A, B):` because the repo targets 3.14. +NUMBER_ERRORS = (TypeError, ValueError) + + +def number(value): + try: + return float(value) + except NUMBER_ERRORS: + return None + + +def headroom(used_share): + if used_share is None or used_share <= 0: + return "n/a" + return f"{(1 - used_share) * 100:.1f}% free" + + +def usage_bar(used_share, width=10): + if used_share is None or used_share <= 0: + return "░" * width + filled = round(min(max(used_share, 0), 1) * width) + return "█" * filled + "░" * (width - filled) + + +def parse_time(value): + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return datetime.strptime(value, "%Y-%m-%d %H:%M:%S %z UTC") + + +def report_end(run): + if run["status"] != "completed": + return datetime.now().astimezone().isoformat() + completed_jobs = [ + parse_time(job["completedAt"]) for job in run["jobs"] if job.get("completedAt") + ] + latest_completion = max(completed_jobs) if completed_jobs else parse_time(run["updatedAt"]) + return (latest_completion + timedelta(seconds=1)).isoformat() + + +def duration(started_at, completed_at): + seconds = round((parse_time(completed_at) - parse_time(started_at)).total_seconds()) + minutes, seconds = divmod(seconds, 60) + hours, minutes = divmod(minutes, 60) + if hours: + return f"{hours}h {minutes}m {seconds}s" + if minutes: + return f"{minutes}m {seconds}s" + return f"{seconds}s" + + +def compute_unit_minutes(instance): + started_at = instance.get("job_started_at") + completed_at = instance.get("job_completed_at") + cpu = number(instance.get("resources_cpu")) + memory_gb = number(instance.get("resources_ram_gb")) + if not started_at or not completed_at or not cpu or not memory_gb: + return None + + elapsed_seconds = (parse_time(completed_at) - parse_time(started_at)).total_seconds() + billable_minutes = max(1, math.ceil(max(0, elapsed_seconds - 15) / 60)) + units_per_minute = max(cpu, memory_gb / 2) + return units_per_minute * billable_minutes + + +def print_workflow(workflow, profiles, total_unit_minutes, missing_compute_jobs): + content_width = 76 + border = "─" * (content_width + 2) + + def line(value): + for paragraph in value.splitlines(): + for wrapped in wrap(paragraph, content_width) or [""]: + print(f"│ {wrapped:<{content_width}} │") + + print(f"┌{border}┐") + line(f"Workflow: {workflow}") + print(f"├{border}┤") + + for (profile, _, _), profile_data in sorted(profiles.items(), key=lambda item: str(item[0])): + job_count = profile_data["job_count"] + unit_minutes = profile_data["unit_minutes"] + partial = " · partial" if profile_data["missing_compute_jobs"] else "" + line(f"Runner: {profile} · {job_count} jobs · compute: {unit_minutes:.1f}m{partial}") + + for resource, capacity_key, peaks_key in ( + ("CPU", "cpu_capacity", "cpu_peaks"), + ("RAM", "memory_capacity", "memory_peaks"), + ): + peaks = profile_data[peaks_key] + if peaks: + peak_share, peak_value = max(peaks, key=lambda peak: peak[0]) + capacity = profile_data[capacity_key] + line( + f" {resource} {usage_bar(peak_share)} {peak_share:.0%} · " + f"{peak_value:.2f}/{capacity:.0f} · {headroom(peak_share)} · " + f"{len(peaks)}/{job_count}" + ) + else: + line(f" {resource} {usage_bar(None)} n/a · 0/{job_count} measured") + + line(" Jobs · bars scaled to the longest job") + jobs = sorted( + profile_data["jobs"], + key=lambda job: (job[2] is None, -(job[2] or 0), job[0]), + ) + longest_job_seconds = max((job[2] or 0 for job in jobs), default=0) + for job_name, job_time, job_seconds in jobs: + filled = ( + max(1, round(job_seconds / longest_job_seconds * 12)) + if job_seconds is not None and longest_job_seconds > 0 + else 0 + ) + bar = "█" * filled + "░" * (12 - filled) + prefix = f" {bar} {job_time:>8} " + job_lines = wrap(job_name, content_width - len(prefix)) or [""] + line(prefix + job_lines[0]) + for job_line in job_lines[1:]: + line(f"{' ' * len(prefix)}{job_line}") + + total_suffix = " (partial)" if missing_compute_jobs else "" + line(f"Total compute: {total_unit_minutes:.1f}m{total_suffix}") + print(f"└{border}┘") + + +def report_rows(report, run_id): + rows = list(csv.reader(io.StringIO(report))) + header_index = next( + (index for index, row in enumerate(rows) if row and row[0] == "instance_id"), + None, + ) + if header_index is None: + raise RuntimeError("nsc instance report did not return CSV data") + + headers = rows[header_index] + data_rows = [row for row in rows[header_index + 1 :] if len(row) == len(headers)] + if "github_run_id" not in headers: + if not data_rows: + return [] + raise RuntimeError("nsc instance report rows are missing GitHub run IDs") + + run_id_index = headers.index("github_run_id") + return [dict(zip(headers, row)) for row in data_rows if row[run_id_index] == run_id] + + +def latest_workflow_run(branch, workflow): + runs = json.loads( + run_command( + "gh", + "run", + "list", + "--repo", + REPOSITORY, + "--workflow", + workflow, + "--branch", + branch, + "--limit", + "1", + "--json", + "databaseId,workflowName,conclusion,status,url", + ) + ) + if not runs: + raise RuntimeError(f"no {workflow!r} runs found on branch {branch!r}") + return runs[0] + + +def main(): + parser = argparse.ArgumentParser( + description="Report Namespace usage for a Sentry JavaScript workflow run." + ) + parser.add_argument( + "workflow_run_id", + nargs="?", + help="Workflow run ID (default: latest workflow run)", + ) + parser.add_argument("--ref", help="Branch to inspect (default: checked-out branch)") + parser.add_argument( + "--workflow", + default="CI: Build & Test", + help="Workflow to inspect when selecting the latest run (default: CI: Build & Test)", + ) + args = parser.parse_args() + + if args.workflow_run_id: + if not args.workflow_run_id.isdigit(): + raise RuntimeError("workflow run ID must contain only digits") + if args.ref: + raise RuntimeError("--ref can only be used when selecting the latest run") + run_id = args.workflow_run_id + else: + branch = args.ref or run_command("git", "branch", "--show-current").strip() + if not branch: + raise RuntimeError("no current Git branch; pass --ref") + run = latest_workflow_run(branch, args.workflow) + run_id = str(run["databaseId"]) + run_summary = ( + f"Latest {run['workflowName']} run on {branch}: " + f"{run['url']} ({run['status']}" + f"{', ' + run['conclusion'] if run['conclusion'] else ''})" + ) + print("\n".join(wrap(run_summary, 80)), flush=True) + + cost_scope = "Cost estimate disabled: cache/storage, builds, and plan pricing aren't captured." + print("\n".join(wrap(cost_scope, 80))) + + run = json.loads( + run_command( + "gh", + "run", + "view", + run_id, + "--repo", + REPOSITORY, + "--json", + "status,createdAt,updatedAt,headSha,jobs", + ) + ) + print(f"Workflow SHA: {run['headSha']}") + + report = run_command( + "nsc", + "instance", + "report", + "--repository", + REPOSITORY, + "--start", + run["createdAt"], + "--end", + report_end(run), + "--out", + "-", + ) + instances = report_rows(report, run_id) + if not instances: + print(f"No Namespace runner instances found for run {run_id}.") + return + + workflows = {} + for instance in sorted(instances, key=lambda row: row.get("job_started_at", "")): + workflow = instance.get("github_job_workflow_name") or "unknown workflow" + profile = instance.get("profile", "unknown").removeprefix("namespace-profile-") + cpu_capacity = number(instance.get("resources_cpu")) + cpu_peak_share = number(instance.get("resources_cpu_actual_max")) + memory_capacity = number(instance.get("resources_ram_gb")) + memory_peak_share = number(instance.get("resources_ram_gb_actual_max_percent")) + + job_time = "n/a" + job_seconds = None + if instance.get("job_started_at") and instance.get("job_completed_at"): + job_time = duration(instance["job_started_at"], instance["job_completed_at"]) + job_seconds = ( + parse_time(instance["job_completed_at"]) - parse_time(instance["job_started_at"]) + ).total_seconds() + + unit_minutes = compute_unit_minutes(instance) + workflow_data = workflows.setdefault( + workflow, + {"profiles": {}, "unit_minutes": 0, "missing_compute_jobs": 0}, + ) + profile_key = (profile, cpu_capacity, memory_capacity) + profile_data = workflow_data["profiles"].setdefault( + profile_key, + { + "job_count": 0, + "cpu_capacity": cpu_capacity, + "memory_capacity": memory_capacity, + "cpu_peaks": [], + "memory_peaks": [], + "jobs": [], + "unit_minutes": 0, + "missing_compute_jobs": 0, + }, + ) + profile_data["job_count"] += 1 + profile_data["jobs"].append( + (instance.get("github_job_name", "unknown job"), job_time, job_seconds) + ) + + if cpu_capacity and cpu_peak_share and cpu_peak_share > 0: + profile_data["cpu_peaks"].append((cpu_peak_share, cpu_capacity * cpu_peak_share)) + if memory_capacity and memory_peak_share and memory_peak_share > 0: + profile_data["memory_peaks"].append( + (memory_peak_share, memory_capacity * memory_peak_share) + ) + + if unit_minutes is None: + workflow_data["missing_compute_jobs"] += 1 + profile_data["missing_compute_jobs"] += 1 + else: + workflow_data["unit_minutes"] += unit_minutes + profile_data["unit_minutes"] += unit_minutes + + for workflow, data in sorted(workflows.items()): + print_workflow( + workflow, + data["profiles"], + data["unit_minutes"], + data["missing_compute_jobs"], + ) + + +if __name__ == "__main__": + try: + main() + except (KeyError, RuntimeError, json.JSONDecodeError) as error: + print(error, file=sys.stderr) + sys.exit(1) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e202edc68667..e9eeedfb9148 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -140,7 +140,7 @@ jobs: - name: Determine which test applications should be run id: matrix run: - pnpm --silent run ci:build-matrix --base=${{ (github.event_name == 'pull_request' && + pnpm --silent run ci:build-matrix --batch-size=4 --base=${{ (github.event_name == 'pull_request' && needs.job_get_metadata.outputs.changed_ci != 'true' && github.event.pull_request.base.sha) || '' }} >> $GITHUB_OUTPUT working-directory: dev-packages/e2e-tests @@ -362,11 +362,18 @@ jobs: job_check_lockfile: name: Check lockfile needs: [job_get_metadata] - # `pnpm dedupe --check` fetches metadata for every package from npm, so this job can take 8+ min on PRs. + # Both checks only resolve dependencies, so they need no node_modules and little CPU. What they need is pnpm's + # package metadata: `pnpm dedupe --check` reads it for every dependency, about 300 MB from npm (1.2 GB unpacked). + # Without a cache, that download took 6 minutes or more. The job's own cache volume keeps the metadata, so pnpm + # only revalidates it. The timeout leaves room for a cold volume. timeout-minutes: 20 - runs-on: namespace-profile-sentry-javascript + runs-on: + namespace-profile-sentry-javascript-small;overrides.cache-tag=sentry-javascript-lockfile${{ github.event_name == + 'pull_request' && '-pr' || '' }} + # Also on CI changes, which can break this job without touching a dependency if: | needs.job_get_metadata.outputs.changed_deps == 'true' || + needs.job_get_metadata.outputs.changed_ci == 'true' || needs.job_get_metadata.outputs.is_base_branch == 'true' || needs.job_get_metadata.outputs.is_release == 'true' steps: @@ -374,16 +381,18 @@ jobs: uses: actions/checkout@v7 with: ref: ${{ env.HEAD_COMMIT }} + - name: Mount pnpm caches + uses: ./.github/actions/mount-pnpm-store - name: Install pnpm uses: ./.github/actions/install-pnpm - name: Set up Node uses: actions/setup-node@v7 with: node-version-file: 'package.json' - - name: Install dependencies - run: pnpm install --no-frozen-lockfile + # `--lockfile-only` updates pnpm-lock.yaml like a full install would, without downloading and linking packages - name: Check that pnpm-lock.yaml is stable run: | + pnpm install --lockfile-only --no-frozen-lockfile if ! git diff --exit-code pnpm-lock.yaml; then echo "::error::pnpm-lock.yaml has uncommitted changes after running 'pnpm install'. Please run 'pnpm install' locally and commit the updated pnpm-lock.yaml." exit 1 @@ -600,14 +609,21 @@ jobs: env: NODE_VERSION: ${{ matrix.node }} + - name: Run script tests + run: pnpm run test:scripts + if: matrix.node == 24 + job_browser_playwright_tests: name: Playwright ${{ matrix.bundle }}${{ matrix.project && matrix.project != 'chromium' && format(' {0}', matrix.project) || ''}}${{ matrix.shard && format(' ({0}/{1})', matrix.shard, matrix.shards) || ''}} Tests needs: [job_get_metadata, job_build, job_build_bundles] if: needs.job_build.outputs.changed_browser_integration == 'true' || github.event_name != 'pull_request' + # The bundle matrices use 2.5-4.7 of 8 vCPUs and up to about 6 of 16 GB, so they fit the 4 vCPU, 8 GB shape. The + # esm shards and the webkit bundle need the large shape. runs-on: - namespace-profile-sentry-javascript-large;overrides.cache-tag=sentry-javascript${{ github.event_name == + ${{ (matrix.bundle == 'esm' || matrix.project == 'webkit') && 'namespace-profile-sentry-javascript-large' || + 'namespace-profile-sentry-javascript' }};overrides.cache-tag=sentry-javascript${{ github.event_name == 'pull_request' && '-pr' || '' }} timeout-minutes: 25 strategy: @@ -689,9 +705,22 @@ jobs: pnpm run test:all${{ matrix.project && format(' --project={0}', matrix.project) || '' }}${{ matrix.shard && format(' --shard={0}/{1}', matrix.shard, matrix.shards) || '' }} + - &report_browser_flaky_tests + name: Report flaky tests + id: flaky + if: ${{ !cancelled() }} + run: node scripts/ci-report-flaky-tests.mjs dev-packages/browser-integration-tests/results.json + + # Only runs if a test passed on retry. scripts/weekly-flaky-tests.mjs looks for this step name. + - &flaky_tests_passed_on_retry + name: Flaky tests passed on retry + if: ${{ !cancelled() && steps.flaky.outputs.flaky == 'true' }} + run: echo "::warning::Some tests only passed on retry, see the job summary." + + # Also for tests that passed on retry: the trace of their failed attempt is what explains the flake. - name: Upload Playwright Traces uses: actions/upload-artifact@v7 - if: failure() + if: failure() || steps.flaky.outputs.flaky == 'true' with: name: playwright-traces-job_browser_playwright_tests-${{ matrix.bundle}}-${{matrix.project}}-${{matrix.shard || @@ -751,9 +780,12 @@ jobs: cd dev-packages/browser-integration-tests pnpm run test:loader + - *report_browser_flaky_tests + - *flaky_tests_passed_on_retry + - name: Upload Playwright Traces uses: actions/upload-artifact@v7 - if: failure() + if: failure() || steps.flaky.outputs.flaky == 'true' with: name: playwright-traces-job_browser_loader_tests-${{ matrix.bundle}} path: dev-packages/browser-integration-tests/test-results @@ -798,10 +830,45 @@ jobs: - name: Restore caches uses: ./.github/actions/restore-cache + # Suites `npm install` their `additionalDependencies` in their setup hook, and slow npm downloads were the most + # common reason for these jobs to fail. The Actions cache keeps those packages when the Namespace cache volume + # comes up empty. The key only changes weekly, so a push that edits a suite doesn't save another copy, and npm + # versions the suites no longer use don't pile up. + - name: Get npm cache key of the suites + id: npm-cache + run: | + echo "dir=$HOME/.cache/node-integration-npm" >> "$GITHUB_OUTPUT" + echo "key=node-integration-npm-shard${{ matrix.shard }}-$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT" + - name: Restore npm cache of the suites + id: npm-cache-restore + uses: actions/cache/restore@v6.1.0 + with: + path: ${{ steps.npm-cache.outputs.dir }} + key: ${{ steps.npm-cache.outputs.key }} + # Only the base branch saves, so PR caches don't evict it. The Node versions install the same packages, so one + # of them is enough. Without a restored cache, the other jobs keep the npm cache on the Namespace cache volume. + - name: Use the npm cache of the suites + id: npm-cache-use + if: + steps.npm-cache-restore.outputs.cache-hit == 'true' || (needs.job_get_metadata.outputs.is_base_branch == + 'true' && matrix.node == 24) + run: echo "npm_config_cache=$NPM_CACHE_DIR" >> "$GITHUB_ENV" + env: + NPM_CACHE_DIR: ${{ steps.npm-cache.outputs.dir }} + - name: Run integration tests working-directory: dev-packages/node-integration-tests run: pnpm run test --shard=${{ matrix.shard }}/3 + - name: Save npm cache of the suites + if: + ${{ !cancelled() && steps.npm-cache-use.outcome == 'success' && steps.npm-cache-restore.outputs.cache-hit != + 'true' && needs.job_get_metadata.outputs.is_base_branch == 'true' && matrix.node == 24 }} + uses: actions/cache/save@v6.1.0 + with: + path: ${{ steps.npm-cache.outputs.dir }} + key: ${{ steps.npm-cache.outputs.key }} + # Runs on the wrangler pinned in the package, so the required run stays on a fixed, older runtime. job_cloudflare_integration_tests: name: Cloudflare Integration Tests (${{ matrix.shard }}/2) @@ -1020,17 +1087,18 @@ jobs: overwrite: true job_e2e_tests: - name: E2E ${{ matrix.label || matrix.test-application }} Test + name: E2E ${{ matrix.label }} Test # We need to add the `always()` check here because the previous step has this as well :( # See: https://github.com/actions/runner/issues/2205 if: always() && needs.job_build_tarballs.result == 'success' && needs.job_build.outputs.e2e-matrix !='{"include":[]}' needs: [job_get_metadata, job_build, job_build_layer, job_build_tarballs] - # Most apps finish in under a minute on 4 vCPUs, so they run on 2. Apps that take longer set `sentryTest.runner`. + # Most apps finish in under a minute on 4 vCPUs, so they run on 2, four apps per job (see `--batch-size` in + # `getTestMatrix.mjs`). Apps that take longer set `sentryTest.runner` and get a job of their own. runs-on: ${{ matrix.runner || 'namespace-profile-sentry-javascript-small' }};overrides.cache-tag=sentry-javascript-e2e${{ github.event_name == 'pull_request' && '-pr' || '' }} - timeout-minutes: 15 + timeout-minutes: ${{ matrix.runner && 25 || 40 }} env: # We just use a dummy DSN here, only send to the tunnel anyhow E2E_TEST_DSN: 'https://username@domain/123' @@ -1040,8 +1108,6 @@ jobs: REACT_APP_E2E_TEST_DSN: 'https://username@domain/123' E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks' E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests' - # The `runtime` of a runtime variant, see dev-packages/e2e-tests/README.md - RUNTIME: ${{ matrix.runtime }} # Without the root node_modules, `pnpm run` would first install the whole workspace. pnpm_config_verify_deps_before_run: 'false' strategy: @@ -1052,66 +1118,126 @@ jobs: uses: actions/checkout@v7 with: ref: ${{ env.HEAD_COMMIT }} - - name: Prepare E2E test application - uses: ./.github/actions/prepare-e2e-test + - name: Set up E2E tests + uses: ./.github/actions/setup-e2e-tests with: - test-application: ${{ matrix.test-application }} - runtime: ${{ matrix.runtime }} - deno-version: ${{ matrix.deno-version }} + aws-serverless-layer: ${{ matrix.aws-serverless-layer }} + aws-serverless-build: ${{ matrix.aws-serverless-build }} - # Installs Playwright while the app installs and builds. Keep it right before the install: the runner sets up a - # background composite action's steps from the job's shared `github` context, which fails with "Collection was - # modified" if a foreground step starts at that moment. Starting next to `download-artifact` also made it fail to - # read the event file. + # Installs Playwright while the first app installs and builds. The apps of a job use the same Playwright version. + # Keep a `run` step right after it: the runner sets up a background composite action's steps from the job's shared + # `github` context, which fails with "Collection was modified" if a foreground step starts at that moment. Starting + # next to `download-artifact` also made it fail to read the event file. - name: Install Playwright id: install-playwright background: true uses: ./.github/actions/install-playwright with: browsers: chromium - cwd: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }} - - # Every build command runs `pnpm install`, so installing first makes that a no-op. Some downloads from the npm - # registry crawl at tens of kB/s for the whole 7 minutes of the build step, and pnpm does not give up on a slow - # download. Another attempt reuses what is already in the store and opens new connections. - # The hoisted variant installs with another node linker, so installing first would only redo the install. - - name: Install E2E app dependencies - if: ${{ !contains(matrix.build-command, 'hoisted') }} - working-directory: ${{ runner.temp }}/test-application - timeout-minutes: 7 - run: | - for attempt in 1 2; do - timeout 150 pnpm install && exit 0 - echo "::warning::pnpm install attempt $attempt failed or did not finish in 150 s, retrying" - done - pnpm install - env: - SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }} + cwd: dev-packages/e2e-tests/test-applications/${{ matrix.playwright-app }} - - name: Build E2E app - working-directory: ${{ runner.temp }}/test-application - timeout-minutes: 7 - run: ${{ matrix.build-command || 'pnpm test:build' }} + - name: List the apps of this job + run: echo "$APPS" env: - SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }} + APPS: ${{ toJSON(matrix.apps) }} + + # Each app runs in its own steps. A failed app does not stop the next one, and `Check E2E results` fails the job. + - name: Build ${{ matrix.apps[0].label }} + id: build-0 + continue-on-error: true + timeout-minutes: 15 + uses: ./.github/actions/build-e2e-app + with: + app: ${{ toJSON(matrix.apps[0]) }} - wait: [install-playwright] - - name: Run E2E test - working-directory: ${{ runner.temp }}/test-application - timeout-minutes: 10 - run: ${{ matrix.assert-command || 'pnpm test:assert' }} + - name: Test ${{ matrix.apps[0].label }} + id: test-0 + if: steps.build-0.outcome == 'success' + continue-on-error: true + timeout-minutes: 11 + uses: ./.github/actions/test-e2e-app + with: + app: ${{ toJSON(matrix.apps[0]) }} + + - name: Build ${{ matrix.apps[1].label }} + id: build-1 + if: matrix.apps[1] + continue-on-error: true + timeout-minutes: 15 + uses: ./.github/actions/build-e2e-app + with: + app: ${{ toJSON(matrix.apps[1]) }} + previous-app: ${{ toJSON(matrix.apps[0]) }} + + - name: Test ${{ matrix.apps[1].label }} + id: test-1 + if: steps.build-1.outcome == 'success' + continue-on-error: true + timeout-minutes: 11 + uses: ./.github/actions/test-e2e-app + with: + app: ${{ toJSON(matrix.apps[1]) }} + + - name: Build ${{ matrix.apps[2].label }} + id: build-2 + if: matrix.apps[2] + continue-on-error: true + timeout-minutes: 15 + uses: ./.github/actions/build-e2e-app + with: + app: ${{ toJSON(matrix.apps[2]) }} + previous-app: ${{ toJSON(matrix.apps[1]) }} + + - name: Test ${{ matrix.apps[2].label }} + id: test-2 + if: steps.build-2.outcome == 'success' + continue-on-error: true + timeout-minutes: 11 + uses: ./.github/actions/test-e2e-app + with: + app: ${{ toJSON(matrix.apps[2]) }} + + - name: Build ${{ matrix.apps[3].label }} + id: build-3 + if: matrix.apps[3] + continue-on-error: true + timeout-minutes: 15 + uses: ./.github/actions/build-e2e-app + with: + app: ${{ toJSON(matrix.apps[3]) }} + previous-app: ${{ toJSON(matrix.apps[2]) }} + + - name: Test ${{ matrix.apps[3].label }} + id: test-3 + if: steps.build-3.outcome == 'success' + continue-on-error: true + timeout-minutes: 11 + uses: ./.github/actions/test-e2e-app + with: + app: ${{ toJSON(matrix.apps[3]) }} + + - name: Report flaky tests + id: flaky + if: ${{ !cancelled() }} + run: node scripts/ci-report-flaky-tests.mjs "$RUNNER_TEMP"/e2e/*/results.json + + - *flaky_tests_passed_on_retry + + - name: Check E2E results + if: ${{ !cancelled() }} + run: node dev-packages/e2e-tests/lib/checkJobResults.mjs env: - SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }} - - - name: Upload Playwright Traces - uses: actions/upload-artifact@v7 - if: failure() - with: - name: playwright-traces-job_e2e_playwright_tests-${{ matrix.test-application}} - path: ${{ runner.temp }}/test-application/test-results - overwrite: true - retention-days: 7 + APPS: ${{ toJSON(matrix.apps) }} + BUILD_0: ${{ steps.build-0.outcome }} + TEST_0: ${{ steps.test-0.outcome }} + BUILD_1: ${{ steps.build-1.outcome }} + TEST_1: ${{ steps.test-1.outcome }} + BUILD_2: ${{ steps.build-2.outcome }} + TEST_2: ${{ steps.test-2.outcome }} + BUILD_3: ${{ steps.build-3.outcome }} + TEST_3: ${{ steps.test-3.outcome }} # - We skip optional tests on release branches # - The tests run in optional-e2e.yml, so their failures do not mark the PR as failed. This job only waits for diff --git a/.github/workflows/warm-e2e-cache.yml b/.github/workflows/warm-e2e-cache.yml new file mode 100644 index 000000000000..ad64631d2bb6 --- /dev/null +++ b/.github/workflows/warm-e2e-cache.yml @@ -0,0 +1,126 @@ +name: 'CI: Warm E2E caches' + +# The E2E jobs share one cache volume per tag, and the last job to finish overwrites what the others saved, so the pnpm +# store on it never holds the packages of every app. Downloads that the store misses are what crawls in the E2E builds. +# This job installs every test application into the store once, and later jobs fork the volume it commits. +# The volume has to stay under its size limit, because Namespace does not clone a cache that outgrew it. `pnpm store +# prune` cannot do that here: it keeps only packages that are hard-linked into a project, and the apps are installed +# outside the cache volume, so their files are copies and prune would empty the store. Instead, the job starts from an +# empty store when the volume is mostly full. + +on: + workflow_dispatch: + schedule: + - cron: '0 4 * * *' + +permissions: + contents: read + # To download the tarballs of the build.yml run + actions: read + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + job_find_build: + name: Find the latest develop build + runs-on: namespace-profile-sentry-javascript-small + timeout-minutes: 5 + outputs: + run-id: ${{ steps.build.outputs.run-id }} + steps: + - name: Find a develop build.yml run with tarballs + id: build + uses: actions/github-script@v9 + with: + retries: 3 + script: | + const { data } = await github.rest.actions.listWorkflowRuns({ + ...context.repo, + workflow_id: 'build.yml', + branch: 'develop', + event: 'push', + status: 'completed', + per_page: 30, + }); + for (const run of data.workflow_runs) { + const { data: artifacts } = await github.rest.actions.listWorkflowRunArtifacts({ + ...context.repo, + run_id: run.id, + per_page: 100, + }); + const names = new Set(artifacts.artifacts.filter(artifact => !artifact.expired).map(artifact => artifact.name)); + if (names.has('build-tarball-output') && names.has('build-test-utils-tarball-output')) { + core.info(`Using the tarballs of ${run.html_url}`); + core.setOutput('run-id', run.id); + return; + } + } + core.setFailed('None of the last 30 develop build.yml runs has tarball artifacts.'); + + job_warm_cache: + name: Warm ${{ matrix.cache-tag }} + needs: [job_find_build] + # The profile and tags of the E2E jobs in build.yml and optional-e2e.yml + runs-on: namespace-profile-sentry-javascript;overrides.cache-tag=${{ matrix.cache-tag }} + timeout-minutes: 90 + env: + # Without the root node_modules, `pnpm run` would first install the whole workspace, like in the E2E jobs. + pnpm_config_verify_deps_before_run: 'false' + strategy: + fail-fast: false + matrix: + cache-tag: [sentry-javascript-e2e, sentry-javascript-e2e-pr] + steps: + # The test applications of this commit with the tarballs of a slightly older one. An app that needs a package the + # tarballs do not have yet fails to install and only gets a warning. + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up E2E tests + uses: ./.github/actions/setup-e2e-tests + with: + artifacts-run-id: ${{ needs.job_find_build.outputs.run-id }} + github-token: ${{ github.token }} + + - name: Install Playwright + uses: ./.github/actions/install-playwright + with: + browsers: chromium + cwd: dev-packages/test-utils + + - name: Empty the pnpm store if the cache volume is mostly full + run: | + df -h "$PNPM_CONFIG_STORE_DIR" + echo "pnpm store: $(du -sh "$PNPM_CONFIG_STORE_DIR" | cut -f1)" + used=$(df --output=pcent "$PNPM_CONFIG_STORE_DIR" | tail -1 | tr -dc '0-9') + if [ "$used" -ge "$MAX_USED_PERCENT" ]; then + echo "::notice::The cache volume is $used% full, so the pnpm store starts empty." + find "$PNPM_CONFIG_STORE_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + fi + env: + MAX_USED_PERCENT: 70 + + - name: Install every test application + working-directory: dev-packages/e2e-tests + run: node lib/warmCache.mjs install "$RUNNER_TEMP/warm" 4 + + # Proves that the store alone holds what the apps need, which is what later E2E jobs rely on. + - name: Reinstall every test application from the store alone + working-directory: dev-packages/e2e-tests + run: node lib/warmCache.mjs verify "$RUNNER_TEMP/warm" 4 + + - name: Report the size of the pnpm store + if: ${{ !cancelled() }} + run: | + df -h "$PNPM_CONFIG_STORE_DIR" + echo "pnpm store: $(du -sh "$PNPM_CONFIG_STORE_DIR" | cut -f1)" + + - name: Check that the workspace was not installed + run: | + if [ -e node_modules ]; then + echo "::error::The workspace root has a node_modules folder, so the job installed the whole workspace." + exit 1 + fi diff --git a/dev-packages/browser-integration-tests/.gitignore b/dev-packages/browser-integration-tests/.gitignore index b258dda7330c..bf71095ac0b2 100644 --- a/dev-packages/browser-integration-tests/.gitignore +++ b/dev-packages/browser-integration-tests/.gitignore @@ -1,3 +1,4 @@ test-results tmp *.junit.xml +results.json diff --git a/dev-packages/browser-integration-tests/playwright.config.ts b/dev-packages/browser-integration-tests/playwright.config.ts index 681de57d4e59..b5a96e94be0f 100644 --- a/dev-packages/browser-integration-tests/playwright.config.ts +++ b/dev-packages/browser-integration-tests/playwright.config.ts @@ -2,7 +2,9 @@ import type { PlaywrightTestConfig } from '@playwright/test'; import { devices } from '@playwright/test'; const config: PlaywrightTestConfig = { - retries: 0, + // A test that passes on retry is reported as flaky in the job summary and in the weekly flaky test report. + // `test:detect-flaky` passes `--retries=0`, so it still fails on the first flaky run. + retries: process.env.CI ? 1 : 0, // Run tests inside of a single file in parallel fullyParallel: true, // Use 3 workers on CI, else use defaults (based on available CPU cores) @@ -30,7 +32,9 @@ const config: PlaywrightTestConfig = { }, ], - reporter: process.env.CI ? [['list'], ['github'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list', + reporter: process.env.CI + ? [['list'], ['github'], ['junit', { outputFile: 'results.junit.xml' }], ['json', { outputFile: 'results.json' }]] + : 'list', globalSetup: require.resolve('./playwright.setup.ts'), globalTeardown: require.resolve('./playwright.teardown.ts'), diff --git a/dev-packages/browser-integration-tests/scripts/detectFlakyTests.ts b/dev-packages/browser-integration-tests/scripts/detectFlakyTests.ts index 21f666adeb0e..5ad3eaf1806a 100644 --- a/dev-packages/browser-integration-tests/scripts/detectFlakyTests.ts +++ b/dev-packages/browser-integration-tests/scripts/detectFlakyTests.ts @@ -48,7 +48,7 @@ ${changedPaths.join('\n')} const cp = childProcess.spawn( `npx playwright test ${ testPaths.length ? testPaths.join(' ') : './suites' - } --repeat-each ${repeatEachCount} --project=chromium`, + } --repeat-each ${repeatEachCount} --retries=0 --project=chromium`, { shell: true, cwd, stdio: 'inherit' }, ); diff --git a/dev-packages/e2e-tests/README.md b/dev-packages/e2e-tests/README.md index d71ba9b1b8a3..f6b6dfb3247d 100644 --- a/dev-packages/e2e-tests/README.md +++ b/dev-packages/e2e-tests/README.md @@ -280,9 +280,13 @@ For example, if something is changed in the browser package, only E2E test apps You can add additional information about the test (e.g. canary versions, optional in CI) by adding `sentryTest` in the `package.json` of a test application. -The required E2E jobs run on 2 vCPUs (`namespace-profile-sentry-javascript-small`). If an app's job takes longer than -about 90 seconds, set `"sentryTest": { "runner": "namespace-profile-sentry-javascript" }` to run it and its variants on -4 vCPUs. +The required E2E jobs run on 2 vCPUs (`namespace-profile-sentry-javascript-small`), four apps or variants per job, one +after another in `$RUNNER_TEMP/e2e/