ci: batch short e2e apps (+re-rightsize), prewarm e2e caches, retry flaky Playwright tests

.github/actions/build-e2e-app/action.yml

@@ -0,0 +1,95 @@
+name: 'Build E2E test application'
+description:
+  'Prepares, installs and builds one application of an E2E job in `runner.temp/e2e/<dir>`. Takes an entry of the `apps`
+  list that `ci:build-matrix --batch-size` generates.'
+inputs:
+  app:
+    description:
+      'The app as JSON: `test-application`, `dir` and the optional `runtime`, `deno-version`, `build-command`.'
+    required: true
+  previous-app:
+    description:
+      'The app that ran before in the same job, as JSON. Its leftover processes and containers are removed first.'
+    default: ''
+
+# Isolation between the apps of a job: the install, build and test steps of an app run with `SENTRY_E2E_APP=<dir>`,
+# which every process they start inherits, also servers that Playwright detaches. Before the next app starts, the
+# processes with the previous app's marker are stopped, and the containers created since the previous app started are
+# removed. This also covers an app whose build or test step timed out. The last app of a job needs no clean-up.
+runs:
+  using: 'composite'
+  steps:
+    # Playwright stops the web servers of an app and the apps stop their containers in their global teardown. This only
+    # matters when an app timed out before that, and its servers would block the ports of the next app.
+    - name: Stop leftovers of the previous app
+      if: inputs.previous-app != ''
+      shell: bash
+      run: |
+        pids=()
+        for proc in /proc/[0-9]*; do
+          if tr '\0' '\n' < "$proc/environ" 2>/dev/null | grep -qxF "SENTRY_E2E_APP=$PREVIOUS_DIR"; then
+            pids+=("${proc#/proc/}")
+            echo "Stopping $(tr '\0' ' ' < "$proc/cmdline" 2>/dev/null)"
+          fi
+        done
+        if [ ${#pids[@]} -gt 0 ]; then
+          kill -TERM "${pids[@]}" 2>/dev/null || true
+          sleep 5
+          kill -KILL "${pids[@]}" 2>/dev/null || true
+        fi
+
+        before="$RUNNER_TEMP/e2e/$PREVIOUS_DIR.containers"
+        if [ -f "$before" ]; then
+          containers=$(comm -13 "$before" <(docker ps -aq --no-trunc 2>/dev/null | sort))
+          if [ -n "$containers" ]; then
+            echo "Removing containers: $containers"
+            docker rm -f $containers > /dev/null || true
+          fi
+        fi
+      env:
+        PREVIOUS_DIR: ${{ inputs.previous-app && fromJSON(inputs.previous-app).dir || '' }}
+
+    - name: Record the containers that exist before the app
+      shell: bash
+      run: |
+        mkdir -p "$RUNNER_TEMP/e2e"
+        docker ps -aq --no-trunc 2>/dev/null | sort > "$RUNNER_TEMP/e2e/$APP_DIR.containers" || true
+      env:
+        APP_DIR: ${{ fromJSON(inputs.app).dir }}
+
+    - name: Prepare E2E test application
+      uses: ./.github/actions/prepare-e2e-app
+      with:
+        test-application: ${{ fromJSON(inputs.app)['test-application'] }}
+        dir: ${{ runner.temp }}/e2e/${{ fromJSON(inputs.app).dir }}
+        runtime: ${{ fromJSON(inputs.app).runtime }}
+        deno-version: ${{ fromJSON(inputs.app)['deno-version'] }}
+
+    # Every build command runs `pnpm install`, so installing first makes that a no-op. Some downloads from the npm
+    # registry crawl at tens of kB/s for the whole 7 minutes of the build step, and pnpm does not give up on a slow
+    # download. Another attempt reuses what is already in the store and opens new connections.
+    # The hoisted variant installs with another node linker, so installing first would only redo the install.
+    - name: Install E2E app dependencies
+      if: ${{ !contains(fromJSON(inputs.app)['build-command'], 'hoisted') }}
+      shell: bash
+      working-directory: ${{ runner.temp }}/e2e/${{ fromJSON(inputs.app).dir }}
+      run: |
+        for attempt in 1 2; do
+          timeout 150 pnpm install && exit 0
+          echo "::warning::pnpm install attempt $attempt failed or did not finish in 150 s, retrying"
+        done
+        timeout 300 pnpm install
+      env:
+        SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }}
+        SENTRY_E2E_APP: ${{ fromJSON(inputs.app).dir }}
+        RUNTIME: ${{ fromJSON(inputs.app).runtime }}
+
+    - name: Build E2E app
+      shell: bash
+      working-directory: ${{ runner.temp }}/e2e/${{ fromJSON(inputs.app).dir }}
+      run: timeout 420 bash -c "$BUILD_COMMAND"
+      env:
+        BUILD_COMMAND: ${{ fromJSON(inputs.app)['build-command'] || 'pnpm test:build' }}
+        SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }}
+        SENTRY_E2E_APP: ${{ fromJSON(inputs.app).dir }}
+        RUNTIME: ${{ fromJSON(inputs.app).runtime }}

.github/actions/install-dependencies/action.yml

@@ -58,7 +58,14 @@ runs:
       shell: bash
 
     # The cache volume can hold node_modules of another commit or branch. This is a no-op when they match
-    # pnpm-lock.yaml.
+    # pnpm-lock.yaml. On an empty cache volume, single downloads from the npm registry can crawl at kB/s for longer
+    # than the job's timeout, and pnpm does not give up on them. Another attempt reuses what is already in the store
+    # and opens new connections.
     - name: Install dependencies
-      run: pnpm install --frozen-lockfile
+      run: |
+        for attempt in 1 2; do
+          timeout 300 pnpm install --frozen-lockfile && exit 0
+          echo "::warning::pnpm install attempt $attempt failed or did not finish in 300 s, retrying"
+        done
+        timeout 600 pnpm install --frozen-lockfile
       shell: bash

.github/actions/install-playwright/action.yml

@@ -34,7 +34,7 @@ runs:
           playwright
           apt
 
-    # `--with-deps` names packages that the runner image already has (glib, mesa, nss, ...), and `apt-get install` would
+    # `install-deps` names packages that the runner image already has (glib, mesa, nss, ...), and `apt-get install` would
     # upgrade them to the latest Ubuntu update. That is half of the download on a cold cache, and the image versions work.
     - name: Keep installed apt packages
       if: steps.playwright-version.outputs.version != ''
@@ -43,11 +43,40 @@ runs:
 
     # Downloads only the browsers and apt packages that are not on the cache volume yet. `npx` runs outside the repo,
     # so it can't pick up a `playwright` that a parallel `pnpm install` is still linking.
+    # A browser download can crawl for minutes, so an attempt that takes too long is retried. apt is not stopped
+    # midway, so it can't leave half-installed packages behind.
+    # On the runner image, the only packages `install-deps` installs for Chromium are fonts, and no test compares
+    # rendered pixels. On an empty cache volume, their download crawled for up to 10 minutes, so Chromium skips them
+    # unless a library is missing. WebKit and Firefox need libraries the image does not have.
     - name: Install Playwright
       if: steps.playwright-version.outputs.version != ''
       env:
         PLAYWRIGHT_BROWSERS: ${{ inputs.browsers || 'chromium webkit firefox' }}
         PLAYWRIGHT_VERSION: ${{ steps.playwright-version.outputs.version }}
-      run: npx -y "playwright@$PLAYWRIGHT_VERSION" install --with-deps $PLAYWRIGHT_BROWSERS
+      run: |
+        read -ra browsers <<< "$PLAYWRIGHT_BROWSERS"
+        playwright=(npx -y "playwright@$PLAYWRIGHT_VERSION")
+        installed=false
+        for attempt in 1 2; do
+          if timeout 300 "${playwright[@]}" install "${browsers[@]}"; then
+            installed=true
+            break
+          fi
+          echo "::warning::playwright install attempt $attempt failed or did not finish in 300 s, retrying"
+        done
+        if [[ "$installed" != true ]]; then
+          timeout 600 "${playwright[@]}" install "${browsers[@]}"
+        fi
+
+        if [[ "${browsers[*]}" == chromium ]]; then
+          missing=$(find ~/.cache/ms-playwright -type f \( -name chrome -o -name chrome-headless-shell \) \
+            -exec ldd {} + 2> /dev/null | grep 'not found' || true)
+          if [[ -z "$missing" ]]; then
+            exit 0
+          fi
+          echo "Chromium is missing libraries, installing its system dependencies:"
+          echo "$missing"
+        fi
+        "${playwright[@]}" install-deps "${browsers[@]}"
       shell: bash
       working-directory: ${{ runner.temp }}

.github/actions/install-pnpm/action.yml

@@ -4,5 +4,76 @@ description: 'Installs the pnpm version of `packageManager` in the root package.
 runs:
   using: 'composite'
   steps:
-    # Without a `version` input, the action reads the version from `packageManager`. With both, it fails.
-    - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
+    # `pnpm/action-setup` installs a bootstrap pnpm with npm, which then downloads the `packageManager` pnpm. Every job
+    # repeated both downloads, and neither gives up on a download that crawls. This keeps the standalone pnpm binary in
+    # the Actions cache, and aborts and retries a download that stays too slow.
+    - name: Get pnpm version
+      id: pnpm
+      shell: bash
+      run: |
+        version=$(node -p 'require("./package.json").packageManager.replace(/^pnpm@/, "").split("+")[0]')
+        case "$RUNNER_ARCH" in
+          X64) arch=x64 ;;
+          ARM64) arch=arm64 ;;
+          *) echo "::error::No standalone pnpm for runner architecture $RUNNER_ARCH" && exit 1 ;;
+        esac
+        {
+          echo "version=$version"
+          echo "package=@pnpm/exe.linux-$arch"
+          echo "key=pnpm-exe-linux-$arch-$version"
+          echo "dir=$HOME/.pnpm-exe/$version-linux-$arch"
+        } >> "$GITHUB_OUTPUT"
+
+    - name: Restore pnpm from the Actions cache
+      id: cache
+      uses: actions/cache/restore@v6.1.0
+      with:
+        path: ${{ steps.pnpm.outputs.dir }}
+        key: ${{ steps.pnpm.outputs.key }}
+
+    - name: Download pnpm
+      if: steps.cache.outputs.cache-hit != 'true'
+      shell: bash
+      env:
+        PNPM_DIR: ${{ steps.pnpm.outputs.dir }}
+        PNPM_PACKAGE: ${{ steps.pnpm.outputs.package }}
+        PNPM_VERSION: ${{ steps.pnpm.outputs.version }}
+      run: |
+        fetch() {
+          curl --fail --silent --show-error --location --retry 4 --retry-all-errors --connect-timeout 10 "$@"
+        }
+        manifest=$(fetch "https://registry.npmjs.org/$PNPM_PACKAGE/$PNPM_VERSION")
+        tarball=$(node -p 'JSON.parse(process.argv[1]).dist.tarball' "$manifest")
+        integrity=$(node -p 'JSON.parse(process.argv[1]).dist.integrity' "$manifest")
+        # Aborts a transfer that stays under 256 kB/s for 15 s, so a retry can open a faster connection. If every
+        # connection is that slow, the last download takes whatever speed it gets.
+        if ! fetch --speed-limit 262144 --speed-time 15 --output "$RUNNER_TEMP/pnpm.tgz" "$tarball"; then
+          echo "::warning::Every download of $tarball was slower than 256 kB/s, downloading it without a speed limit"
+          fetch --max-time 600 --output "$RUNNER_TEMP/pnpm.tgz" "$tarball"
+        fi
+        actual="sha512-$(openssl dgst -sha512 -binary "$RUNNER_TEMP/pnpm.tgz" | base64 -w0)"
+        if [[ "$actual" != "$integrity" ]]; then
+          echo "::error::$tarball has integrity $actual, but the registry lists $integrity"
+          exit 1
+        fi
+        mkdir -p "$PNPM_DIR"
+        tar -xzf "$RUNNER_TEMP/pnpm.tgz" -C "$PNPM_DIR" --strip-components=1 package/pnpm
+        rm "$RUNNER_TEMP/pnpm.tgz"
+
+    # Saved right away rather than at the end of the job, so jobs that fail later still fill the cache. PRs restore the
+    # base branch's copy, so they don't save their own.
+    - name: Save pnpm to the Actions cache
+      if: steps.cache.outputs.cache-hit != 'true' && github.event_name != 'pull_request'
+      uses: actions/cache/save@v6.1.0
+      with:
+        path: ${{ steps.pnpm.outputs.dir }}
+        key: ${{ steps.pnpm.outputs.key }}
+
+    - name: Add pnpm to PATH
+      shell: bash
+      env:
+        PNPM_DIR: ${{ steps.pnpm.outputs.dir }}
+      run: |
+        echo "PNPM_HOME=$PNPM_DIR" >> "$GITHUB_ENV"
+        echo "$PNPM_DIR" >> "$GITHUB_PATH"
+        "$PNPM_DIR/pnpm" --version

.github/actions/mount-pnpm-store/action.yml

@@ -1,7 +1,5 @@
 name: 'Mount pnpm store'
-description:
-  'Mounts the pnpm store, the pnpm and npm caches and the given paths from the Namespace cache volume. Use it before
-  `install-pnpm`, so `install-pnpm` takes pnpm from the cache volume.'
+description: 'Mounts the pnpm store, the pnpm and npm caches and the given paths from the Namespace cache volume.'
 
 inputs:
   path:
@@ -12,8 +10,7 @@ inputs:
 runs:
   using: 'composite'
   steps:
-    # Replaces the `pnpm` mode of nscloud-cache-action until it works without pnpm on PATH. The mode runs `pnpm store
-    # path`, so it only works after `install-pnpm`, when `pnpm self-update` has already downloaded pnpm.
+    # Replaces the `pnpm` mode of nscloud-cache-action, which runs `pnpm store path` and so needs pnpm on PATH.
     # The cache directory holds `lockfile-verified.jsonl`. With it, `pnpm install` skips the supply-chain check of a
     # lockfile that was already verified, which otherwise fetches the metadata of every package from the registry.
     - name: Set pnpm store and cache directories
@@ -22,7 +19,7 @@ runs:
         echo "PNPM_CONFIG_CACHE_DIR=$HOME/.cache/pnpm" >> "$GITHUB_ENV"
       shell: bash
 
-    # The `npm` mode keeps the npm cache of the `npm ci` in `pnpm/action-setup`, which installs the bootstrap pnpm
+    # The `npm` mode keeps the npm cache of `npx` and of the `npm install`s in the integration tests
     - name: Mount cache paths
       uses: namespacelabs/nscloud-cache-action@1124a6f3ce44e5cf84cc22111530961f4d2a15f9 # v1.7.0
       with:

.github/actions/prepare-e2e-app/action.yml

@@ -0,0 +1,59 @@
+name: 'Prepare E2E test application'
+description:
+  'Sets up the runtimes for an E2E test application and copies it to `dir` with pnpm overrides pointing at the tarballs.
+  Run `setup-e2e-tests` first.'
+inputs:
+  test-application:
+    description: 'The name of the test application in dev-packages/e2e-tests/test-applications.'
+    required: true
+  dir:
+    description: 'The directory to copy the application to.'
+    required: true
+  runtime:
+    description: 'The `runtime` of a runtime variant, see dev-packages/e2e-tests/README.md.'
+    default: ''
+  deno-version:
+    description: 'The Deno version to install, if the application needs Deno.'
+    default: 'v2.8.3'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Set up Node
+      uses: actions/setup-node@v7
+      with:
+        node-version-file: 'dev-packages/e2e-tests/test-applications/${{ inputs.test-application }}/package.json'
+    - name: Set up Bun
+      if:
+        inputs.test-application == 'node-exports-test-app' || contains(inputs.test-application, 'bun') || inputs.runtime
+        == 'bun'
+      uses: oven-sh/setup-bun@v2
+      with:
+        bun-version: '1.3.14'
+    - name: Set up AWS SAM
+      if: contains(inputs.test-application, 'aws-serverless')
+      uses: aws-actions/setup-sam@v3
+      with:
+        use-installer: true
+        token: ${{ github.token }}
+    - name: Set up Deno
+      if: contains(inputs.test-application, 'deno') || inputs.runtime == 'deno'
+      uses: denoland/setup-deno@v2.0.5
+      with:
+        deno-version: ${{ inputs.deno-version || 'v2.8.3' }}
+
+    - name: Copy to temp
+      run: pnpm run ci:copy-to-temp "./test-applications/$TEST_APPLICATION" "$APP_DIR"
+      shell: bash
+      working-directory: dev-packages/e2e-tests
+      env:
+        TEST_APPLICATION: ${{ inputs.test-application }}
+        APP_DIR: ${{ inputs.dir }}
+        RUNTIME: ${{ inputs.runtime }}
+
+    - name: Add pnpm overrides
+      run: pnpm run ci:pnpm-overrides "$APP_DIR" "$GITHUB_WORKSPACE/dev-packages/e2e-tests/packed"
+      shell: bash
+      working-directory: dev-packages/e2e-tests
+      env:
+        APP_DIR: ${{ inputs.dir }}