From 3fe9f9e4f28338ba76d7ccea7b24b118e3f7c787 Mon Sep 17 00:00:00 2001 From: Ricardo Reis Date: Fri, 30 Oct 2026 12:41:19 +0100 Subject: [PATCH 1/2] test(perf): add bundle size, build time and memory regression check Adds `tests/perf`, which bundles apps/tester-app in production mode (rspack and webpack, ios and android) for the current checkout and a base checkout, and compares the results: - output size (main bundle raw/gzip, all chunks, source maps, assets), module and chunk counts, and the list of emitted files; - build time (median of interleaved runs, each in a fresh process); - peak memory (process maxRSS plus the sampled process tree). Size metrics fail on a >1% increase. Time and memory use loose thresholds and only fail when head and base samples don't overlap. Metrics that vary between runs of one checkout (currently webpack output) are detected and compared like timings. Run locally with `pnpm bench --base-ref origin/main`. In CI, `.github/workflows/perf.yml` runs only on demand (the `perf` PR label or a manual dispatch) and is independent of the regular CI checks. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/perf.yml | 104 ++++++ AGENTS.md | 16 + agent_context/README.md | 1 + .../perf-regressions-oct2026/design.md | 77 ++++ package.json | 1 + pnpm-lock.yaml | 12 +- tests/perf/.gitignore | 1 + tests/perf/README.md | 66 ++++ tests/perf/package.json | 14 + tests/perf/src/bench.mjs | 333 ++++++++++++++++++ tests/perf/src/build-once.mjs | 57 +++ tests/perf/src/config.mjs | 125 +++++++ tests/perf/src/report.mjs | 237 +++++++++++++ tests/perf/src/sampleTreeRSS.mjs | 46 +++ 14 files changed, 1085 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/perf.yml create mode 100644 agent_context/perf-regressions-oct2026/design.md create mode 100644 tests/perf/.gitignore create mode 100644 tests/perf/README.md create mode 100644 tests/perf/package.json create mode 100644 tests/perf/src/bench.mjs create mode 100644 tests/perf/src/build-once.mjs create mode 100644 tests/perf/src/config.mjs create mode 100644 tests/perf/src/report.mjs create mode 100644 tests/perf/src/sampleTreeRSS.mjs diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 000000000..ef0e92ea4 --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,104 @@ +name: Performance + +# On-demand check, independent of the regular CI checks: it never runs on its +# own. Trigger it by adding the `perf` label to a PR, or from the Actions tab +# (optionally with a PR number). Head and base are benchmarked side by side on +# the same runner and the report is written to the job summary. +on: + pull_request: + types: [labeled] + workflow_dispatch: + inputs: + pr_number: + description: PR to benchmark against its base (leave empty to benchmark the selected branch) + required: false + base_ref: + description: Ref to compare the selected branch against (ignored when a PR number is given) + default: main + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || inputs.pr_number || github.ref }} + cancel-in-progress: true + +jobs: + benchmark: + name: Bundle size, time & memory + if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'perf' + runs-on: ubuntu-latest + timeout-minutes: 45 + + steps: + # Base and head are checked out side by side with identical path depth, + # so neither sits inside the other's `node_modules` resolution chain. + - name: Checkout head + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2 + with: + path: head + fetch-depth: 0 + ref: ${{ inputs.pr_number && format('refs/pull/{0}/merge', inputs.pr_number) || '' }} + + # For a PR, head is its merge commit and base is that commit's first + # parent, so exactly the PR's changes are measured. + - name: Resolve base + id: base + working-directory: head + env: + IS_PR: ${{ github.event_name == 'pull_request' || inputs.pr_number != '' }} + BASE_REF: ${{ inputs.base_ref || 'main' }} + run: | + if [ "$IS_PR" = "true" ]; then + sha=$(git rev-parse HEAD^1) + else + git fetch origin "$BASE_REF" + sha=$(git merge-base HEAD FETCH_HEAD) + fi + echo "sha=$sha" >> "$GITHUB_OUTPUT" + + - name: Checkout base + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2 + with: + path: base + ref: ${{ steps.base.outputs.sha }} + + - uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # tag=v4.1.0 + with: + package_json_file: head/package.json + + - name: Use Node.js 24 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # tag=v4.4.0 + with: + node-version: '24' + cache: pnpm + cache-dependency-path: | + head/pnpm-lock.yaml + base/pnpm-lock.yaml + + - name: Install dependencies (head) + working-directory: head + run: pnpm install --frozen-lockfile + + - name: Install dependencies (base) + working-directory: base + run: pnpm install --frozen-lockfile + + # Builds both checkouts, then runs head and base builds interleaved. + # Fails on a πŸ”΄ regression; it is not a required check. + - name: Run benchmarks + run: > + node head/tests/perf/src/bench.mjs + --head head + --base base + --runs 5 + --out perf-results + + - name: Upload results + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # tag=v4.6.2 + with: + name: perf-results + path: | + perf-results/results.json + perf-results/report.md diff --git a/AGENTS.md b/AGENTS.md index afc7276ab..78ae9e3aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,22 @@ Re.Pack is a toolkit for building and developing React Native applications with - `tests/integration/`: Integration-level automated coverage. - `tests/metro-compat/`: Metro compatibility behavior coverage. - `tests/resolver-cases/`: Resolver behavior and edge-case coverage. +- `tests/perf/`: Bundle size, build time and memory benchmarks (CI: `perf.yml`). + +### Performance + +Changes to bundling code (`packages/repack/src/{loaders,plugins,modules,commands}`, +plugin packages, bundler config defaults) can regress bundle size, build time +or memory. Before opening a PR for such a change, run +`pnpm bench --base-ref origin/main` (narrow with `--fixtures rspack/ios --runs 3` +while iterating) and read `tests/perf/results/report.md`: + +- πŸ”΄ must be fixed, or justified in the PR description if intended. +- 🟑 should be explained in the PR description. +- Treat "made the build output nondeterministic" as a bug. + +The same check runs in CI only on demand (add the `perf` label to a PR, or run +the `Performance` workflow manually); it is not part of the regular CI checks. ## Agent Context diff --git a/agent_context/README.md b/agent_context/README.md index 35109711a..c10cbc0f3 100644 --- a/agent_context/README.md +++ b/agent_context/README.md @@ -9,3 +9,4 @@ works, and is kept in sync with the implementation as it evolves. | Folder | Topic | | --- | --- | | [rspackv2-jul2026](./rspackv2-jul2026/design.md) | Dual Rspack 1.x/2.x support | +| [perf-regressions-oct2026](./perf-regressions-oct2026/design.md) | Bundle size, build time and memory regression guards | diff --git a/agent_context/perf-regressions-oct2026/design.md b/agent_context/perf-regressions-oct2026/design.md new file mode 100644 index 000000000..8de0eab38 --- /dev/null +++ b/agent_context/perf-regressions-oct2026/design.md @@ -0,0 +1,77 @@ +# Performance Regression Guards β€” Technical Design + +A change can be benchmarked against its base for bundle size, build time and +peak memory, so changes (including AI-generated ones) don't silently regress +performance. It runs locally (`pnpm bench --base-ref origin/main`) or as an +on-demand CI check that is independent of the regular CI checks +(`.github/workflows/perf.yml`, triggered by a `perf` PR label or manually). +Whether it should eventually run on every PR depends on how fast it turns out +to be in CI. Implementation: `tests/perf/`. Usage is in `tests/perf/README.md`. + +## Prior art (surveyed Oct 2026) + +- **No React Native bundler gates on performance publicly.** Metro, Expo CLI, + rnx-kit and the Vite-for-RN efforts have no perf CI. +- **webpack** (closest to Re.Pack: JS plugins/loaders) runs tinybench + + CodSpeed walltime and memory modes on PRs + (`.github/workflows/benchmarks.yml`), deterministic V8 flags, and a separate + output-size job comparing against a main baseline artifact (`code-size.yml`). +- **Rspack** uses CodSpeed simulation for Rust micro-benchmarks, plus an + ecosystem benchmark (`web-infra-dev/rspack-ecosystem-benchmark`) on a + dedicated runner that flags β‰₯5% time and β‰₯30% memory (RSS is noisy), and a + binary `size-limit` check. +- **Rolldown, Turbopack, Next.js, Parcel**: CodSpeed or + `github-action-benchmark` with loose thresholds (e.g. 110%), PR comments, and + rarely failing checks β€” except on size. + +The common pattern, adopted here: **hard-fail on deterministic metrics (size), +use loose thresholds with noise controls for time and memory.** + +## Design + +- **Fixtures** are real apps (`apps/tester-app` with rspack and webpack, ios + and android) bundled through the public `bundle` command, so loaders + (`babel-swc-loader`, assets), `RepackPlugin` and the NativeWind/Reanimated + plugins are all exercised. Defined in `tests/perf/src/config.mjs`. +- **Same-job A/B**: base and head are built and measured on the same machine, + with runs interleaved and order alternated. No stored baselines, no external + service, so it works for fork PRs and locally (`--base-ref`). +- **Process isolation**: each build runs in a fresh process + (`build-once.mjs`) that resolves `@callstack/repack` from the measured + checkout, so one harness can drive any checkout. +- **Memory**: `process.resourceUsage().maxRSS` (exact, includes threads) plus a + `ps`-sampled process-tree peak for webpack's worker processes. +- **Classification** (`report.mjs`): size metrics fail at >1% (warn on any + change); sampled metrics fail only when the median change exceeds the + threshold *and* head/base samples don't overlap. Memory also has an absolute + `failDelta` because Rspack's multi-GB peak hides large leaks in relative terms. + +## Pitfalls found while building it + +- **Checkout location matters.** A base worktree under `node_modules/` made + `babel-swc-loader` treat app code as third-party (`/node_modules/` checks), + changing output and slowing webpack builds ~60%. A symlinked root (macOS + `/var` β†’ `/private/var`) made asset and chunk names absolute. Base worktrees + live in `realpath(os.tmpdir())`; CI checks out `head/` and `base/` side by + side. +- **Source maps embed absolute paths**, so their size depends on checkout path + length; the repo root is replaced before measuring. +- **webpack output is nondeterministic** with `thread-loader` + + `babel-swc-loader`: the same React Native files get different + `@babel/runtime` helpers (e.g. `classPrivateFieldLoose*` vs `readOnlyError`) + depending on the worker, changing module counts and bundle bytes by a few + KB per run. The harness detects per-metric variation across runs and falls + back to sampled comparison for those metrics. Rspack output is + byte-identical across runs and checkouts. +- **Rspack peak RSS is ~4.5 GB** for tester-app on a 14-core machine (webpack: + ~1.2 GB main process), most likely from parallel loader workers scaling with + CPU count. + +## Not covered yet + +- Dev server startup, rebuild latency and development-mode bundles. +- Module Federation testers. +- Instruction-count benchmarks (CodSpeed simulation) for low-noise timing of + JS hot paths; needs CodSpeed set up for the org. +- PR comments (the report is in the job summary; comments from fork PRs need a + `workflow_run` follow-up job). diff --git a/package.json b/package.json index c20ae6901..d0aa7f94f 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "lint:ci": "biome check", "typecheck": "turbo run typecheck", "test": "turbo run test", + "bench": "pnpm --filter perf-benchmark run bench", "version-packages": "pnpm changeset version && pnpm lint", "version-packages:canary": "pnpm changeset version --snapshot canary && pnpm lint", "release": "pnpm changeset publish", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b55062204..eb6d8d2b8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1128,6 +1128,8 @@ importers: specifier: ^4.12.0 version: 4.41.0 + tests/perf: {} + tests/resolver-cases: devDependencies: '@callstack/repack': @@ -10313,7 +10315,7 @@ snapshots: '@babel/helper-member-expression-to-functions@7.29.7': dependencies: '@babel/traverse': 7.29.7 - '@babel/types': 7.29.7 + '@babel/types': 7.29.8 transitivePeerDependencies: - supports-color @@ -10335,7 +10337,7 @@ snapshots: '@babel/helper-optimise-call-expression@7.29.7': dependencies: - '@babel/types': 7.29.7 + '@babel/types': 7.29.8 '@babel/helper-plugin-utils@7.29.7': {} @@ -10374,7 +10376,7 @@ snapshots: dependencies: '@babel/template': 7.29.7 '@babel/traverse': 7.29.7 - '@babel/types': 7.29.7 + '@babel/types': 7.29.8 transitivePeerDependencies: - supports-color @@ -15627,7 +15629,7 @@ snapshots: babel-plugin-jest-hoist@29.6.3: dependencies: '@babel/template': 7.29.7 - '@babel/types': 7.29.7 + '@babel/types': 7.29.8 '@types/babel__core': 7.20.5 '@types/babel__traverse': 7.28.0 @@ -18083,7 +18085,7 @@ snapshots: '@babel/generator': 7.29.7 '@babel/plugin-syntax-jsx': 7.29.7(@babel/core@7.29.7) '@babel/plugin-syntax-typescript': 7.25.4(@babel/core@7.29.7) - '@babel/types': 7.29.7 + '@babel/types': 7.29.8 '@jest/expect-utils': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 diff --git a/tests/perf/.gitignore b/tests/perf/.gitignore new file mode 100644 index 000000000..fbca22537 --- /dev/null +++ b/tests/perf/.gitignore @@ -0,0 +1 @@ +results/ diff --git a/tests/perf/README.md b/tests/perf/README.md new file mode 100644 index 000000000..3fe8183be --- /dev/null +++ b/tests/perf/README.md @@ -0,0 +1,66 @@ +# Performance benchmarks + +Guards against bundle size, build time and memory regressions by bundling the +fixture apps in production mode and comparing the results with another +checkout (normally the PR base). + +```sh +pnpm bench # measure this checkout only +pnpm bench --base-ref origin/main # compare with the merge-base of origin/main +pnpm bench --fixtures rspack/ios --runs 3 +``` + +`--base-ref` creates a git worktree of the merge-base under the OS temp +directory (`/repack-perf/base-`), installs it once and reuses it on +later runs. Both checkouts are rebuilt with turbo before measuring (skip with +`--skip-build`). Reports go to `results/report.md` and `results/results.json`. + +| Option | Default | | +| --- | --- | --- | +| `--base-ref ` | – | Compare with the merge-base of `HEAD` and `` | +| `--base ` / `--head ` | – / repo root | Compare two existing checkouts (used in CI) | +| `--runs ` | `5` | Timed runs per fixture and checkout, after 1 warmup | +| `--fixtures ` | all | Substring filter on fixture names | +| `--out ` | `results/` | Output directory | +| `--no-fail` | – | Exit 0 even when a regression is detected | + +## What is measured + +Each build runs in a fresh Node process (`src/build-once.mjs`), and head and +base runs are interleaved so machine load affects both alike. + +- **Output size** (exact): main bundle raw/gzip, all JS chunks gzip, source maps + (repo path normalized), static assets, plus module and chunk counts and the + list of emitted files. +- **Build time**: median wall time of the `bundle` command. +- **Peak memory**: `maxRSS` of the build process (covers Rspack's native + threads and worker threads), and the sampled peak RSS of the whole process + tree (covers webpack's `thread-loader` and Terser child processes). + +Fixtures and thresholds live in `src/config.mjs`: + +- Size metrics fail at a >1% increase (5% for maps/assets) and warn on any + increase. +- Time fails at >15% and memory at >30% or +512 MB, but only when head and base + samples don't overlap; smaller changes only warn. +- If a checkout produces different output across runs of the same fixture + (currently the webpack fixtures), its size metrics are marked `ΒΉ` and compared + like timings instead of failing on any change. A PR that makes previously + stable output nondeterministic gets a warning. + +## CI + +`.github/workflows/perf.yml` is an on-demand check, separate from the regular CI +checks: it never runs automatically. Trigger it by adding the `perf` label to a +PR (re-add it to run again), or from the Actions tab, optionally with a PR +number. It checks out the base and head side by side, runs 5 interleaved +builds per fixture, and writes the report to the job summary (artifact: +`perf-results`). The run fails on a πŸ”΄ regression, but it is not a required +check. + +## Adding a fixture + +Add an entry to `FIXTURES` in `src/config.mjs`. The config file must honour +`TEST_WEBPACK_OUTPUT_DIR` for `output.path` (see +`apps/tester-app/__tests__/configs/`). A fixture that doesn't exist in the base +checkout yet is reported for head only. diff --git a/tests/perf/package.json b/tests/perf/package.json new file mode 100644 index 000000000..e7900e26c --- /dev/null +++ b/tests/perf/package.json @@ -0,0 +1,14 @@ +{ + "name": "perf-benchmark", + "version": "0.0.1", + "description": "Performance regression benchmarks for @callstack/repack", + "private": true, + "type": "module", + "engines": { + "node": ">=22" + }, + "scripts": { + "bench": "node src/bench.mjs", + "bench:compare": "node src/bench.mjs --base-ref origin/main" + } +} diff --git a/tests/perf/src/bench.mjs b/tests/perf/src/bench.mjs new file mode 100644 index 000000000..c9b8428be --- /dev/null +++ b/tests/perf/src/bench.mjs @@ -0,0 +1,333 @@ +// Benchmarks production bundling of the fixture apps and, optionally, compares +// the results against another checkout (the PR base). +// +// Usage: +// node src/bench.mjs # measure this checkout +// node src/bench.mjs --base-ref main # compare against merge-base with main +// node src/bench.mjs --base ../repack-base --head . +// +// See ../README.md for all options. +import { execFileSync, spawn } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { parseArgs } from 'node:util'; +import zlib from 'node:zlib'; +import { FIXTURES } from './config.mjs'; +import { compareResults, renderReport } from './report.mjs'; +import { sampleTreeRSS } from './sampleTreeRSS.mjs'; + +const HERE = path.dirname(new URL(import.meta.url).pathname); +const REPO_ROOT = path.resolve(HERE, '../../..'); +const RESULT_MARKER = '__PERF_RESULT__'; +const SIZE_KEYS = new Set([ + 'mainBundleRaw', + 'mainBundleGzip', + 'jsTotalGzip', + 'sourceMapRaw', + 'assetsRaw', +]); + +const { values: options } = parseArgs({ + options: { + head: { type: 'string', default: REPO_ROOT }, + base: { type: 'string' }, + 'base-ref': { type: 'string' }, + runs: { type: 'string', default: '5' }, + fixtures: { type: 'string' }, + out: { type: 'string', default: path.join(HERE, '../results') }, + 'skip-build': { type: 'boolean', default: false }, + 'no-fail': { type: 'boolean', default: false }, + }, +}); + +const runs = Number(options.runs); +const fixtures = options.fixtures + ? FIXTURES.filter((f) => + options.fixtures.split(',').some((name) => f.name.includes(name)) + ) + : FIXTURES; + +if (fixtures.length === 0) { + throw new Error(`No fixtures match '${options.fixtures}'`); +} + +function log(message) { + process.stderr.write(`[perf] ${message}\n`); +} + +/** + * Creates (or reuses) a git worktree of the merge-base between HEAD and `ref`, + * installed and built, so it can be benchmarked side by side with HEAD. + */ +function prepareBaseWorktree(ref) { + const git = (...args) => + execFileSync('git', args, { cwd: REPO_ROOT, encoding: 'utf-8' }).trim(); + const sha = git('merge-base', 'HEAD', ref); + // Must live outside any `node_modules`: loaders and bundler heuristics treat + // such paths as third-party code, which skews both output and timings. The + // temp dir also has no parent `node_modules` for resolution to walk into. + // realpath: macOS tmpdir is a symlink, and a symlinked project root makes + // emitted asset/chunk names absolute (module paths no longer sit under + // `context`). + const dir = path.join( + fs.realpathSync(os.tmpdir()), + 'repack-perf', + `base-${sha.slice(0, 12)}` + ); + const readyMarker = path.join(dir, 'node_modules/.repack-perf-ready'); + if (!fs.existsSync(readyMarker)) { + log(`Creating worktree for ${ref} (${sha.slice(0, 12)}) at ${dir}`); + if (fs.existsSync(dir)) git('worktree', 'remove', '--force', dir); + git('worktree', 'prune'); + git('worktree', 'add', '--detach', dir, sha); + execFileSync('pnpm', ['install', '--frozen-lockfile'], { + cwd: dir, + stdio: ['ignore', 'ignore', 'inherit'], + }); + fs.writeFileSync(readyMarker, sha); + } + return dir; +} + +function buildPackages(repoDir) { + log(`Building Re.Pack packages in ${repoDir}`); + const filters = [...new Set(fixtures.map((f) => f.appDir))].map( + (appDir) => `--filter=./${appDir}^...` + ); + execFileSync('pnpm', ['turbo', 'run', 'build', ...filters], { + cwd: repoDir, + stdio: ['ignore', 'ignore', 'inherit'], + }); +} + +async function buildOnce(repoDir, fixture, outDir, statsFile) { + fs.rmSync(outDir, { recursive: true, force: true }); + const job = { + appDir: path.join(repoDir, fixture.appDir), + configFile: fixture.configFile, + platform: fixture.platform, + outDir, + statsFile, + }; + + const child = spawn( + process.execPath, + [path.join(HERE, 'build-once.mjs'), JSON.stringify(job)], + { stdio: ['ignore', 'pipe', 'pipe'] } + ); + const stopSampling = sampleTreeRSS(child.pid); + + let output = ''; + child.stdout.on('data', (chunk) => { + output += chunk; + }); + child.stderr.on('data', (chunk) => { + output += chunk; + }); + const code = await new Promise((resolve) => child.on('close', resolve)); + const treeRSSBytes = await stopSampling(); + + const line = output.split('\n').find((l) => l.startsWith(RESULT_MARKER)); + if (code !== 0 || !line) { + process.stderr.write(output); + throw new Error(`Build failed for ${fixture.name} in ${repoDir}`); + } + const result = JSON.parse(line.slice(RESULT_MARKER.length)); + return { + ...result, + treeRSSBytes: Math.max(treeRSSBytes, result.maxRSSBytes), + }; +} + +function listFiles(dir) { + return fs + .readdirSync(dir, { recursive: true, withFileTypes: true }) + .filter((entry) => entry.isFile()) + .map((entry) => + path.relative(dir, path.join(entry.parentPath ?? entry.path, entry.name)) + ) + .sort(); +} + +/** + * Output size metrics of a finished build. Measured after every run, because + * some setups don't produce the same output twice (see `nondeterministic` in + * the report). + */ +function measureOutput(outDir, platform, repoDir) { + // The bundle command also copies the main bundle to `bundleOutput`. Skip + // that copy so it isn't counted twice. + const bundleOutput = + platform === 'ios' ? 'main.jsbundle' : `index.${platform}.bundle`; + const files = listFiles(outDir).filter( + (file) => file !== bundleOutput && file !== `${bundleOutput}.map` + ); + + const sizes = {}; + const metrics = { + mainBundleRaw: 0, + mainBundleGzip: 0, + jsTotalGzip: 0, + sourceMapRaw: 0, + assetsRaw: 0, + }; + for (const file of files) { + let content = fs.readFileSync(path.join(outDir, file)); + const isMap = file.endsWith('.map'); + const isJs = !isMap && file.endsWith('.bundle'); + if (isMap) { + // Source maps embed absolute paths, so their size would otherwise + // depend on where the checkout lives. + content = Buffer.from(content.toString().replaceAll(repoDir, '')); + } + const entry = { raw: content.length }; + if (isJs) { + entry.gzip = zlib.gzipSync(content, { level: 9 }).length; + metrics.jsTotalGzip += entry.gzip; + if (file === 'index.bundle') { + metrics.mainBundleRaw = entry.raw; + metrics.mainBundleGzip = entry.gzip; + } + } else if (isMap) { + metrics.sourceMapRaw += entry.raw; + } else { + metrics.assetsRaw += entry.raw; + } + sizes[file] = entry; + } + return { metrics, files: sizes }; +} + +function readStatsMetrics(statsFile) { + const stats = JSON.parse(fs.readFileSync(statsFile, 'utf-8')); + const compilation = stats.children?.[0] ?? stats; + // Concatenated modules hold their inner modules in `modules`; count leaves. + const countModules = (modules = []) => + modules.reduce( + (sum, m) => sum + (m.modules ? countModules(m.modules) : 1), + 0 + ); + return { + modules: countModules(compilation.modules), + chunks: compilation.chunks?.length ?? 0, + }; +} + +const median = (values) => { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; +}; + +async function main() { + const targets = { head: fs.realpathSync(path.resolve(options.head)) }; + if (options['base-ref']) { + targets.base = prepareBaseWorktree(options['base-ref']); + } else if (options.base) { + targets.base = fs.realpathSync(path.resolve(options.base)); + } + + if (!options['skip-build']) { + for (const dir of Object.values(targets)) buildPackages(dir); + } + + const workDir = path.join(options.out, 'work'); + const results = {}; + for (const [target, repoDir] of Object.entries(targets)) { + results[target] = { repoDir, fixtures: {} }; + } + + for (const fixture of fixtures) { + const slug = fixture.name.replaceAll('/', '-'); + + // A fixture added by the PR doesn't exist in the base checkout yet; it is + // then reported for head only. + const fixtureTargets = Object.entries(targets).filter( + ([target, repoDir]) => { + const exists = fs.existsSync( + path.join(repoDir, fixture.appDir, fixture.configFile) + ); + if (!exists) + log(`${fixture.name} [${target}] skipped: not in checkout`); + return exists; + } + ); + for (const [target] of fixtureTargets) { + results[target].fixtures[fixture.name] = { samples: [] }; + } + + // Warmup build: primes the file system cache and produces the stats for + // module/chunk counts. Not included in timings because it also serializes + // stats. + for (const [target, repoDir] of fixtureTargets) { + log(`${fixture.name} [${target}] warmup`); + const outDir = path.join(workDir, target, slug); + const statsFile = path.join(workDir, `${target}-${slug}.stats.json`); + await buildOnce(repoDir, fixture, outDir, statsFile); + const entry = results[target].fixtures[fixture.name]; + entry.metrics = readStatsMetrics(statsFile); + entry.files = measureOutput(outDir, fixture.platform, repoDir).files; + } + + // Interleave targets and alternate their order between runs so drift in + // machine load (thermal throttling, noisy neighbours) affects both alike. + const order = fixtureTargets.map(([target]) => target); + for (let run = 0; run < runs; run++) { + for (const target of run % 2 ? [...order].reverse() : order) { + const outDir = path.join(workDir, target, slug); + const sample = await buildOnce(targets[target], fixture, outDir); + Object.assign( + sample, + measureOutput(outDir, fixture.platform, targets[target]).metrics + ); + log( + `${fixture.name} [${target}] run ${run + 1}/${runs}: ` + + `${Math.round(sample.durationMs)}ms, ` + + `${Math.round(sample.maxRSSBytes / 1024 / 1024)}MB` + ); + results[target].fixtures[fixture.name].samples.push(sample); + } + } + + for (const target of order) { + const entry = results[target].fixtures[fixture.name]; + entry.nondeterministic = []; + for (const key of Object.keys(entry.samples[0])) { + const values = entry.samples.map((s) => s[key]); + entry.metrics[key] = median(values); + if (SIZE_KEYS.has(key) && new Set(values).size > 1) { + entry.nondeterministic.push(key); + } + } + } + } + + fs.rmSync(workDir, { recursive: true, force: true }); + + const comparison = compareResults(results.head, results.base); + const markdown = renderReport(comparison, { + runs, + platform: `${process.platform}-${process.arch}`, + node: process.version, + }); + + fs.mkdirSync(options.out, { recursive: true }); + fs.writeFileSync( + path.join(options.out, 'results.json'), + JSON.stringify({ results, comparison }, null, 2) + ); + fs.writeFileSync(path.join(options.out, 'report.md'), markdown); + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown); + } + + process.stdout.write(`${markdown}\n`); + log(`Results written to ${options.out}`); + + if (comparison.status === 'fail' && !options['no-fail']) { + process.exitCode = 1; + } +} + +await main(); diff --git a/tests/perf/src/build-once.mjs b/tests/perf/src/build-once.mjs new file mode 100644 index 000000000..15b724b6e --- /dev/null +++ b/tests/perf/src/build-once.mjs @@ -0,0 +1,57 @@ +// Runs a single production bundle in this process and prints a JSON result +// line. Spawned in a fresh process per sample by `bench.mjs`, so peak RSS and +// timings are not polluted by previous builds. +// +// Usage: node build-once.mjs +// job = { appDir, configFile, platform, outDir, statsFile? } +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const job = JSON.parse(process.argv[2]); + +// Resolve Re.Pack from the app being measured, not from this package, so the +// same harness can benchmark a different checkout (e.g. the PR base). +const requireFromApp = createRequire(path.join(job.appDir, 'package.json')); +const commandsPath = requireFromApp.resolve('@callstack/repack/commands'); +const { default: commands } = await import(pathToFileURL(commandsPath).href); +const bundleCommand = commands.find((command) => command.name === 'bundle'); + +const bundleOutput = path.join( + job.outDir, + job.platform === 'ios' ? 'main.jsbundle' : `index.${job.platform}.bundle` +); + +// Read by the fixture configs to place every emitted file under outDir. +process.env.TEST_WEBPACK_OUTPUT_DIR = job.outDir; +process.env.REPACK_BUNDLE_FILENAME = bundleOutput; +process.env.NODE_ENV = 'production'; + +const cliConfig = { + root: job.appDir, + platforms: { ios: {}, android: {} }, + reactNativePath: path.join(job.appDir, 'node_modules/react-native'), +}; + +const args = { + platform: job.platform, + entryFile: 'index.js', + bundleOutput, + dev: false, + webpackConfig: path.join(job.appDir, job.configFile), + ...(job.statsFile ? { json: job.statsFile, stats: 'normal' } : {}), +}; + +const start = process.hrtime.bigint(); +await bundleCommand.func([''], cliConfig, args); +const durationMs = Number(process.hrtime.bigint() - start) / 1e6; + +// maxRSS covers this process including worker threads (Rspack's native +// threads and parallel loaders). Child processes (webpack's thread-loader, +// terser workers) are not included. +const { maxRSS } = process.resourceUsage(); + +process.stdout.write( + `\n__PERF_RESULT__${JSON.stringify({ durationMs, maxRSSBytes: maxRSS * 1024 })}\n` +); +process.exit(0); diff --git a/tests/perf/src/config.mjs b/tests/perf/src/config.mjs new file mode 100644 index 000000000..89af523de --- /dev/null +++ b/tests/perf/src/config.mjs @@ -0,0 +1,125 @@ +// Fixtures and regression thresholds for the performance benchmarks. +// +// Every fixture is resolved relative to the checkout being measured, so the +// same definitions work for the PR head and its base. + +export const FIXTURES = [ + { + name: 'tester-app/rspack/ios', + appDir: 'apps/tester-app', + configFile: '__tests__/configs/rspack.config.mjs', + platform: 'ios', + }, + { + name: 'tester-app/rspack/android', + appDir: 'apps/tester-app', + configFile: '__tests__/configs/rspack.config.mjs', + platform: 'android', + }, + { + name: 'tester-app/webpack/ios', + appDir: 'apps/tester-app', + configFile: '__tests__/configs/webpack.config.mjs', + platform: 'ios', + }, + { + name: 'tester-app/webpack/android', + appDir: 'apps/tester-app', + configFile: '__tests__/configs/webpack.config.mjs', + platform: 'android', + }, +]; + +// `kind: 'exact'` metrics are deterministic for a given checkout, so any +// change is real and they use tight thresholds. `kind: 'sampled'` metrics vary +// between runs; they only fail when the change is large *and* the head and +// base samples don't overlap. Hosted CI runners are noisy, which is why the +// sampled thresholds are deliberately loose (Rspack's ecosystem benchmark uses +// 5% time / 30% memory on a dedicated runner). `failDelta` additionally fails +// on an absolute increase, regardless of the relative change. +export const METRICS = { + mainBundleGzip: { + label: 'Main bundle (gzip)', + unit: 'bytes', + kind: 'exact', + warn: 0, + fail: 0.01, + minDelta: 512, + }, + mainBundleRaw: { + label: 'Main bundle (raw)', + unit: 'bytes', + kind: 'exact', + warn: 0, + fail: 0.01, + minDelta: 1024, + }, + jsTotalGzip: { + label: 'All JS chunks (gzip)', + unit: 'bytes', + kind: 'exact', + warn: 0, + fail: 0.01, + minDelta: 512, + }, + sourceMapRaw: { + label: 'Source maps (raw)', + unit: 'bytes', + kind: 'exact', + warn: 0, + fail: 0.05, + minDelta: 4096, + }, + assetsRaw: { + label: 'Static assets (raw)', + unit: 'bytes', + kind: 'exact', + warn: 0, + fail: 0.05, + minDelta: 1024, + }, + modules: { + label: 'Modules', + unit: 'count', + kind: 'exact', + warn: 0, + fail: null, + }, + chunks: { + label: 'Chunks', + unit: 'count', + kind: 'exact', + warn: 0, + fail: null, + }, + durationMs: { + label: 'Build time (median)', + unit: 'ms', + kind: 'sampled', + warn: 0.05, + fail: 0.15, + minDelta: 300, + }, + maxRSSBytes: { + label: 'Peak RSS (main process)', + unit: 'bytes', + kind: 'sampled', + warn: 0.05, + fail: 0.3, + // Peak RSS can be several GB (Rspack scales parallel loader workers with + // CPU count), which would hide a large leak behind a relative threshold. + failDelta: 512 * 1024 * 1024, + minDelta: 32 * 1024 * 1024, + }, + treeRSSBytes: { + label: 'Peak RSS (process tree, sampled)', + unit: 'bytes', + kind: 'sampled', + warn: 0.05, + fail: 0.3, + // Peak RSS can be several GB (Rspack scales parallel loader workers with + // CPU count), which would hide a large leak behind a relative threshold. + failDelta: 512 * 1024 * 1024, + minDelta: 32 * 1024 * 1024, + }, +}; diff --git a/tests/perf/src/report.mjs b/tests/perf/src/report.mjs new file mode 100644 index 000000000..7369f0fe2 --- /dev/null +++ b/tests/perf/src/report.mjs @@ -0,0 +1,237 @@ +import { METRICS } from './config.mjs'; + +const STATUS_ICON = { + fail: 'πŸ”΄', + warn: '🟑', + improved: '🟒', + ok: 'βšͺ', +}; + +const STATUS_RANK = { ok: 0, improved: 0, warn: 1, fail: 2 }; + +function formatValue(value, unit) { + if (value === undefined) return '–'; + if (unit === 'bytes') { + const abs = Math.abs(value); + if (abs >= 1024 ** 3) return `${(value / 1024 ** 3).toFixed(2)} GB`; + if (abs >= 1024 * 1024) return `${(value / 1024 / 1024).toFixed(2)} MB`; + if (abs >= 1024) return `${(value / 1024).toFixed(1)} KB`; + return `${value} B`; + } + if (unit === 'ms') { + return Math.abs(value) >= 1000 + ? `${(value / 1000).toFixed(2)} s` + : `${Math.round(value)} ms`; + } + return String(value); +} + +function formatDelta(delta, pct, unit) { + if (delta === 0) return '0'; + const sign = delta > 0 ? '+' : '-'; + const pctText = Number.isFinite(pct) + ? ` (${sign}${Math.abs(pct * 100).toFixed(1)}%)` + : ''; + return `${sign}${formatValue(Math.abs(delta), unit)}${pctText}`; +} + +/** + * Classifies a single metric change. Sampled metrics need both a large median + * change and non-overlapping samples to fail, which keeps CI noise from + * failing builds while still catching real regressions. + */ +function classify(metric, head, base, headSamples, baseSamples, noisy) { + // An "exact" metric that varied between runs of one checkout is handled + // like a sampled one, so flaky output doesn't fail builds on its own. + if (noisy) metric = { ...metric, kind: 'sampled', warn: metric.fail ?? 0.01 }; + const delta = head - base; + const pct = base === 0 ? (delta === 0 ? 0 : Infinity) : delta / base; + const significant = Math.abs(delta) >= (metric.minDelta ?? 0); + + if (delta === 0 || !significant) return { delta, pct, status: 'ok' }; + const belowWarn = + Math.abs(pct) < metric.warn && + !(metric.failDelta && Math.abs(delta) > metric.failDelta); + if (metric.kind === 'sampled' && belowWarn) { + return { delta, pct, status: 'ok' }; + } + if (delta < 0) return { delta, pct, status: 'improved' }; + + let status = pct > metric.warn ? 'warn' : 'ok'; + const exceedsFail = + metric.fail !== null && + (pct > metric.fail || (metric.failDelta && delta > metric.failDelta)); + if (exceedsFail) { + const separated = + metric.kind === 'exact' || + Math.min(...headSamples) > Math.max(...baseSamples); + status = separated ? 'fail' : 'warn'; + } + return { delta, pct, status }; +} + +export function compareResults(head, base) { + const fixtures = []; + let status = 'ok'; + + for (const [name, headEntry] of Object.entries(head.fixtures)) { + const baseEntry = base?.fixtures[name]; + const rows = []; + + const nondeterministic = new Set([ + ...headEntry.nondeterministic, + ...(baseEntry?.nondeterministic ?? []), + ]); + // Only flag nondeterminism the PR introduced; pre-existing flakiness is + // reported but doesn't affect the status. + const introducedNondeterminism = baseEntry + ? headEntry.nondeterministic.filter( + (key) => !baseEntry.nondeterministic.includes(key) + ) + : []; + + for (const [key, metric] of Object.entries(METRICS)) { + const headValue = headEntry.metrics[key]; + const row = { + key, + label: metric.label, + unit: metric.unit, + head: headValue, + // Module/chunk counts come from a single build; if that fixture's + // output is unstable, so are they. + noisy: + nondeterministic.has(key) || + (nondeterministic.size > 0 && metric.unit === 'count'), + }; + if (baseEntry) { + row.base = baseEntry.metrics[key]; + Object.assign( + row, + classify( + metric, + headValue, + row.base, + headEntry.samples.map((s) => s[key]), + baseEntry.samples.map((s) => s[key]), + row.noisy + ) + ); + } else { + row.status = 'ok'; + } + if (STATUS_RANK[row.status] > STATUS_RANK[status]) status = row.status; + rows.push(row); + } + + let addedFiles = []; + let removedFiles = []; + if (baseEntry) { + const headFiles = Object.keys(headEntry.files); + const baseFiles = Object.keys(baseEntry.files); + addedFiles = headFiles.filter((f) => !baseEntry.files[f]); + removedFiles = baseFiles.filter((f) => !headEntry.files[f]); + if ( + (addedFiles.length || + removedFiles.length || + introducedNondeterminism.length) && + STATUS_RANK[status] < STATUS_RANK.warn + ) { + status = 'warn'; + } + } + + fixtures.push({ + name, + rows, + addedFiles, + removedFiles, + nondeterministic: [...nondeterministic], + introducedNondeterminism, + }); + } + + return { status, hasBase: Boolean(base), fixtures }; +} + +export function renderReport(comparison, { runs, platform, node }) { + const lines = []; + const headline = { + fail: 'πŸ”΄ Performance regression detected', + warn: '🟑 Performance changed β€” review the details', + improved: '🟒 No regressions', + ok: 'βšͺ No significant changes', + }[comparison.status]; + + lines.push('## Re.Pack performance benchmark', ''); + lines.push( + comparison.hasBase + ? `**${headline}**` + : 'Measured head only (no base to compare against).' + ); + lines.push( + '', + `${runs} timed runs per fixture after 1 warmup Β· ${platform} Β· Node ${node}. ` + + 'Size metrics are exact; timing and memory are medians and only fail when ' + + 'head and base samples do not overlap.', + '' + ); + + for (const fixture of comparison.fixtures) { + let worst = fixture.rows.reduce( + (acc, row) => + STATUS_RANK[row.status] > STATUS_RANK[acc] ? row.status : acc, + 'ok' + ); + const fixtureChanged = + fixture.addedFiles.length || + fixture.removedFiles.length || + fixture.introducedNondeterminism.length; + if (fixtureChanged && STATUS_RANK[worst] < STATUS_RANK.warn) worst = 'warn'; + const open = comparison.hasBase && STATUS_RANK[worst] > 0 ? ' open' : ''; + lines.push( + `${STATUS_ICON[worst]} ${fixture.name}`, + '' + ); + if (comparison.hasBase) { + lines.push( + '| | Metric | Base | Head | Change |', + '|---|---|---:|---:|---:|' + ); + for (const row of fixture.rows) { + const label = row.noisy ? `${row.label} ΒΉ` : row.label; + lines.push( + `| ${STATUS_ICON[row.status]} | ${label} | ${formatValue(row.base, row.unit)} | ` + + `${formatValue(row.head, row.unit)} | ${formatDelta(row.delta, row.pct, row.unit)} |` + ); + } + } else { + lines.push('| Metric | Value |', '|---|---:|'); + for (const row of fixture.rows) { + const label = row.noisy ? `${row.label} ΒΉ` : row.label; + lines.push(`| ${label} | ${formatValue(row.head, row.unit)} |`); + } + } + if (fixture.nondeterministic.length) { + lines.push( + '', + 'ΒΉ Output differs between runs of the same checkout (nondeterministic ' + + 'build), so this is compared as a median and does not fail on its own.' + ); + } + if (fixture.introducedNondeterminism.length) { + lines.push( + '', + `⚠️ This change made the build output nondeterministic ` + + `(${fixture.introducedNondeterminism.join(', ')} vary between runs; base is stable).` + ); + } + if (fixture.addedFiles.length || fixture.removedFiles.length) { + lines.push('', 'Output files changed:', ''); + for (const file of fixture.addedFiles) lines.push(`- \`+ ${file}\``); + for (const file of fixture.removedFiles) lines.push(`- \`- ${file}\``); + } + lines.push('', '', ''); + } + + return lines.join('\n'); +} diff --git a/tests/perf/src/sampleTreeRSS.mjs b/tests/perf/src/sampleTreeRSS.mjs new file mode 100644 index 000000000..36e9bc3c4 --- /dev/null +++ b/tests/perf/src/sampleTreeRSS.mjs @@ -0,0 +1,46 @@ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +/** Peak RSS of `rootPid` and all its descendants, sampled with `ps`. */ +export function sampleTreeRSS(rootPid) { + let peak = 0; + let stopped = false; + const tick = async () => { + while (!stopped) { + try { + const { stdout } = await execFileAsync('ps', [ + '-A', + '-o', + 'pid=,ppid=,rss=', + ]); + const children = new Map(); + const rss = new Map(); + for (const line of stdout.trim().split('\n')) { + const [pid, ppid, kb] = line.trim().split(/\s+/).map(Number); + rss.set(pid, kb * 1024); + if (!children.has(ppid)) children.set(ppid, []); + children.get(ppid).push(pid); + } + let total = 0; + const queue = [rootPid]; + while (queue.length) { + const pid = queue.pop(); + total += rss.get(pid) ?? 0; + queue.push(...(children.get(pid) ?? [])); + } + peak = Math.max(peak, total); + } catch { + // The process may exit between spawn and sampling. + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + }; + const done = tick(); + return async () => { + stopped = true; + await done; + return peak; + }; +} From bb80f61e7a1cc75be3461c72a1a6253ff7e1640a Mon Sep 17 00:00:00 2001 From: Ricardo Reis Date: Fri, 30 Oct 2026 12:41:34 +0100 Subject: [PATCH 2/2] test(perf): add Re.Pack vs Metro competitor benchmark Adds two neutral apps that both Re.Pack and Metro build without changes: - apps/bench-expo: Expo Router app (Metro via Expo CLI, optionally with Expo's experimental tree shaking, vs Re.Pack with repack-expo); - apps/bench-bare: plain React Native app (Metro via the React Native CLI with @react-native/metro-config defaults, vs Re.Pack with the repack-init template config). `tests/perf/src/competitors.mjs` bundles each app at several sizes (screens added by `generate-screens.mjs`) and measures cold and warm release builds (wall time, CPU time, peak memory), bundle and Hermes bytecode size, and dev server startup and edit-to-bundle latency. It is not a regression gate: it is for comparing bundlers and finding performance wins. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 + .../perf-regressions-oct2026/design.md | 40 +- apps/bench-bare/.gitignore | 3 + apps/bench-bare/app.json | 4 + apps/bench-bare/assets/repack.png | Bin 0 -> 2518 bytes apps/bench-bare/babel.config.js | 3 + apps/bench-bare/index.js | 5 + apps/bench-bare/metro.config.js | 9 + apps/bench-bare/package.json | 32 ++ apps/bench-bare/react-native.config.js | 8 + apps/bench-bare/rspack.config.mjs | 47 ++ apps/bench-bare/src/App.tsx | 120 +++++ apps/bench-bare/src/LazyCard.tsx | 19 + apps/bench-bare/src/generated/index.ts | 5 + apps/bench-bare/tsconfig.json | 9 + apps/bench-expo/.gitignore | 4 + apps/bench-expo/app.json | 19 + apps/bench-expo/app/_layout.tsx | 5 + apps/bench-expo/app/density-image.tsx | 55 ++ apps/bench-expo/app/details.tsx | 24 + apps/bench-expo/app/image.tsx | 42 ++ apps/bench-expo/app/index.tsx | 35 ++ apps/bench-expo/app/lazy.tsx | 39 ++ apps/bench-expo/assets/README.md | 7 + apps/bench-expo/assets/images/density.png | Bin 0 -> 3221 bytes apps/bench-expo/assets/images/density@2x.png | Bin 0 -> 6522 bytes apps/bench-expo/assets/images/density@3x.png | Bin 0 -> 10257 bytes apps/bench-expo/assets/images/repack.png | Bin 0 -> 2518 bytes apps/bench-expo/babel.config.cjs | 6 + apps/bench-expo/components/LazyCard.local.tsx | 25 + apps/bench-expo/metro.config.js | 14 + apps/bench-expo/package.json | 46 ++ apps/bench-expo/react-native.config.js | 3 + apps/bench-expo/rspack.config.mjs | 53 ++ apps/bench-expo/tsconfig.json | 12 + pnpm-lock.yaml | 131 +++++ tests/perf/README.md | 40 ++ tests/perf/src/competitors.mjs | 506 ++++++++++++++++++ tests/perf/src/generate-screens.mjs | 224 ++++++++ 39 files changed, 1595 insertions(+), 1 deletion(-) create mode 100644 apps/bench-bare/.gitignore create mode 100644 apps/bench-bare/app.json create mode 100644 apps/bench-bare/assets/repack.png create mode 100644 apps/bench-bare/babel.config.js create mode 100644 apps/bench-bare/index.js create mode 100644 apps/bench-bare/metro.config.js create mode 100644 apps/bench-bare/package.json create mode 100644 apps/bench-bare/react-native.config.js create mode 100644 apps/bench-bare/rspack.config.mjs create mode 100644 apps/bench-bare/src/App.tsx create mode 100644 apps/bench-bare/src/LazyCard.tsx create mode 100644 apps/bench-bare/src/generated/index.ts create mode 100644 apps/bench-bare/tsconfig.json create mode 100644 apps/bench-expo/.gitignore create mode 100644 apps/bench-expo/app.json create mode 100644 apps/bench-expo/app/_layout.tsx create mode 100644 apps/bench-expo/app/density-image.tsx create mode 100644 apps/bench-expo/app/details.tsx create mode 100644 apps/bench-expo/app/image.tsx create mode 100644 apps/bench-expo/app/index.tsx create mode 100644 apps/bench-expo/app/lazy.tsx create mode 100644 apps/bench-expo/assets/README.md create mode 100644 apps/bench-expo/assets/images/density.png create mode 100644 apps/bench-expo/assets/images/density@2x.png create mode 100644 apps/bench-expo/assets/images/density@3x.png create mode 100644 apps/bench-expo/assets/images/repack.png create mode 100644 apps/bench-expo/babel.config.cjs create mode 100644 apps/bench-expo/components/LazyCard.local.tsx create mode 100644 apps/bench-expo/metro.config.js create mode 100644 apps/bench-expo/package.json create mode 100644 apps/bench-expo/react-native.config.js create mode 100644 apps/bench-expo/rspack.config.mjs create mode 100644 apps/bench-expo/tsconfig.json create mode 100644 tests/perf/src/competitors.mjs create mode 100644 tests/perf/src/generate-screens.mjs diff --git a/AGENTS.md b/AGENTS.md index 78ae9e3aa..7a51b40f1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,8 @@ Re.Pack is a toolkit for building and developing React Native applications with - `apps/tester-app/`: Primary manual/integration validation app. - `apps/tester-federation/`: Module Federation v1 validation app. - `apps/tester-federation-v2/`: Module Federation v2 validation app. +- `apps/bench-expo/`, `apps/bench-bare/`: Neutral apps for benchmarking Re.Pack + against Metro (`tests/perf/src/competitors.mjs`). ### Tests diff --git a/agent_context/perf-regressions-oct2026/design.md b/agent_context/perf-regressions-oct2026/design.md index 8de0eab38..ec9bf9875 100644 --- a/agent_context/perf-regressions-oct2026/design.md +++ b/agent_context/perf-regressions-oct2026/design.md @@ -67,9 +67,47 @@ use loose thresholds with noise controls for time and memory.** ~1.2 GB main process), most likely from parallel loader workers scaling with CPU count. +## Competitor benchmark + +`tests/perf/src/competitors.mjs` compares Re.Pack with other bundlers at 0 / +1,000 / 3,000 generated screens (`tests/perf/src/generate-screens.mjs`): + +- `apps/bench-expo`: Metro via Expo CLI (optionally with Expo's experimental + tree shaking) vs Re.Pack with `repack-expo`. +- `apps/bench-bare`: Metro via the React Native CLI vs Re.Pack with the + `repack-init` template config. +- Planned: One (Vite native mode), pending a feasibility spike. + +Expo app results from Oct 2026 (M4 Pro, iOS): + +- **Re.Pack ahead:** JS 16–17% smaller, Hermes bytecode 12–13% smaller, ~3Γ— + less CPU per build, edit-to-bundle 1.5–2.5Γ— faster, dev server memory 37–62% + lower. +- **Metro ahead:** + - Cold builds are 1.3–1.9Γ— faster in wall time (Metro keeps ~8–9 cores busy, + Re.Pack ~1.7). + - Warm builds are 2.4–3.6Γ— faster. + - Dev server start is 1.3–2.0Γ— faster. + - Peak memory is lower at 3,000 screens. + +Pitfalls: + +- Metro's `--reset-cache` only resets once, so the harness deletes Metro's + temp-dir caches. +- Metro in `CI=1` mode doesn't watch files, so dev servers run without it. +- In a monorepo, Expo serves `/.expo/.virtual-metro-entry.bundle`. +- Bare Metro needs `watchFolders` for pnpm's root `node_modules`; watching the + whole repo made cold builds ~70% slower (8.7 s vs 5.1 s), which would be + unfair to Metro. +- Early smoke runs: on the bare app Re.Pack builds faster than Metro (3.0 s vs + 6.8 s cold at 100 screens) and starts its dev server faster (1.6 s vs 5.2 s), + the opposite of the Expo app. The Expo integration's overhead is a lead + for performance work. + ## Not covered yet -- Dev server startup, rebuild latency and development-mode bundles. +- Dev server startup, rebuild latency and development-mode bundles in the + per-PR gate (only the competitor benchmark measures them). - Module Federation testers. - Instruction-count benchmarks (CodSpeed simulation) for low-noise timing of JS hot paths; needs CodSpeed set up for the org. diff --git a/apps/bench-bare/.gitignore b/apps/bench-bare/.gitignore new file mode 100644 index 000000000..0d26bb65c --- /dev/null +++ b/apps/bench-bare/.gitignore @@ -0,0 +1,3 @@ +build/ +src/generated/screens/ +src/generated/components/ diff --git a/apps/bench-bare/app.json b/apps/bench-bare/app.json new file mode 100644 index 000000000..ceca6d27d --- /dev/null +++ b/apps/bench-bare/app.json @@ -0,0 +1,4 @@ +{ + "name": "BenchBare", + "displayName": "BenchBare" +} diff --git a/apps/bench-bare/assets/repack.png b/apps/bench-bare/assets/repack.png new file mode 100644 index 0000000000000000000000000000000000000000..0a4d653726aea283dc6ca0c9453563eb22a9798e GIT binary patch literal 2518 zcmV;{2`To8P)Px;kV!;ARCodHnR|?0MHRpcrI>1Ip*+g7-4-lT5ey2Ihi+m-0hQtdqY(p2Ma70{ z2>43cf=Z$gibmoq3K&Wh6#-KO^`90bK`^0co1zu^!t$;ywme0DzwI~M)3bATzulIE zbib4QcFs9-X71d%=g!RicEpIjvOBIX!(wALfbq(kr|5YFm`AQ%p;~SDU#91KKwYYAyB)!8ax7 zsvF>`9hsD4x!*vLx#{7}CmR1+tT;mIA*=_}O7k~CJgslmt?f_?rUhWjAix7%K? z#Fc4py!-UlY*A)W+>?AAb9l|bf4JS7TetiO<=x@-_e0+c*6Z8VKP}m_D4|>#^p4t@ zfm%NF=1okd+El$~Rkp!g)542OezsJ~CG zg5Se}mQK+t`=gx+r@Cr}vII&fh!b9ry#i0BbTpa9i=pTR?;B~y;QDEOI;ivE$@cIMEn+f;?bi+h& zT=ug9Zh~LKMi`9P=bY5Q!}SI5V)D0T9!gX^sjKHt@Ys=GO_#v4klRK3x!4H(XfSbN zOilk2F$Y4HnK*L%V87=4GQF+RPwv-cy-f1hHsVYB<-`^-(fsM4)HRjMi1DE|iO<4h zX=-2mx1O~xUydHxttbtp!8P%%a?q#NDwA)QWbcmM z&h~l&ajp#0p=lgc4O)B&ZiIHnzTcp)ux$hH)q5UDjv!zmyaAp`!IQA>f`gO4X6E^> z#T4IbP+SoYDGo~K`YiZ!@D;V+=MW$2o~1=7^G%hi`6x{fV*_=*(M^(X)>v1!nz#*} zQmfbwpZ&SLMp=V#9sCd$!;_FDk2}}5Yx_Wa^K0Ttd~T{R5pRQ9A~|=?_fWtOaznCU#VSDu7pjH3loh_kg=azpDrJMJ_@)yBMwVN0gT*Uo?H` zJ)cD}ZyZC9yu@DzkhyS6b)UaiRARr|Cu*xB{}5Udcx%K~?J(3lj_+)(?$JJ7iSZ#7 z8Htknm|rR>y{}wT%C)=zMYphm5Nq*7v;j_i)L*OWBnuyUxxUSEWlmbI?xY>>!6*v^-N`c6ftkkgAW#m<+~ten|rFsmgQC)F;2ZZ z2Q}F5uC%=opI=y0A&*LBaiaPs_^=@Nsne9qmnY{i;xD#T(>K8$O;@sx@d1tMG z4>NaOdkbb(;vU8~xzed$EnkDO@6Scg*NDrjlb7rL!iciH$>f7@91~)vL9g{cJJhTG z0P;8wOEH=st;C&>;@nDOUyD-xFH^m-H$z+>?-ip}){@DpN$_uo*LM3m9-73% zeT-K>v;@bMryPt+ou!$W<$4^elCul@0dSH(4|Bjtco@C`CY2Jex5cw%i**aw9ZoOK zP`WDk$?j6TBg8SxgVnGA?E4+CA54Q|Ad9h4vTdCY_+q#M7J^dSHa?DjUvPEkhVc;P zSd0tc$d>am%KU6Rnk)t9_I$V;ZU?upv!NaF8G7$gcEk8N)D{ur9CMaDww`)3%uI`T z(=~Ato`)FQpFX|S$tewOd?Q|Js4M-R*y>|uCGMzJorO1C+LSVGb0&82w^Obqel$e! zf%n&Tc`JBIL!1v_p!xe$s}GC0(BJi9UEXq=R;JSZwUxeaDV3>oHILU)pIl;gEy^Iv zMZ~-BKiy_KFRI)DH}q)R{_%~dW&O5MJI3-X*sg3K@Kks>!C^fiQ`gu0ZKbd$NdY4G6!Ql z#5na8@EMo~UNhF8jh$sWj)l;)ivNwe3sa+kRwl!b19yjbeeX+ulnEFyMtyP#aAJ!b zhVqYMv`WtrPb6Q>9EH;hcflAiiPpda;EmVBE!6G8gnS&l_A)4PFv=UjgQk;I#-=?E zqBzEQodaSex=FJ!`83_We2ESO+qgxn1g}+#$K6MqV7J*Wcna=^haijRLSKl9;ELyJ z;QW-8()lev zg<3y-V?*^x{kG~&Wn$0e{MQEBLkP g*Y0w71p3JT0aplb{WN5(_W%F@07*qoM6N<$f|Qc}&;S4c literal 0 HcmV?d00001 diff --git a/apps/bench-bare/babel.config.js b/apps/bench-bare/babel.config.js new file mode 100644 index 000000000..98655448c --- /dev/null +++ b/apps/bench-bare/babel.config.js @@ -0,0 +1,3 @@ +module.exports = { + presets: ['@react-native/babel-preset'], +}; diff --git a/apps/bench-bare/index.js b/apps/bench-bare/index.js new file mode 100644 index 000000000..9250f395b --- /dev/null +++ b/apps/bench-bare/index.js @@ -0,0 +1,5 @@ +import { AppRegistry } from 'react-native'; +import { name as appName } from './app.json'; +import App from './src/App'; + +AppRegistry.registerComponent(appName, () => App); diff --git a/apps/bench-bare/metro.config.js b/apps/bench-bare/metro.config.js new file mode 100644 index 000000000..a526ca9d8 --- /dev/null +++ b/apps/bench-bare/metro.config.js @@ -0,0 +1,9 @@ +const path = require('node:path'); +const { getDefaultConfig, mergeConfig } = require('@react-native/metro-config'); + +// React Native's default Metro config. The only addition lets Metro see the +// monorepo's root node_modules, where pnpm keeps the actual package files; +// watching the whole repo would make Metro crawl every other app too. +module.exports = mergeConfig(getDefaultConfig(__dirname), { + watchFolders: [path.resolve(__dirname, '../../node_modules')], +}); diff --git a/apps/bench-bare/package.json b/apps/bench-bare/package.json new file mode 100644 index 000000000..d14c9ea0f --- /dev/null +++ b/apps/bench-bare/package.json @@ -0,0 +1,32 @@ +{ + "name": "bench-bare", + "version": "0.0.1", + "private": true, + "description": "Neutral bare React Native app for benchmarking Re.Pack against plain Metro", + "scripts": { + "start": "react-native start", + "start:metro": "BENCH_BUNDLER=metro react-native start", + "bundle:ios": "react-native bundle --platform ios --dev false --entry-file index.js --bundle-output build/repack/main.jsbundle", + "bundle:ios:metro": "BENCH_BUNDLER=metro react-native bundle --platform ios --dev false --entry-file index.js --bundle-output build/metro/main.jsbundle --assets-dest build/metro", + "generate": "node ../../tests/perf/src/generate-screens.mjs bare .", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "react": "catalog:", + "react-native": "catalog:" + }, + "devDependencies": { + "@babel/core": "catalog:", + "@callstack/repack": "workspace:*", + "@react-native-community/cli": "catalog:testers", + "@react-native-community/cli-platform-android": "catalog:testers", + "@react-native-community/cli-platform-ios": "catalog:testers", + "@react-native/babel-preset": "catalog:testers", + "@react-native/metro-config": "catalog:testers", + "@react-native/typescript-config": "catalog:testers", + "@rspack/core": "catalog:", + "@swc/helpers": "catalog:", + "@types/react": "catalog:testers", + "typescript": "catalog:" + } +} diff --git a/apps/bench-bare/react-native.config.js b/apps/bench-bare/react-native.config.js new file mode 100644 index 000000000..c62c2e12f --- /dev/null +++ b/apps/bench-bare/react-native.config.js @@ -0,0 +1,8 @@ +// BENCH_BUNDLER=metro keeps the CLI's built-in Metro commands, so the same app +// can be bundled by both Metro and Re.Pack. +module.exports = { + commands: + process.env.BENCH_BUNDLER === 'metro' + ? [] + : require('@callstack/repack/commands'), +}; diff --git a/apps/bench-bare/rspack.config.mjs b/apps/bench-bare/rspack.config.mjs new file mode 100644 index 000000000..bc5702b1f --- /dev/null +++ b/apps/bench-bare/rspack.config.mjs @@ -0,0 +1,47 @@ +import path from 'node:path'; +import * as Repack from '@callstack/repack'; + +const dirname = Repack.getDirname(import.meta.url); + +// The config `@callstack/repack-init` installs (templates/rspack.config.mjs), +// plus an output path for the benchmark and its opt-in persistent cache variant. +export default Repack.defineRspackConfig((env) => ({ + context: dirname, + entry: './index.js', + resolve: { + ...Repack.getResolveOptions({ enablePackageExports: true }), + }, + output: { + path: + process.env.BENCH_OUTPUT_DIR ?? + path.join(dirname, 'build', 'rspack', env.platform), + }, + module: { + rules: [ + { + test: /\.[cm]?[jt]sx?$/, + type: 'javascript/auto', + use: { + loader: '@callstack/repack/babel-swc-loader', + parallel: true, + options: {}, + }, + }, + ...Repack.getAssetTransformRules(), + ], + }, + plugins: [new Repack.RepackPlugin()], + // Metro caches transforms by default; this is the equivalent opt-in. + ...(process.env.BENCH_RSPACK_CACHE && { + cache: true, + experiments: { + cache: { + type: 'persistent', + storage: { + type: 'filesystem', + directory: path.join(dirname, 'node_modules/.cache/rspack'), + }, + }, + }, + }), +})); diff --git a/apps/bench-bare/src/App.tsx b/apps/bench-bare/src/App.tsx new file mode 100644 index 000000000..ad7631e8c --- /dev/null +++ b/apps/bench-bare/src/App.tsx @@ -0,0 +1,120 @@ +import { lazy, type ReactNode, Suspense, useState } from 'react'; +import { + Button, + FlatList, + Image, + Pressable, + SafeAreaView, + StatusBar, + StyleSheet, + Text, + View, +} from 'react-native'; +import { screens } from './generated'; + +// Same features as apps/bench-expo, without Expo or a navigation library: a +// details view, a packaged image, a lazily loaded component, and the +// generated screens. +const LazyCard = lazy(() => import('./LazyCard')); + +type Route = + | { name: 'home' | 'details' | 'image' | 'lazy' } + | { name: 'screen'; index: number }; + +function Details({ onBack }: { onBack: () => void }) { + return ( + + React Native benchmark app +