feat(#575): capture the cloud E2E baseline before it becomes unrecoverable

scripts/__tests__/e2e-parity-diff.test.js

@@ -0,0 +1,190 @@
+/**
+ * Tests for the E2E parity comparator (#575).
+ *
+ * The comparator is the ONLY thing standing between "we switched E2E to a local
+ * Supabase" and "we switched E2E to a local Supabase and quietly stopped running 228
+ * messaging tests". If it can't fail, the switch is unlicensed. So these tests are
+ * mostly about proving it REJECTS things.
+ *
+ * Runs under `pnpm test:scripts` (node:test), which ci.yml executes. vitest cannot
+ * load `node:test`, hence the placement here — see vitest.config.ts:20-21.
+ */
+
+const test = require('node:test');
+const assert = require('node:assert');
+const path = require('node:path');
+const { pathToFileURL } = require('node:url');
+
+const MOD = pathToFileURL(
+  path.join(__dirname, '..', 'e2e-parity-diff.mjs')
+).href;
+
+/** Minimal Playwright-JSON-shaped report. */
+function report(entries) {
+  return {
+    suites: entries.map(([project, file, title, status]) => ({
+      file,
+      specs: [{ file, title, tests: [{ projectName: project, status }] }],
+      suites: [],
+    })),
+  };
+}
+
+test('extractTests flattens nested suites and keys on project|file|title', async () => {
+  const { extractTests } = await import(MOD);
+  const r = {
+    suites: [
+      {
+        file: 'a.spec.ts',
+        specs: [
+          {
+            file: 'a.spec.ts',
+            title: 'top',
+            tests: [{ projectName: 'chromium-gen', status: 'expected' }],
+          },
+        ],
+        suites: [
+          {
+            // No `file` here — must inherit from the parent suite, or nested describes
+            // key on an empty path and silently collide.
+            specs: [
+              {
+                title: 'nested',
+                tests: [{ projectName: 'chromium-gen', status: 'skipped' }],
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  };
+  const out = extractTests(r);
+  assert.strictEqual(out['chromium-gen|a.spec.ts|top'], 'expected');
+  assert.strictEqual(out['chromium-gen|a.spec.ts|nested'], 'skipped');
+  assert.strictEqual(Object.keys(out).length, 2);
+});
+
+test('identical input passes', async () => {
+  const { compare } = await import(MOD);
+  const b = { 'p|f|t': 'expected', 'p|f|u': 'skipped' };
+  assert.strictEqual(compare(b, { ...b }).ok, true);
+});
+
+test('expected -> skipped is a coverage LOSS and fails', async () => {
+  const { compare } = await import(MOD);
+  const b = { 'p|f|t': 'expected' };
+  const res = compare(b, { 'p|f|t': 'skipped' });
+  assert.strictEqual(res.ok, false);
+  assert.deepStrictEqual(res.lost, ['p|f|t']);
+});
+
+test('a test vanishing entirely fails', async () => {
+  const { compare } = await import(MOD);
+  const res = compare({ 'p|f|t': 'expected' }, {});
+  assert.strictEqual(res.ok, false);
+  assert.deepStrictEqual(res.missing, ['p|f|t']);
+});
+
+test('same COUNT but different tests still fails', async () => {
+  // The whole reason this compares identities. A count gate passes this.
+  const { compare } = await import(MOD);
+  const res = compare({ 'p|f|a': 'expected' }, { 'p|f|b': 'expected' });
+  assert.strictEqual(res.ok, false);
+  assert.strictEqual(res.missing.length, 1);
+  assert.strictEqual(res.added.length, 1);
+});
+
+test('skipped -> expected is a GAIN: reported, allowed', async () => {
+  // A local stack can legitimately enable a spec the cloud project could not run.
+  // That must not fail the gate, but it must be visible.
+  const { compare } = await import(MOD);
+  const res = compare({ 'p|f|t': 'skipped' }, { 'p|f|t': 'expected' });
+  assert.strictEqual(res.ok, true);
+  assert.deepStrictEqual(res.gained, ['p|f|t']);
+});
+
+test('a genuine failure (expected -> unexpected) fails the gate', async () => {
+  const { compare } = await import(MOD);
+  const res = compare({ 'p|f|t': 'expected' }, { 'p|f|t': 'unexpected' });
+  assert.strictEqual(res.ok, false);
+  assert.strictEqual(res.changed.length, 1);
+});
+
+test('flaky is not treated as a pass', async () => {
+  const { compare } = await import(MOD);
+  const res = compare({ 'p|f|t': 'expected' }, { 'p|f|t': 'flaky' });
+  assert.strictEqual(res.ok, false);
+});
+
+test('brand-new tests are allowed but reported', async () => {
+  const { compare } = await import(MOD);
+  const res = compare({}, { 'p|f|new': 'expected' });
+  assert.strictEqual(res.ok, true);
+  assert.deepStrictEqual(res.added, ['p|f|new']);
+});
+
+test('the committed baseline is well-formed and self-consistent', async () => {
+  const fs = require('node:fs');
+  const p = path.join(
+    __dirname,
+    '..',
+    '..',
+    'tests',
+    'e2e',
+    'parity',
+    'baseline-de0f7f0.json'
+  );
+  const m = JSON.parse(fs.readFileSync(p, 'utf8'));
+
+  assert.strictEqual(m.sha, 'de0f7f080c8d75949e4e6c89fdf66ab7d3da8029');
+  assert.strictEqual(m.backend, 'cloud');
+  assert.deepStrictEqual(
+    m.duplicateKeys,
+    [],
+    'keys must be unique or the diff is unsound'
+  );
+
+  const statuses = Object.values(m.tests);
+  assert.strictEqual(statuses.length, m.totals.tests);
+  assert.strictEqual(
+    statuses.filter((s) => s === 'expected').length,
+    m.totals.expected
+  );
+  assert.strictEqual(
+    statuses.filter((s) => s === 'skipped').length,
+    m.totals.skipped
+  );
+  // The numbers quoted throughout #575 and its PRs. If these ever change, the
+  // baseline was regenerated and every claim referencing them needs revisiting.
+  assert.strictEqual(m.totals.expected, 1807);
+  assert.strictEqual(m.totals.skipped, 194);
+  assert.strictEqual(m.totals.tests, 2001);
+});
+
+test('the baseline round-trips through compare() against itself', async () => {
+  const fs = require('node:fs');
+  const { compare } = await import(MOD);
+  const p = path.join(
+    __dirname,
+    '..',
+    '..',
+    'tests',
+    'e2e',
+    'parity',
+    'baseline-de0f7f0.json'
+  );
+  const m = JSON.parse(fs.readFileSync(p, 'utf8'));
+  const res = compare(m.tests, { ...m.tests });
+  assert.strictEqual(res.ok, true);
+});
+
+test('report() shape: extract on a synthetic report matches compare expectations', async () => {
+  const { extractTests, compare } = await import(MOD);
+  const base = extractTests(
+    report([['chromium-gen', 'x.spec.ts', 'one', 'expected']])
+  );
+  const now = extractTests(
+    report([['chromium-gen', 'x.spec.ts', 'one', 'skipped']])
+  );
+  assert.strictEqual(compare(base, now).ok, false);
+});

scripts/e2e-parity-diff.mjs

@@ -0,0 +1,195 @@
+#!/usr/bin/env node
+/**
+ * E2E parity diff — compare a Playwright JSON report against a captured baseline.
+ *
+ * #575. Switching E2E from the shared cloud project to a per-runner ephemeral stack
+ * is only safe if the local run covers the SAME TESTS. Counts alone cannot show that:
+ * a suite can drop one test and gain another and still total 2001.
+ *
+ * So this compares per-test IDENTITIES, not totals, and it is directional — a test
+ * that went `expected -> skipped` is a coverage LOSS and fails; the reverse is a gain
+ * and is reported but allowed (a local stack legitimately enables things cloud could
+ * not run, e.g. a spec gated on a service the cloud project lacks).
+ *
+ *   node scripts/e2e-parity-diff.mjs <merged-report.json> [--baseline <path>] [--json]
+ *   node scripts/e2e-parity-diff.mjs --selftest
+ *
+ * WHY A SELFTEST. Every probe written in this repo that could not fail turned out to
+ * be wrong (#396, and four in one session). `--selftest` mutates a copy of the baseline
+ * three ways and asserts this script REJECTS each. Run it in CI next to the real diff;
+ * a comparison tool that cannot fail is worse than none, because it licenses the switch.
+ */
+
+import { readFileSync, existsSync } from 'node:fs';
+import { pathToFileURL } from 'node:url';
+
+const DEFAULT_BASELINE = 'tests/e2e/parity/baseline-de0f7f0.json';
+
+/** Extract {key -> status} from a Playwright JSON report. Key excludes line numbers
+ *  deliberately: they shift on any edit above a test and would churn without any
+ *  change in coverage. */
+export function extractTests(report) {
+  const out = {};
+  const walk = (suite, inheritedFile) => {
+    const file = suite.file || inheritedFile;
+    for (const spec of suite.specs || []) {
+      for (const t of spec.tests || []) {
+        out[`${t.projectName || ''}|${spec.file || file}|${spec.title || ''}`] =
+          t.status || '';
+      }
+    }
+    for (const s of suite.suites || []) walk(s, file);
+  };
+  for (const s of report.suites || []) walk(s, s.file);
+  return out;
+}
+
+/** Directional comparison. Returns {lost, missing, added, gained, changed, ok}. */
+export function compare(baselineTests, actualTests) {
+  const lost = []; // expected -> skipped: COVERAGE LOSS
+  const missing = []; // present in baseline, absent entirely: COVERAGE LOSS
+  const added = []; // absent from baseline, present now: report only
+  const gained = []; // skipped -> expected: report only
+  const changed = []; // any other status transition
+
+  for (const [k, was] of Object.entries(baselineTests)) {
+    if (!(k in actualTests)) {
+      missing.push(k);
+      continue;
+    }
+    const now = actualTests[k];
+    if (was === now) continue;
+    if (was === 'expected' && now === 'skipped') lost.push(k);
+    else if (was === 'skipped' && now === 'expected') gained.push(k);
+    else changed.push(`${k}  (${was} -> ${now})`);
+  }
+  for (const k of Object.keys(actualTests)) {
+    if (!(k in baselineTests)) added.push(k);
+  }
+
+  // `changed` includes transitions to `unexpected` (a real failure) and `flaky`.
+  // Both must fail the gate — a parity run that goes red on a test the cloud passed
+  // is precisely the signal this exists to surface.
+  return {
+    lost,
+    missing,
+    added,
+    gained,
+    changed,
+    ok: lost.length === 0 && missing.length === 0 && changed.length === 0,
+  };
+}
+
+function loadReport(path) {
+  const raw = JSON.parse(readFileSync(path, 'utf8'));
+  // Accept either a raw Playwright report or a previously-written baseline manifest.
+  if (raw.tests && !raw.suites) return raw.tests;
+  return extractTests(raw);
+}
+
+function report(res, baseCount, actualCount) {
+  const line = (label, arr, fatal) => {
+    if (!arr.length) return;
+    console.log(`\n${fatal ? '✗' : 'ℹ'} ${label}: ${arr.length}`);
+    for (const k of arr.slice(0, 25)) console.log(`    ${k}`);
+    if (arr.length > 25) console.log(`    … and ${arr.length - 25} more`);
+  };
+  console.log(`baseline tests: ${baseCount}`);
+  console.log(`actual tests:   ${actualCount}`);
+  line('COVERAGE LOST (ran on baseline, skipped now)', res.lost, true);
+  line('MISSING ENTIRELY (in baseline, absent now)', res.missing, true);
+  line('STATUS REGRESSION', res.changed, true);
+  line('gained (skipped on baseline, runs now)', res.gained, false);
+  line('new tests (not in baseline)', res.added, false);
+}
+
+function selftest() {
+  if (!existsSync(DEFAULT_BASELINE)) {
+    console.error(`✗ selftest needs ${DEFAULT_BASELINE}`);
+    process.exit(1);
+  }
+  const base = JSON.parse(readFileSync(DEFAULT_BASELINE, 'utf8')).tests;
+  const keys = Object.keys(base);
+  const ranKey = keys.find((k) => base[k] === 'expected');
+  let failures = 0;
+  const check = (name, actual, shouldPass) => {
+    const res = compare(base, actual);
+    const good = res.ok === shouldPass;
+    console.log(
+      `  ${good ? '✓' : '✗'} ${name} — ok=${res.ok}, want ${shouldPass}`
+    );
+    if (!good) failures++;
+  };
+
+  // 1. Identity: the baseline against itself MUST pass, or every other case is noise.
+  check('identity (baseline vs itself) passes', { ...base }, true);
+
+  // 2. A test silently flipping to skipped MUST be rejected. This is the exact
+  //    failure mode: 228 *-msg-iso tests sit behind `test.skip(!fixture, …)`.
+  const flipped = { ...base, [ranKey]: 'skipped' };
+  check('expected -> skipped is REJECTED', flipped, false);
+
+  // 3. A test vanishing entirely MUST be rejected — a dropped shard or project.
+  const dropped = { ...base };
+  delete dropped[ranKey];
+  check('missing test is REJECTED', dropped, false);
+
+  // 4. Swapping one test for another keeps the COUNT identical. A count-based gate
+  //    passes here; this one must not.
+  const swapped = { ...base };
+  delete swapped[ranKey];
+  swapped['chromium-gen|tests/e2e/fake.spec.ts|invented'] = 'expected';
+  check('same count, different tests is REJECTED', swapped, false);
+
+  console.log(
+    failures ? `\n✗ selftest FAILED (${failures})` : '\n✓ selftest passed'
+  );
+  process.exit(failures ? 1 : 0);
+}
+
+// Only run the CLI when invoked directly. Without this guard, importing this module
+// from a test executes the argument parsing below and exits the test runner.
+const isMain =
+  process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
+if (!isMain) {
+  // Imported for its exports (see scripts/__tests__/e2e-parity-diff.test.js).
+} else {
+  runCli();
+}
+
+function runCli() {
+  const argv = process.argv.slice(2);
+  if (argv.includes('--selftest')) selftest();
+
+  const asJson = argv.includes('--json');
+  const bi = argv.indexOf('--baseline');
+  // `bi + 1` is only a real argument when --baseline was actually passed. Without this
+  // guard bi is -1, argv[0] IS the report path, and the filter below discards it —
+  // which made every real invocation print usage and exit 2 while --selftest passed.
+  const baselineArg = bi >= 0 ? argv[bi + 1] : null;
+  const baselinePath = baselineArg ?? DEFAULT_BASELINE;
+  const reportPath = argv.find((a) => !a.startsWith('--') && a !== baselineArg);
+
+  if (!reportPath) {
+    console.error(
+      'Usage: node scripts/e2e-parity-diff.mjs <merged-report.json> [--baseline <path>] [--json]\n' +
+        '       node scripts/e2e-parity-diff.mjs --selftest'
+    );
+    process.exit(2);
+  }
+
+  const baseline = JSON.parse(readFileSync(baselinePath, 'utf8')).tests;
+  const actual = loadReport(reportPath);
+  const res = compare(baseline, actual);
+
+  if (asJson) console.log(JSON.stringify(res, null, 2));
+  else report(res, Object.keys(baseline).length, Object.keys(actual).length);
+
+  if (!res.ok) {
+    console.log(
+      `\n✗ PARITY FAILED — ${res.lost.length} lost, ${res.missing.length} missing, ${res.changed.length} regressed.`
+    );
+    process.exit(1);
+  }
+  console.log('\n✓ parity holds — every baseline test still runs.');
+}