perf(studio): speed up the edit accuracy bench and trim its grid

packages/studio/tests/e2e/edit-accuracy/case.mjs

@@ -231,7 +231,8 @@ export async function measure(ctx) {
   return { map, quad, size, visible: visibleQuad(quad, size, parseInset(box.clip)) };
 }
 
-const STILL_MS = 1000;
+// 500 ms, from 1000: the window restarts on any change, so a shorter one still waits out a slow save or swap.
+const STILL_MS = Number(process.env.EDIT_ACCURACY_STILL_MS ?? 500);
 
 // Studio reloads an edited preview in a shadow iframe (`_t` in its URL) and swaps it in when painted.
 const previewFrames = (page) =>
@@ -303,7 +304,6 @@ export async function openStudio(ctx) {
     seek = null;
   }
   if (!seek) throw new Error("studio never reported a seekable composition");
-  await sleep(1000);
   return settled(ctx);
 }
 
@@ -675,8 +675,23 @@ export function smoothness(rec) {
 const CONTROL_PAGE = `data:text/html,<body style="margin:0;background:%23202020"><div id="box"
   style="position:absolute;left:600px;top:300px;width:240px;height:160px;background:%23f0c020"></div>`;
 
+// Reported, never gated, so it runs once per gesture per browser rather than once per case.
+const controls = new WeakMap();
+
 /** The case's drag schedule and per-frame reads on a blank page in the same Chrome: the machine's own frame drops. */
-export async function controlDrag(browser, gesture) {
+export function controlDrag(browser, gesture) {
+  const byGesture = controls.get(browser) ?? new Map();
+  controls.set(browser, byGesture);
+  if (!byGesture.has(gesture)) {
+    const measured = measureControl(browser, gesture);
+    // A failed control is not cached, so one transient failure does not fail every later case.
+    measured.catch(() => byGesture.get(gesture) === measured && byGesture.delete(gesture));
+    byGesture.set(gesture, measured);
+  }
+  return byGesture.get(gesture);
+}
+
+async function measureControl(browser, gesture) {
   const context = await browser.createBrowserContext();
   try {
     const page = await context.newPage();

packages/studio/tests/e2e/edit-accuracy/grid.mjs

@@ -27,13 +27,16 @@ export function localAsset(url) {
 
 // Studio has corner handles only (ResizeHandle is nw|ne|sw|se); its edge strips crop, so there is no edge resize.
 const GESTURES = ["move", "resize", "rotate", "crop", "nudge"];
+// Trimmed from 2061 cases: across 444 historical result snapshots, zoom 200, the pct and center placements and most nested
+// keyframed cases never failed where a kept case with the same gesture, GSAP mode and playhead did not. Drag paths and
+// sequences still cover pct, center and zoom 200.
 const AXES = {
   // idle: GSAP loaded and a paused timeline tweens another element; the target itself is plain CSS.
   gsap: ["none", "idle", "tween", "hold"],
-  placement: ["px", "pct", "center", "xpercent"],
+  placement: ["px", "xpercent"],
   rotation: [0, 30],
   nesting: ["root", "nested"],
-  zoom: [50, 100, 200],
+  zoom: [50, 100],
 };
 
 // Timelines with keyframes at `times` (0, 2 and 3 s unless set) on the property each animates; `css` is it in CSS.
@@ -105,23 +108,30 @@ function keyframedCases() {
     nesting: AXES.nesting,
     zoom: AXES.zoom,
     gesture: GESTURES,
-  }).flatMap((c) => {
-    const times = KEYFRAMED[c.gsap].times ?? KEY_TIMES;
-    return playheadsFor(times).map((at) => ({
-      id: `${caseId(c)}-${at}`,
-      ...c,
-      at,
-      playhead: AT[at],
-      keys: {
-        times: times.filter((t) => t !== AT[at]),
-        props: KEYFRAMED[c.gsap].props,
-        css: KEYFRAMED[c.gsap].css,
-        render: times.at(-1),
-      },
-    }));
-  });
+  })
+    .flatMap((c) => {
+      const times = KEYFRAMED[c.gsap].times ?? KEY_TIMES;
+      return playheadsFor(times).map((at) => ({
+        id: `${caseId(c)}-${at}`,
+        ...c,
+        at,
+        playhead: AT[at],
+        keys: {
+          times: times.filter((t) => t !== AT[at]),
+          props: KEYFRAMED[c.gsap].props,
+          css: KEYFRAMED[c.gsap].css,
+          render: times.at(-1),
+        },
+      }));
+    })
+    .filter((c) => c.nesting === "root" || NESTED_KEYFRAMED(c));
 }
 
+// Nested keyframed cases are kept only for a resize on a keyframe at zoom 100 without rotation, so the PR slice
+// still catches the nested resize regression two historical gate runs failed on (resize-scale-px-r0-nested-z100-on).
+const NESTED_KEYFRAMED = (c) =>
+  c.gesture === "resize" && c.at === "on" && c.zoom === 100 && c.rotation === 0;
+
 const product = (axes) =>
   Object.entries(axes).reduce(
     (rows, [key, values]) => rows.flatMap((row) => values.map((v) => ({ ...row, [key]: v }))),

packages/studio/tests/e2e/edit-accuracy/grid.test.mjs

@@ -28,3 +28,12 @@ describe("grid fixtures", () => {
     expect([...new Set(unserved)]).toEqual([]);
   }, 30_000);
 });
+
+describe("grid size", () => {
+  it("holds 747 cases, 271 of them in the pull request slice, each id once", () => {
+    const full = buildGrid("full");
+    expect(full).toHaveLength(747);
+    expect(new Set(full.map((c) => c.id)).size).toBe(747);
+    expect(buildGrid("pr")).toHaveLength(271);
+  });
+});

packages/studio/tests/e2e/edit-accuracy/keyframes.test.mjs

@@ -81,8 +81,16 @@ describe("strayCss", () => {
 describe("the keyframed grid", () => {
   const grid = buildGrid("keyframes");
 
-  it("crosses 6 variants, 5 gestures and 3 playheads each with rotation, nesting and zoom", () => {
-    expect(grid).toHaveLength(6 * 5 * 3 * 2 * 2 * 3);
+  it("crosses 6 variants, 5 gestures and 3 playheads each with rotation and zoom at the root, plus a nested resize per variant", () => {
+    expect(grid).toHaveLength(6 * 5 * 3 * 2 * 2 + 6);
+    const nested = grid.filter((c) => c.nesting === "nested");
+    expect(nested.map((c) => c.id)).toContain("resize-scale-px-r0-nested-z100-on");
+    expect(nested).toHaveLength(6);
+    expect(
+      nested.every(
+        (c) => c.gesture === "resize" && c.at === "on" && c.zoom === 100 && c.rotation === 0,
+      ),
+    ).toBe(true);
     expect(new Set(grid.map((c) => c.id)).size).toBe(grid.length);
     expect(new Set(grid.map((c) => c.gsap))).toEqual(
       new Set(["size", "scale", "spin", "keys", "fromto", "late"]),
@@ -108,7 +116,11 @@ describe("the keyframed grid", () => {
   });
 
   it("writes each variant's timeline into the file that holds the target", () => {
-    const spec = grid.find((c) => c.id === "move-fromto-px-r30-nested-z100-on");
+    // Nested keyframed cases left the grid; the fixture still writes them.
+    const spec = {
+      ...grid.find((c) => c.id === "move-fromto-px-r30-root-z100-on"),
+      nesting: "nested",
+    };
     const dir = mkdtempSync(join(tmpdir(), "edit-bench-keys-"));
     writeFixture(spec, dir);
     const sub = readFileSync(join(dir, "compositions/sub.html"), "utf8");

packages/studio/tests/e2e/edit-accuracy/ratchet.mjs

@@ -59,10 +59,12 @@ export const QUARANTINED = {};
 
 /**
  * Every run of every case: each shard's run plus the re-runs of the cases it flipped. `scope`, for a run of
- * part of the grid, limits the base branch's passing count to the cases that ran.
+ * part of the grid, limits the base branch's passing count to the cases that ran. A case removed from the grid
+ * on purpose leaves the head's baseline.json in the same change: it is listed as retired and no longer counted.
  */
 // fallow-ignore-next-line complexity
 export function gate(base, head, runs, quarantine = QUARANTINED, scope = null) {
+  const counted = (id) => Object.hasOwn(head.cases, id) && (!scope || scope.has(id));
   const seen = new Map();
   for (const r of runs) seen.set(r.id, [...(seen.get(r.id) ?? []), entry(r)]);
   const all = [...seen].map(([id, entries]) => {
@@ -81,7 +83,7 @@ export function gate(base, head, runs, quarantine = QUARANTINED, scope = null) {
       return { id, fixer, passed: c?.passed ?? null, runs: c?.entries.map(accurate) ?? [] };
     }),
     basePassing: Object.entries(base.cases).filter(
-      ([id, e]) => !Object.hasOwn(quarantine, id) && (!scope || scope.has(id)) && accurate(e),
+      ([id, e]) => !Object.hasOwn(quarantine, id) && counted(id) && accurate(e),
     ).length,
     headPassing: passing.length,
     headSmooth: passing.filter((c) => mostlySmooth(c.entries)).length,
@@ -92,7 +94,8 @@ export function gate(base, head, runs, quarantine = QUARANTINED, scope = null) {
     newlyPassing: passing.filter((c) => !c.basePassed).map((c) => c.id),
     unbanked: passing.filter((c) => !accurate(head.cases[c.id])).map((c) => c.id),
     overclaimed: cases.filter((c) => !c.passed && accurate(head.cases[c.id])).map((c) => c.id),
-    missing: Object.keys(base.cases).filter((id) => (!scope || scope.has(id)) && !seen.has(id)),
+    missing: Object.keys(base.cases).filter((id) => counted(id) && !seen.has(id)),
+    retired: Object.keys(base.cases).filter((id) => !Object.hasOwn(head.cases, id)),
   };
   const reasons = [
     result.regressed.length &&
@@ -131,6 +134,7 @@ export function comment(g) {
     ),
     ...list("Marked passing in baseline.json but failing", g.overclaimed),
     ...list("In the base grid but not run", g.missing),
+    ...list("Retired: removed from baseline.json with the grid, not counted", g.retired),
     `**Quarantined, measured but not gated** (${g.quarantined.length})`,
     ...g.quarantined.map(
       (q) =>

packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs

@@ -158,6 +158,12 @@ describe("gate", () => {
     expect(g.reasons.join()).toContain("fell from 2 to 1");
   });
 
+  it("does not count a case retired from the grid with its baseline.json entry, and lists it", () => {
+    const g = gate(base, baseline({ a: good }), [run("a")]);
+    expect(g).toMatchObject({ basePassing: 1, missing: [], retired: ["b"], ok: true });
+    expect(comment(g)).toContain("Retired");
+  });
+
   it("counts only the cases that ran when a run covers part of the grid, still failing a regression", () => {
     const slice = new Set(["a"]);
     expect(gate(base, base, [run("a")], {}, slice)).toMatchObject({ missing: [], ok: true });

packages/studio/tests/e2e/edit-accuracy/render.mjs

@@ -1,5 +1,5 @@
 /** Render drift: the target's box in a frame the producer captures, found by its flat colour. */
-import { mkdirSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 // Source import, as scripts/preview-capture.ts does: bun runs the producer's TypeScript directly.
 import {
@@ -63,6 +63,22 @@ const luminance = (hex) => {
   return 0.299 * r + 0.587 * g + 0.114 * b;
 };
 
+/** A saved project with no GSAP timeline; a keyframe added in Studio writes one, so the files decide, not the spec. */
+const noTimeline = (dir) =>
+  !["index.html", "compositions/sub.html"].some(
+    (f) => existsSync(join(dir, f)) && readFileSync(join(dir, f), "utf8").includes("__timelines"),
+  );
+
+/** Runs in the producer's page: marks every composition host `data-no-timeline`, the engine's opt-out of the wait. */
+function markNoTimeline() {
+  const mark = (e) => e.matches("[data-composition-id]") && e.setAttribute("data-no-timeline", "");
+  new MutationObserver((records) => {
+    for (const r of records)
+      for (const n of r.addedNodes)
+        if (n.nodeType === 1) [n, ...n.querySelectorAll("[data-composition-id]")].forEach(mark);
+  }).observe(document, { childList: true, subtree: true });
+}
+
 /** Renders `dir` at `time` and measures the target's box in the frame. */
 export async function renderBox(dir, decoder, time = PLAYHEAD) {
   const framesDir = join(dir, ".bench-frames");
@@ -87,6 +103,8 @@ export async function renderBox(dir, decoder, time = PLAYHEAD) {
     );
     // The producer's own browser would fetch GSAP from the CDN; a stalled fetch renders the 0 s frame.
     await serveFixtureAssetsLocally(session.page);
+    // Without the opt-out, a project with no timeline waits out playerReadyTimeout on every render.
+    if (noTimeline(dir)) await session.page.evaluateOnNewDocument(markNoTimeline);
     await initializeSession(session);
     const { buffer } = await captureFrameToBuffer(session, 0, time);
     // Diagnostic only: a DOM rect ignores clip-path, so it cannot score a crop.

packages/studio/tests/e2e/edit-accuracy/render.test.mjs

@@ -1,4 +1,4 @@
-import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { createRequire } from "node:module";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -18,7 +18,9 @@ const chromeUrlPattern = (pattern) =>
 const sent = [];
 const handlers = {};
 const sessionArgs = [];
+const onNewDocument = [];
 const page = {
+  evaluateOnNewDocument: async (fn) => void onNewDocument.push(fn),
   createCDPSession: async () => ({
     on: (event, handler) => (handlers[event] = handler),
     send: async (method, params) => void sent.push({ method, params }),
@@ -36,16 +38,25 @@ vi.mock("../../../../producer/src/index.js", () => ({
   closeCaptureSession: async () => undefined,
 }));
 
+const decoder = { evaluate: async () => ({ left: 0, right: 1, top: 0, bottom: 1, area: 1 }) };
+
+async function render(html) {
+  const { renderBox } = await import("./render.mjs");
+  const dir = mkdtempSync(join(tmpdir(), "edit-bench-render-"));
+  if (html) writeFileSync(join(dir, "index.html"), html);
+  onNewDocument.length = 0;
+  try {
+    await renderBox(dir, decoder);
+  } finally {
+    rmSync(dir, { recursive: true });
+  }
+  return onNewDocument.map((fn) => fn.name);
+}
+
 describe("renderBox", () => {
   it("serves the fixture's GSAP from the repo and opens no second page that would fetch it", async () => {
-    const { renderBox } = await import("./render.mjs");
-    const dir = mkdtempSync(join(tmpdir(), "edit-bench-render-"));
-    const decoder = { evaluate: async () => ({ left: 0, right: 1, top: 0, bottom: 1, area: 1 }) };
-    try {
-      await renderBox(dir, decoder);
-    } finally {
-      rmSync(dir, { recursive: true });
-    }
+    sent.length = 0;
+    await render();
     expect(sent.map((s) => s.method)).toEqual(["Fetch.enable", "Fetch.fulfillRequest"]);
     const { patterns } = sent[0].params;
     expect(patterns).toEqual([{ urlPattern: patterns[0].urlPattern }]);
@@ -54,4 +65,9 @@ describe("renderBox", () => {
     expect(sent[1].params.body).toBe(gsap);
     expect(sessionArgs[0][4]).toMatchObject({ staticFrameDedup: false });
   });
+
+  it("opts a project with no timeline out of the engine's timeline wait, and only such a project", async () => {
+    expect(await render("<div data-composition-id='main'></div>")).toEqual(["markNoTimeline"]);
+    expect(await render("<script>window.__timelines = {};</script>")).toEqual([]);
+  });
 });

packages/studio/tests/e2e/edit-accuracy/smoothness.test.mjs

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { smoothness } from "./case.mjs";
+import { controlDrag, smoothness } from "./case.mjs";
 
 const thread = { pid: 1, tid: 2 };
 const mark = { name: "edit-bench-end", cat: "blink.user_timing", ts: 1_000_000, ...thread };
@@ -26,3 +26,18 @@ describe("smoothness work per frame", () => {
     expect(s.unknown).toBe("no end mark in the trace");
   });
 });
+
+describe("controlDrag", () => {
+  it("measures again after a failed control instead of failing every later case", async () => {
+    let calls = 0;
+    const browser = {
+      createBrowserContext: async () => {
+        calls += 1;
+        throw new Error(`blank page ${calls} failed`);
+      },
+    };
+    await expect(controlDrag(browser, "move")).rejects.toThrow("blank page 1 failed");
+    await expect(controlDrag(browser, "move")).rejects.toThrow("blank page 2 failed");
+    expect(calls).toBe(2);
+  });
+});