perf(studio): settle the edit accuracy bench on frames and network, and reuse the producer Chrome

packages/studio/AGENTS.md

@@ -134,6 +134,14 @@ bun run --cwd packages/studio test:edit-accuracy -- --grid pr --filter '^resize-
 - Each run writes its results, a table and a candidate baseline to
   `tests/e2e/evidence/edit-accuracy/<run>/` (git-ignored; `--out` moves it); a
   failing case also gets its screens and saved files under `cases/<id>/`.
+- A measurement waits until the box and the preview frames read the same on 3
+  animation frames in a row, with no Studio request in flight and every file
+  write announced by the server. `EDIT_ACCURACY_STILL_MS=500` swaps that for
+  the older fixed window of 500 ms with no network signal, if the faster check
+  ever proves flaky on CI.
+- The run keeps one producer Chrome open for every render; each render still
+  gets its own page and file server. Studio's server is started per case,
+  because it serves one project folder and keeps that folder's undo history.
 - CI runs the full grid in shards and `ratchet.mjs` gates the result against the
   base branch's `baseline.json`; its header states the rules. When cases newly
   pass, commit the `baseline.json` from the `edit-accuracy-gate` artifact.

packages/studio/tests/e2e/edit-accuracy/case.mjs

@@ -2,7 +2,7 @@
 import { spawn } from "node:child_process";
 import { classifyTweenPropertyGroup } from "../../../../parsers/src/gsapConstants.ts";
 import { parseGsapScript } from "../../../../parsers/src/gsapParser.ts";
-import { readFileSync } from "node:fs";
+import { readFileSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { COMPOSITION, FIXTURE_CDN, PLAYHEAD, localAsset } from "./grid.mjs";
 import {
@@ -85,7 +85,7 @@ export async function startServer(cli, dir, port, log, home) {
   child.once("exit", () => liveServers.delete(child));
   child.stdout.on("data", (d) => log.push(String(d)));
   child.stderr.on("data", (d) => log.push(String(d)));
-  for (const deadline = Date.now() + 60_000; Date.now() < deadline; await sleep(200)) {
+  for (const deadline = Date.now() + 60_000; Date.now() < deadline; await sleep(50)) {
     if (child.exitCode !== null)
       throw new Error(`studio exited ${child.exitCode}: ${log.join("").slice(-500)}`);
     const announced = Number(announcedPort(log));
@@ -231,8 +231,64 @@ export async function measure(ctx) {
   return { map, quad, size, visible: visibleQuad(quad, size, parseInset(box.clip)) };
 }
 
-// 500 ms, from 1000: the window restarts on any change, so a shorter one still waits out a slow save or swap.
-const STILL_MS = Number(process.env.EDIT_ACCURACY_STILL_MS ?? 500);
+// Still means the same reads on STILL_FRAMES animation frames in a row with the page's network quiet.
+// EDIT_ACCURACY_STILL_MS restores the old check: a fixed window, in ms, of unchanged reads with no network signal.
+const STILL_MS = process.env.EDIT_ACCURACY_STILL_MS
+  ? Number(process.env.EDIT_ACCURACY_STILL_MS)
+  : null;
+export const STILL_FRAMES = 3;
+
+// The server's event stream stays open for the page's life; timeline thumbnails are rendered by the server
+// for the timeline strip, can take seconds, and never change the preview.
+const backgroundRequest = (request) =>
+  request.resourceType() === "eventsource" ||
+  /\/api\/projects\/[^/]+\/thumbnail\//.test(request.url());
+
+/**
+ * The Studio page's network, for the stillness check: requests in flight, a counter bumped by any request or
+ * server event, and when the server last announced each project file. Studio's save requests, its reads of
+ * the files and the preview's loads all show here; a server `file-change` event can start another reload.
+ */
+async function watchNetwork(page) {
+  const net = { inflight: new Set(), activity: 0, announced: new Map(), since: Date.now() };
+  const done = (request) => {
+    net.inflight.delete(request);
+    net.activity++;
+  };
+  page.on("request", (request) => {
+    if (!backgroundRequest(request)) net.inflight.add(request);
+    net.activity++;
+  });
+  page.on("requestfinished", done);
+  page.on("requestfailed", done);
+  const cdp = await page.createCDPSession();
+  cdp.on("Network.eventSourceMessageReceived", ({ eventName, data }) => {
+    net.activity++;
+    if (eventName === "file-change") net.announced.set(JSON.parse(data).path, Date.now());
+  });
+  await cdp.send("Network.enable");
+  return net;
+}
+
+/** A file written after the server last announced it: its `file-change` event, and what Studio does on it, is still to come. */
+const unannounced = ({ dir, files, net }) =>
+  files.some(
+    (f) => statSync(join(dir, f)).mtimeMs > Math.max(net.since, net.announced.get(f) ?? 0),
+  );
+
+/**
+ * Whether the page is waiting on the network. A request whose frame is gone never finishes, so it is dropped,
+ * and counted as activity like a finished one: a window that began while it was in flight then restarts.
+ */
+// fallow-ignore-next-line complexity
+export function networkBusy(ctx) {
+  for (const request of ctx.net.inflight)
+    if (request.frame()?.detached !== false) {
+      ctx.net.inflight.delete(request);
+      ctx.net.activity++;
+    }
+  return { busy: ctx.net.inflight.size > 0 || unannounced(ctx), activity: ctx.net.activity };
+}
 
 // Studio reloads an edited preview in a shadow iframe (`_t` in its URL) and swaps it in when painted.
 const previewFrames = (page) =>
@@ -255,55 +311,81 @@ async function hiddenTarget(page, selector = "#target") {
 // Keyframed cases only: waiting out the swap lands a later undo in the preview's burst of requests.
 export const swapPending = (ctx) => Boolean(ctx.keys) && hiddenTarget(ctx.page, ctx.selector);
 
-/** Restart the stillness window: a pending swap, a changed set of preview frames, or the box moved. */
+// fallow-ignore-next-line complexity
+const netMoved = (start, now) =>
+  Boolean(now.net?.busy) || now.net?.activity !== start.net?.activity;
+
+/** Restart the stillness window: a pending swap, a changed set of preview frames, the box moved, or network. */
+// fallow-ignore-next-line complexity
 export const unsettledBy = (start, now) =>
   start.pending ||
   now.pending ||
+  netMoved(start, now) ||
   now.frames !== start.frames ||
   quadDistance(now.m.visible, start.m.visible) >= 0.01;
 
-/** Measures once the shown preview and the box have held still for STILL_MS; Studio updates both after a save. */
+/** Whether a window that began at `start`, with `still` unchanged reads since, has held long enough. */
+// fallow-ignore-next-line complexity
+export const heldStill = (start, still, since, stillMs = STILL_MS) =>
+  !start.pending &&
+  !start.net?.busy &&
+  (stillMs === null ? still >= STILL_FRAMES : Date.now() - since >= stillMs);
+
+/** Measures once the shown preview and the box have held still; Studio updates both after a save. */
 // fallow-ignore-next-line complexity
 export async function settled(ctx, timeout = 15_000) {
   const deadline = Date.now() + timeout;
+  const watchNet = STILL_MS === null && ctx.net;
   const read = async () => ({
     m: await measure(ctx),
     frames: previewFrames(ctx.page),
     pending: await swapPending(ctx),
+    ...(watchNet && { net: networkBusy(ctx) }),
   });
   let start = await read();
   let now = start;
   // Compared with the window's first read, so a drift too slow to show read to read still restarts it.
-  for (let since = Date.now(); start.pending || Date.now() - since < STILL_MS; ) {
+  for (let [since, still] = [Date.now(), 0]; !heldStill(start, still, since); ) {
     // A preview that never holds still is a Studio defect: the metrics it feeds fail, the rest still count.
     if (Date.now() > deadline) return { ...now.m, unsettled: true };
     await nextFrame(ctx.page);
     now = await read();
-    if (unsettledBy(start, now)) [start, since] = [now, Date.now()];
+    if (unsettledBy(start, now)) [start, since, still] = [now, Date.now(), 0];
+    else still++;
   }
   return now.m;
 }
 
-/** Ready once Studio's own seek tool reports the composition and the playhead landed. */
+const callTool = (page, name, args) =>
+  page.evaluate((n, a) => window.__editBench.call(n, a), name, args).catch(String);
+
+/**
+ * Seeks to the playhead once Studio reports the composition's duration, and returns the landed seek. A seek
+ * before that is replayed by the runtime's timeline bind from near 0, which skips the first keyframe of a
+ * tween starting at the playhead: the preview then holds a box no fresh seek or render shows.
+ */
 // fallow-ignore-next-line complexity
-export async function openStudio(ctx) {
-  ctx.handles = null;
-  await ctx.page.waitForFunction(() => window.__editBench?.has("studio_seek"), { timeout: 90_000 });
-  let seek = null;
-  for (const deadline = Date.now() + 30_000; Date.now() < deadline; await sleep(250)) {
-    seek = await ctx.page
-      .evaluate((time) => window.__editBench.call("studio_seek", { time }), ctx.playhead)
-      .catch(String);
+export async function seekBound(ctx) {
+  for (const deadline = Date.now() + 30_000; Date.now() < deadline; await sleep(50)) {
+    const look = await callTool(ctx.page, "studio_look", {});
+    if (!(look?.ok && look.duration > 0)) continue;
+    const seek = await callTool(ctx.page, "studio_seek", { time: ctx.playhead });
     if (
       seek?.ok &&
       seek.duration > 0 &&
       seek.playhead === ctx.playhead &&
       (await findTarget(ctx.page))
     )
-      break;
-    seek = null;
+      return seek;
   }
-  if (!seek) throw new Error("studio never reported a seekable composition");
+  return null;
+}
+
+/** Ready once Studio's own seek tool reports the composition and the playhead landed. */
+export async function openStudio(ctx) {
+  ctx.handles = null;
+  await ctx.page.waitForFunction(() => window.__editBench?.has("studio_seek"), { timeout: 90_000 });
+  if (!(await seekBound(ctx))) throw new Error("studio never reported a seekable composition");
   return settled(ctx);
 }
 
@@ -876,6 +958,7 @@ export async function inStudio({ browser, spec, dir, files, url, evidence }, dri
     handles: null,
     playhead: spec.playhead ?? PLAYHEAD,
     keys: spec.keys,
+    net: await watchNetwork(page),
   };
   const consoleErrors = [];
   page.on("pageerror", (e) => consoleErrors.push(e.message));

packages/studio/tests/e2e/edit-accuracy/render.mjs

@@ -79,28 +79,87 @@ function markNoTimeline() {
   }).observe(document, { childList: true, subtree: true });
 }
 
+const CAPTURE = {
+  width: COMPOSITION.width,
+  height: COMPOSITION.height,
+  fps: FPS,
+  format: "jpeg",
+  quality: 100,
+};
+// A composition with no GSAP never registers a timeline; the default 45 s wait for one changes no pixel.
+// Static-frame dedup opens a second page that would fetch GSAP from the CDN; one frame needs none.
+const CAPTURE_CONFIG = { playerReadyTimeout: 10_000, staticFrameDedup: false };
+
+/** A capture session with the renders' launch configuration and its page closed: only its browser lease is left. */
+async function openHold(framesDir) {
+  // requiresWebGpu as the renders find it from the fixtures, so the launch configuration matches theirs.
+  const session = await createCaptureSession(
+    "about:blank",
+    framesDir,
+    { ...CAPTURE, requiresWebGpu: false },
+    null,
+    CAPTURE_CONFIG,
+  );
+  await session.page.close();
+  // closeCaptureSession then only releases the browser lease.
+  session.pageReleased = true;
+  return session;
+}
+
+let hold = null;
+
+/**
+ * Keeps the producer's Chrome open between renders. The engine pools a browser per launch configuration and
+ * closes it when its last session closes; this session holds a lease with the same configuration and no page,
+ * so each render opens a page in that browser instead of launching one. Call the result to let it close.
+ */
+export async function holdProducerBrowser(framesDir) {
+  const current = { framesDir, session: openHold(framesDir) };
+  hold = current;
+  // A hold that fails here is taken again by the next render.
+  await current.session.catch(() => undefined);
+  return async () => {
+    if (hold === current) hold = null;
+    const session = await current.session.catch(() => null);
+    if (session) await closeCaptureSession(session);
+  };
+}
+
+/** The held browser; a hold that failed or whose Chrome died is dropped and taken again, never reused. */
+// fallow-ignore-next-line complexity
+export async function heldBrowser() {
+  if (!hold) return null;
+  const taken = hold.session;
+  const session = await taken.catch(() => null);
+  if (session?.browser.connected) return session.browser;
+  if (hold.session === taken) {
+    hold.session = openHold(hold.framesDir);
+    if (session) void closeCaptureSession(session).catch(() => undefined);
+  }
+  return (await hold.session.catch(() => null))?.browser ?? null;
+}
+
+let warnedNotHeld = false;
+function warnUnlessHeld(held, browser) {
+  if (!held || browser === held || warnedNotHeld) return;
+  warnedNotHeld = true;
+  console.warn("edit bench: the render did not reuse the held producer browser");
+}
+
 /** Renders `dir` at `time` and measures the target's box in the frame. */
 export async function renderBox(dir, decoder, time = PLAYHEAD) {
   const framesDir = join(dir, ".bench-frames");
   mkdirSync(framesDir, { recursive: true });
   const server = await createFileServer({ projectDir: dir, port: 0, fps: FPS });
   let session;
   try {
-    session = await createCaptureSession(
-      server.url,
-      framesDir,
-      {
-        width: COMPOSITION.width,
-        height: COMPOSITION.height,
-        fps: FPS,
-        format: "jpeg",
-        quality: 100,
-      },
-      null,
-      // A composition with no GSAP never registers a timeline; the default 45 s wait for one changes no pixel.
-      // Static-frame dedup opens a second page that would fetch GSAP from the CDN; one frame needs none.
-      { playerReadyTimeout: 10_000, staticFrameDedup: false },
-    );
+    const held = await heldBrowser();
+    session = await createCaptureSession(server.url, framesDir, CAPTURE, null, CAPTURE_CONFIG);
+    warnUnlessHeld(held, session.browser);
+    // A fresh page in a reused browser: nothing an earlier case's render cached may serve this one.
+    const cdp = await session.page.createCDPSession();
+    await cdp.send("Network.clearBrowserCache");
+    await cdp.detach();
     // The producer's own browser would fetch GSAP from the CDN; a stalled fetch renders the 0 s frame.
     await serveFixtureAssetsLocally(session.page);
     // Without the opt-out, a project with no timeline waits out playerReadyTimeout on every render.