diff --git a/.gitignore b/.gitignore index 64dc7b4b14..2317e928aa 100644 --- a/.gitignore +++ b/.gitignore @@ -99,6 +99,8 @@ examples/* !examples/gcp-cloud-run/** !examples/docs-reference-project !examples/docs-reference-project/** +!examples/golden-baseline +!examples/golden-baseline/** # …but never the local smoke run's build/render artifacts. examples/gcp-cloud-run/scripts/gcp-smoke-artifacts/ packages/studio/data/ diff --git a/docs/docs.json b/docs/docs.json index f2744d4f09..ea9f1f7217 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -780,7 +780,8 @@ "pages": [ "developers/cli", "packages/cli", - "packages/lint" + "packages/lint", + "guides/visual-regression" ] }, { @@ -825,7 +826,8 @@ "pages": [ "reference/html-schema", "reference/color-grading", - "reference/audio-effects" + "reference/audio-effects", + "reference/cli-golden" ] }, { diff --git a/docs/guides/visual-regression.mdx b/docs/guides/visual-regression.mdx new file mode 100644 index 0000000000..c871a44226 --- /dev/null +++ b/docs/guides/visual-regression.mdx @@ -0,0 +1,140 @@ +--- +title: "Visual regression testing" +description: "Commit golden baseline frames and let hyperframes check --golden fail the build when a composition's pixels regress." +--- + +Agents (and humans) regress layouts silently: a refactor nudges a headline, a +token change recolors a button, a timing tweak leaves a beat blank — and every +individual command still exits 0. The golden baseline gate turns "the video +still looks right" into a hard, committable contract: reference frames live in +the repo, and `hyperframes check --golden` fails whenever the rendered pixels +drift from them. + +This guide covers the workflow. For the exact directory convention, manifest +fields, and JSON shapes, see the [golden baseline reference](/reference/cli-golden). + +## The loop + +```bash +# 1. From a state you have visually reviewed, freeze the baselines. +hyperframes check --update-golden + +# 2. Commit them — they are small PNGs, one per sample time. +git add golden/ && git commit -m "test: golden baselines for intro" + +# 3. From now on, gate every change. +hyperframes check --golden +``` + +The gate re-captures the composition at the committed sample times (same +font-localized, settled-page capture path as `hyperframes snapshot`), +pixel-diffs each frame against its baseline, and: + +- **Pass** — every frame matches within tolerance; exit code 0. +- **Fail** — the failing times are listed with how much differs, a + `golden-diff/` folder appears with per-frame red-highlight diff PNGs and a + `baseline | current | diff` contact sheet, and the exit code is 1. + +```text +Golden + ✗ t=2s 0.949% pixels differ (max channel delta 130) + 1 of 3 frame(s) regressed vs golden/golden-baseline/ + Diff sheet: golden-diff/golden-baseline/contact-sheet.jpg + Intended change? Refresh baselines with hyperframes check --update-golden +``` + +An intentional change is a one-command ritual: re-run with `--update-golden`, +review, and commit the refreshed PNGs. The diff shows up in the PR as an image +diff, so reviewers see exactly what changed on screen. + +## For agent loops + +Add `--json` and read the summary instead of parsing logs: + +```bash +hyperframes check --golden --json +``` + +```json +{ "ok": false, "golden": { "failed": [{ "id": "intro", "time": 2, "maxDelta": 130 }], "diffSheet": "golden-diff/intro/contact-sheet.jpg" } } +``` + +The `diffSheet` is a single image an agent can open to see every regression at +once — baseline, current, and highlighted diff side by side — before deciding +whether to fix the code or refresh the baseline. + +## CI wiring + +The gate is one exit code, so any CI is a few lines: + +```yaml +# .github/workflows/visual-regression.yml +name: visual-regression +on: [pull_request] +jobs: + golden: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: oven-sh/setup-bun@v2 + - run: bun install + - run: npx hyperframes check --golden --no-browser-gpu + working-directory: my-composition + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: golden-diff + path: my-composition/golden-diff/ +``` + +Uploading `golden-diff/` on failure gives reviewers the contact sheet without +re-running anything locally. + +## Keeping captures deterministic + +Pixel equality is only meaningful when the two captures come from the same +rendering stack: + +- **Capture baselines where you gate.** Create and refresh baselines on the + same OS/runner class that CI uses. A macOS-made baseline gated on Linux will + differ in font rasterization. +- **Pin the GPU path.** Pass `--no-browser-gpu` in CI (and when refreshing + baselines) to force SwiftShader's deterministic software rendering. +- **Follow the composition determinism rules** — no `Date.now()`, no unseeded + randomness, no render-time network fetches. Remote fonts are localized + automatically during capture, matching the render path. + +If your environment still produces benign single-pixel jitter, tune rather than +abandon the gate: `ignoreAntialiasing` (on by default) absorbs 1px edge shifts, +`threshold` widens the per-channel tolerance, and `maxDiffRatio` grants a small +pixel budget. All three live in `golden//golden.json` — see the +[reference](/reference/cli-golden#the-goldenjson-manifest). + +## Choosing sample times + +`--update-golden` defaults to five evenly spaced frames with a readable +end-of-timeline tail. For most compositions, better times are the ones a +reviewer would screenshot: the settled end of each beat, not mid-transition. +Pin them explicitly: + +```bash +hyperframes snapshot --update-golden --at 0.5,2,3.5 +``` + +Mid-transition times work — the capture is seek-exact — but they make every +retiming of the animation an (intentional) baseline refresh. + +## Try it + +A runnable example with committed baselines lives at +[`examples/golden-baseline`](https://github.com/heygen-com/hyperframes/tree/main/examples/golden-baseline): +break the accent color, watch the gate fail with a diff sheet, refresh, commit. + +## Related topics + +- [Golden baseline reference](/reference/cli-golden) — the exact directory + convention, every flag and manifest field, and the JSON summary shape. +- [Deterministic rendering](/concepts/determinism) — the composition rules that + make pixel equality meaningful in the first place. +- [Render from the command line](/guides/rendering) — the wider CLI loop the + gate slots into. diff --git a/docs/reference/cli-golden.mdx b/docs/reference/cli-golden.mdx new file mode 100644 index 0000000000..94f7fe600a --- /dev/null +++ b/docs/reference/cli-golden.mdx @@ -0,0 +1,126 @@ +--- +title: "Golden baseline gate" +sidebarTitle: "Golden baselines" +description: "The golden/ directory convention, the check --golden and --update-golden flags, the golden.json manifest, the diff artifacts, and the JSON summary agents read." +--- + +Use this reference for the exact contract of the golden baseline regression +gate: where baselines live, every flag and manifest field, what the gate writes +on failure, and the JSON shape agents consume. For the workflow — when to +commit baselines and how to wire the gate into CI — start with +[Visual regression testing](/guides/visual-regression). + +## The convention + +Committed reference frames live beside the composition: + +``` +/ + index.html + golden/ + / # from the root's data-composition-id + golden.json # optional manifest + .png # one baseline per sample time (e.g. 1500.png = t=1.5s) +``` + +Filenames are the sample time in **milliseconds** (`Math.round(seconds * 1000)`). +Anything that does not match `.png` is ignored by the scanner, so the +manifest and stray notes never collide with baselines. + +## Commands + +```bash +hyperframes check --golden # gate: re-capture, pixel-diff, fail on regression +hyperframes check --update-golden # refresh baselines (runs the full check too) +hyperframes snapshot --update-golden # refresh baselines only, no check pipeline +hyperframes check --golden --json # agent-readable envelope +``` + +| Flag | Command | Meaning | +| --------------------------- | ----------------- | -------------------------------------------------------------------------------------------------- | +| `--golden` | `check` | Run the gate after the normal check pipeline; either failing makes the exit code non-zero. | +| `--update-golden` | `check`, `snapshot` | Capture at the resolved sample times and (re)write `golden//` plus its manifest. | +| `--golden-threshold <0-1>` | `check` | Per-channel pixel tolerance override for this run (does not enable the gate by itself). | +| `--at ` | `snapshot` | With `--update-golden`: explicit sample times in seconds, written into the manifest. | + +Sample times resolve in precedence order: explicit `--at` (snapshot only) → +manifest `times` → times encoded in existing baseline filenames → the snapshot +default spread (5 evenly spaced frames with a readable end-of-timeline tail). + +`check --golden` with no baselines and no manifest fails with instructions to +run `--update-golden` first — a missing baseline is never silently a pass. + +## The `golden.json` manifest + +```json +{ + "times": [0.5, 2, 3.5], + "threshold": 0.1, + "maxDiffRatio": 0, + "ignoreAntialiasing": true +} +``` + +| Field | Default | Meaning | +| -------------------- | ------- | -------------------------------------------------------------------------------------------------------------- | +| `times` | — | Timeline sample times in seconds. Written automatically by `--update-golden`. | +| `threshold` | `0.1` | Per-channel intensity tolerance as a fraction of 255; a pixel differs only when a channel deviates more. | +| `maxDiffRatio` | `0` | Fraction of differing pixels allowed before the gate fails. `0` = fail on any counted diff. | +| `ignoreAntialiasing` | `true` | Exclude 1px rasterization edge shifts (each side finds the other's color within its own 8-neighborhood). | + +A dimension change (resized canvas) always fails, regardless of `maxDiffRatio`. + +## Failure artifacts + +On regression the gate writes `golden-diff//` in the project: + +``` +golden-diff// + -current.png # what the composition renders now + -diff.png # red = counted diff, amber = ignored anti-aliasing, pale gray = unchanged + contact-sheet.jpg # one baseline | current | diff row per failed time +``` + +The directory is cleared at the start of every gate run and only re-created on +failure. Add `golden-diff/` to your `.gitignore` — it is derived output. + +## JSON summary + +With `check --golden --json`, the golden summary rides inside the normal check +envelope, and the top-level `ok` is false when either the pipeline or the gate +fails: + +```json +{ + "ok": false, + "golden": { + "ok": false, + "updated": false, + "compositionId": "golden-baseline", + "compared": 3, + "times": [0.5, 2, 3.5], + "failed": [ + { + "id": "golden-baseline", + "time": 2, + "timeMs": 2000, + "maxDelta": 130, + "diffRatio": 0.009488, + "reason": "pixel-diff" + } + ], + "diffSheet": "golden-diff/golden-baseline/contact-sheet.jpg", + "baselines": ["golden/golden-baseline/500.png", "golden/golden-baseline/2000.png"] + } +} +``` + +`reason` is one of `pixel-diff`, `dimension-mismatch`, or `missing-baseline`. +`maxDelta` is the largest per-channel deviation (0–255) among counted differing +pixels; `diffRatio` is the differing fraction of all pixels. + +## Example + +A complete runnable project lives at +[`examples/golden-baseline`](https://github.com/heygen-com/hyperframes/tree/main/examples/golden-baseline) +with committed baselines and a scripted regression to try. diff --git a/examples/golden-baseline/.gitignore b/examples/golden-baseline/.gitignore new file mode 100644 index 0000000000..b57db5e0f4 --- /dev/null +++ b/examples/golden-baseline/.gitignore @@ -0,0 +1,4 @@ +# Derived verification artifacts — regenerated by `hyperframes check --golden` +# and `hyperframes snapshot`; only golden/ baselines are meant to be committed. +golden-diff/ +snapshots/ diff --git a/examples/golden-baseline/README.md b/examples/golden-baseline/README.md new file mode 100644 index 0000000000..94391ad31a --- /dev/null +++ b/examples/golden-baseline/README.md @@ -0,0 +1,92 @@ +# Golden baseline regression gate + +A minimal project showing the `hyperframes check --golden` workflow: commit +reference frames ("golden baselines") of a composition, then let CI — or an +AI agent's own verification loop — fail loudly whenever a change moves pixels +it was not supposed to move. + +## Layout + +``` +examples/golden-baseline/ + index.html # 640x360, 4s composition + golden/ + golden-baseline/ # + golden.json # sample times + diff tuning (optional) + 500.png # baseline at t=0.5s (.png) + 2000.png # baseline at t=2.0s + 3500.png # baseline at t=3.5s +``` + +Baselines follow the convention `golden//.png`. The +`golden.json` manifest pins the sample times and can tune the diff: + +```json +{ + "times": [0.5, 2, 3.5], + "threshold": 0.1, + "maxDiffRatio": 0, + "ignoreAntialiasing": true +} +``` + +- `times` — timeline sample times in seconds. +- `threshold` — per-channel pixel tolerance as a fraction of 255 (default 0.1). +- `maxDiffRatio` — fraction of differing pixels allowed before the gate fails + (default 0: fail on any counted diff). +- `ignoreAntialiasing` — exclude 1px rasterization edge shifts from the count + (default true). + +## Workflow + +```bash +cd examples/golden-baseline + +# 1. Create (or refresh) the baselines from a state you have reviewed, then +# commit golden/ to the repo. +npx hyperframes check --update-golden # or: npx hyperframes snapshot --update-golden + +# 2. Gate every subsequent change. Exits non-zero on any regression. +npx hyperframes check --golden + +# 3. Agent-readable result (the golden summary rides inside the check report). +npx hyperframes check --golden --json +``` + +On failure the gate writes `golden-diff//` containing the +current frame, a red-highlight diff PNG per failed time, and a +`contact-sheet.jpg` with one `baseline | current | diff` row per failure — +one image an agent (or a human) can read to see exactly what moved. + +The JSON envelope includes: + +```json +{ + "ok": false, + "golden": { + "ok": false, + "failed": [ + { + "id": "golden-baseline", + "time": 2, + "maxDelta": 210, + "diffRatio": 0.0042, + "reason": "pixel-diff" + } + ], + "diffSheet": "golden-diff/golden-baseline/contact-sheet.jpg" + } +} +``` + +## Try a regression + +Change `#accent-bar`'s `background-color` in `index.html` (say `#f5a623` → +`#e0245e`) and run `npx hyperframes check --golden` — the gate fails, names the +regressed times, and writes the diff sheet. If the change was intentional, +refresh with `--update-golden` and commit the new baselines. + +Deterministic rendering matters here: capture baselines and gate on the same +rendering stack (CI runner, `--no-browser-gpu` for SwiftShader determinism if +your machines differ). See the visual regression guide in the docs for CI +wiring. diff --git a/examples/golden-baseline/golden/golden-baseline/2000.png b/examples/golden-baseline/golden/golden-baseline/2000.png new file mode 100644 index 0000000000..522b2768fa Binary files /dev/null and b/examples/golden-baseline/golden/golden-baseline/2000.png differ diff --git a/examples/golden-baseline/golden/golden-baseline/3500.png b/examples/golden-baseline/golden/golden-baseline/3500.png new file mode 100644 index 0000000000..143f0ba3cf Binary files /dev/null and b/examples/golden-baseline/golden/golden-baseline/3500.png differ diff --git a/examples/golden-baseline/golden/golden-baseline/500.png b/examples/golden-baseline/golden/golden-baseline/500.png new file mode 100644 index 0000000000..1938e854c6 Binary files /dev/null and b/examples/golden-baseline/golden/golden-baseline/500.png differ diff --git a/examples/golden-baseline/golden/golden-baseline/golden.json b/examples/golden-baseline/golden/golden-baseline/golden.json new file mode 100644 index 0000000000..a294c2109e --- /dev/null +++ b/examples/golden-baseline/golden/golden-baseline/golden.json @@ -0,0 +1,3 @@ +{ + "times": [0.5, 2, 3.5] +} diff --git a/examples/golden-baseline/index.html b/examples/golden-baseline/index.html new file mode 100644 index 0000000000..9e25fce50a --- /dev/null +++ b/examples/golden-baseline/index.html @@ -0,0 +1,97 @@ + + + + + + Golden Baseline Demo + + + + +
+
+
+
GOLDEN GATE
+
pixel-diffed against committed baselines
+
+
+
+ + + + diff --git a/packages/cli/src/capture/contactSheet.ts b/packages/cli/src/capture/contactSheet.ts index ffb5b37a22..8f154e2020 100644 --- a/packages/cli/src/capture/contactSheet.ts +++ b/packages/cli/src/capture/contactSheet.ts @@ -236,6 +236,42 @@ export async function createSnapshotContactSheet( ); } +export interface GoldenDiffSheetRow { + /** Row label, e.g. "t=1.5s". */ + label: string; + baselinePath: string; + currentPath: string; + diffPath: string; +} + +/** + * Contact sheet for golden baseline failures: one row per failed sample time, + * cells ordered baseline | current | diff. Paginated at 3 rows per page so + * cells stay readable. Returns the written file paths. + */ +export async function createGoldenDiffContactSheet( + rows: GoldenDiffSheetRow[], + outputPath: string, + budget: Pick = {}, +): Promise { + if (rows.length === 0) return []; + + const paths = rows.flatMap((row) => [row.baselinePath, row.currentPath, row.diffPath]); + const labels = rows.flatMap((row) => [ + `${row.label} baseline`, + `${row.label} current`, + `${row.label} diff`, + ]); + + return createContactSheetPages( + paths, + outputPath, + { cols: 3, cellWidth: 600, pageSize: 9, ...budget }, + 0, + labels, + ); +} + /** * Contact sheet for captured assets. Paginated — all assets covered. * Labels: "1. filename" diff --git a/packages/cli/src/commands/check.test.ts b/packages/cli/src/commands/check.test.ts index 73f548ef88..37988a10f3 100644 --- a/packages/cli/src/commands/check.test.ts +++ b/packages/cli/src/commands/check.test.ts @@ -27,6 +27,7 @@ import { type CheckFindingCropRequest, type CheckOptions, type CheckReport, + type CheckSection, type ContrastAuditEntry, type MotionSpecResolution, } from "../utils/checkPipeline.js"; @@ -1798,3 +1799,146 @@ describe("dense motion-overlap re-sampling", () => { expect(report.layout.findings.some((f) => f.code === "content_overlap")).toBe(true); }); }); + +describe("golden gate flags", () => { + it("parses --golden-threshold and rejects out-of-range values", async () => { + const { parseGoldenThreshold } = await import("./check.js"); + expect(parseGoldenThreshold(undefined)).toBeUndefined(); + expect(parseGoldenThreshold("0")).toBe(0); + expect(parseGoldenThreshold("0.25")).toBe(0.25); + expect(parseGoldenThreshold("1")).toBe(1); + expect(() => parseGoldenThreshold("1.5")).toThrow("Invalid --golden-threshold"); + expect(() => parseGoldenThreshold("-0.1")).toThrow("Invalid --golden-threshold"); + expect(() => parseGoldenThreshold("0.1px")).toThrow("Invalid --golden-threshold"); + expect(() => parseGoldenThreshold("")).toThrow("Invalid --golden-threshold"); + }); + + it("only enables the gate for --golden or --update-golden", async () => { + const { parseGoldenGateArgs } = await import("./check.js"); + const base = { timeout: 3000, autoProxy: undefined, browserGpuMode: undefined }; + expect(parseGoldenGateArgs({}, base)).toBeUndefined(); + expect(parseGoldenGateArgs({ "golden-threshold": "0.2" }, base)).toBeUndefined(); + expect(parseGoldenGateArgs({ golden: true }, base)).toEqual( + expect.objectContaining({ update: false, timeoutMs: 3000 }), + ); + expect(parseGoldenGateArgs({ "update-golden": true, "golden-threshold": "0.2" }, base)).toEqual( + expect.objectContaining({ update: true, threshold: 0.2 }), + ); + }); +}); + +describe("golden gate wiring", () => { + function passingReport(): CheckReport { + const section: CheckSection = { + ok: true, + errorCount: 0, + warningCount: 0, + infoCount: 0, + findings: [], + }; + return { + ok: true, + strict: false, + lint: { ...section, filesScanned: 1 }, + runtime: { ...section }, + layout: { + ...section, + findings: [], + duration: 1, + samples: [0.5], + transitionSamples: [], + transitionSamplesDropped: 0, + tolerance: 2, + totalIssueCount: 0, + truncated: false, + }, + motion: { ...section, enabled: false, samples: 0 }, + contrast: { ...section, findings: [], enabled: true, samples: [0.5], checked: 1, passed: 1 }, + snapshots: { enabled: false, files: [], times: [], findingFiles: [] }, + }; + } + + it("fails an otherwise passing check when the golden gate regresses", async () => { + const logs: string[] = []; + vi.spyOn(console, "log").mockImplementation((line?: unknown) => { + logs.push(String(line)); + }); + const runGolden = vi.fn(async () => ({ + ok: false, + updated: false, + compositionId: "intro", + compared: 2, + times: [0, 1.5], + failed: [ + { + id: "intro", + time: 1.5, + timeMs: 1500, + maxDelta: 210, + diffRatio: 0.0042, + reason: "pixel-diff" as const, + }, + ], + diffSheet: "golden-diff/intro/contact-sheet.jpg", + baselines: ["golden/intro/0.png", "golden/intro/1500.png"], + })); + const command = createCheckCommand({ + resolveProject: () => PROJECT, + runPipeline: async () => passingReport(), + withMeta: (value) => value, + runGolden, + }); + + await runCommand(command, { rawArgs: ["--json", "--golden"] }); + + expect(runGolden).toHaveBeenCalledWith(PROJECT, expect.objectContaining({ update: false })); + const payload = JSON.parse(logs.join("\n")); + expect(payload.ok).toBe(false); + expect(payload.golden.failed).toEqual([ + expect.objectContaining({ id: "intro", time: 1.5, maxDelta: 210 }), + ]); + expect(payload.golden.diffSheet).toBe("golden-diff/intro/contact-sheet.jpg"); + expect(consumeCommandResult().exitCode).toBe(1); + }); + + it("does not run the golden gate when not requested", async () => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + const runGolden = vi.fn(); + const command = createCheckCommand({ + resolveProject: () => PROJECT, + runPipeline: async () => passingReport(), + withMeta: (value) => value, + runGolden, + }); + + await runCommand(command, { rawArgs: ["--json"] }); + + expect(runGolden).not.toHaveBeenCalled(); + expect(consumeCommandResult().exitCode).toBe(0); + }); + + it("keeps a passing exit code when --update-golden refreshes baselines", async () => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + const runGolden = vi.fn(async () => ({ + ok: true, + updated: true, + compositionId: "intro", + compared: 2, + times: [0, 1.5], + failed: [], + diffSheet: null, + baselines: ["golden/intro/0.png", "golden/intro/1500.png"], + })); + const command = createCheckCommand({ + resolveProject: () => PROJECT, + runPipeline: async () => passingReport(), + withMeta: (value) => value, + runGolden, + }); + + await runCommand(command, { rawArgs: ["--json", "--update-golden"] }); + + expect(runGolden).toHaveBeenCalledWith(PROJECT, expect.objectContaining({ update: true })); + expect(consumeCommandResult().exitCode).toBe(0); + }); +}); diff --git a/packages/cli/src/commands/check.ts b/packages/cli/src/commands/check.ts index f90b494398..1f045c4267 100644 --- a/packages/cli/src/commands/check.ts +++ b/packages/cli/src/commands/check.ts @@ -19,10 +19,13 @@ import { } from "../utils/checkPipeline.js"; import type { CaptionZoneOptions, FrameCheckOptions, LayoutOptions } from "../utils/checkTypes.js"; import { resolveLocalBrowserGpuMode } from "../browser/gpuPolicy.js"; +import type { GoldenGateOptions, GoldenSummary } from "../golden/baseline.js"; export const examples: Example[] = [ ["Run the full verification gate", "hyperframes check"], ["Output one agent-readable envelope", "hyperframes check --json"], + ["Gate against committed golden baselines", "hyperframes check --golden"], + ["Refresh the golden baselines", "hyperframes check --update-golden"], ["Persist the five audited contrast frames", "hyperframes check --snapshots"], ["Also fail on warnings", "hyperframes check --strict"], ]; @@ -31,6 +34,8 @@ export interface CheckCommandDependencies { resolveProject(dir: string | undefined): ProjectDir; runPipeline(project: ProjectDir, options: CheckOptions): Promise; withMeta(value: object): object; + /** Golden baseline gate; defaults to a lazy import so sharp only loads when requested. */ + runGolden?(project: ProjectDir, options: GoldenGateOptions): Promise; } const DEFAULT_COMMAND_DEPENDENCIES: CheckCommandDependencies = { @@ -42,6 +47,9 @@ const DEFAULT_COMMAND_DEPENDENCIES: CheckCommandDependencies = { const CHECK_COMMAND_ARGS = { dir: { type: "positional", description: "Project directory", required: false }, json: { type: "boolean", description: "Output agent-readable JSON", default: false }, + // The sampling args below intentionally mirror the deprecated `layout` + // command's grammar (check superseded it) — an inherited clone, not new code. + // fallow-ignore-next-line code-duplication samples: { type: "string", description: "Number of midpoint samples across the duration (default: 9)", @@ -110,6 +118,22 @@ const CHECK_COMMAND_ARGS = { description: "Save the five contrast-pass PNGs under snapshots/", default: false, }, + golden: { + type: "boolean", + description: + "Also gate against committed golden baselines (golden//.png): re-capture at the manifest times, pixel-diff, and fail on regressions", + default: false, + }, + "update-golden": { + type: "boolean", + description: "Refresh the golden baselines from the current render instead of gating", + default: false, + }, + "golden-threshold": { + type: "string", + description: + "Per-channel pixel tolerance for the golden gate, 0-1 (default: golden.json threshold, else 0.1)", + }, "caption-zone": { type: "string", description: @@ -141,18 +165,7 @@ export function createCheckCommand( const asJson = args.json === true; try { - const project = dependencies.resolveProject(args.dir); - const options = parseCheckOptions(args); - if (!asJson) { - console.log(`${c.accent("◆")} Checking ${c.accent(project.name)}`); - } - const report = await dependencies.runPipeline(project, options); - if (asJson) { - console.log(JSON.stringify(dependencies.withMeta(report), null, 2)); - } else { - printHumanReport(report); - } - setCommandExitCode(checkExitCode(report)); + setCommandExitCode(await executeCheck(dependencies, args, asJson)); } catch (error) { const message = normalizeErrorMessage(error); if (asJson) { @@ -168,6 +181,49 @@ export function createCheckCommand( }); } +async function resolveGoldenSummary( + dependencies: CheckCommandDependencies, + project: ProjectDir, + goldenOptions: GoldenGateOptions | undefined, +): Promise { + if (!goldenOptions) return undefined; + const runGolden = dependencies.runGolden ?? (await import("../golden/baseline.js")).runGoldenGate; + return runGolden(project, goldenOptions); +} + +function emitCheckReport( + dependencies: CheckCommandDependencies, + report: CheckReport, + golden: GoldenSummary | undefined, + ok: boolean, + asJson: boolean, +): void { + if (!asJson) { + printHumanReport(report, golden); + return; + } + const payload = golden ? { ...report, ok, golden } : report; + console.log(JSON.stringify(dependencies.withMeta(payload), null, 2)); +} + +async function executeCheck( + dependencies: CheckCommandDependencies, + args: Record, + asJson: boolean, +): Promise<0 | 1> { + const project = dependencies.resolveProject(typeof args.dir === "string" ? args.dir : undefined); + const options = parseCheckOptions(args); + const goldenOptions = parseGoldenGateArgs(args, options); + if (!asJson) { + console.log(`${c.accent("◆")} Checking ${c.accent(project.name)}`); + } + const report = await dependencies.runPipeline(project, options); + const golden = await resolveGoldenSummary(dependencies, project, goldenOptions); + const ok = checkExitCode(report) === 0 && (golden?.ok ?? true); + emitCheckReport(dependencies, report, golden, ok, asJson); + return ok ? 0 : 1; +} + function normalizeFrameCheckRawArgs(rawArgs: string[]): string[] { return rawArgs.map((arg, index) => { if (arg !== "--frame-check") return arg; @@ -198,6 +254,36 @@ function parseCheckOptions(args: Record): CheckOptions { }; } +/** + * `--golden` / `--update-golden` / `--golden-threshold` → gate options, or + * undefined when the golden gate was not requested. The threshold flag alone + * does not enable the gate — it only tunes an explicitly requested run. + */ +export function parseGoldenGateArgs( + args: Record, + options: Pick, +): GoldenGateOptions | undefined { + const update = args["update-golden"] === true; + if (args.golden !== true && !update) return undefined; + return { + update, + threshold: parseGoldenThreshold(args["golden-threshold"]), + timeoutMs: options.timeout, + autoProxy: options.autoProxy, + browserGpuMode: options.browserGpuMode, + }; +} + +export function parseGoldenThreshold(value: unknown): number | undefined { + if (value === undefined || value === null) return undefined; + const raw = typeof value === "string" ? value.trim() : ""; + const parsed = parseNumberStrict(raw); + if (parsed === null || parsed < 0 || parsed > 1) { + throw new Error("Invalid --golden-threshold: expected a number from 0 to 1"); + } + return parsed; +} + const CAPTION_ZONE_FIELDS = new Set(["x0", "y0", "x1", "y1", "severity", "seek"]); const FRAME_CHECK_FIELDS = new Set(["severity", "seek", "tol"]); @@ -403,16 +489,52 @@ function nonNegativeNumber(value: unknown, fallback: number): number { return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; } -function printHumanReport(report: CheckReport): void { +function printHumanReport(report: CheckReport, golden?: GoldenSummary): void { printSection("Lint", report.lint); printSection("Runtime", report.runtime); printLayoutSection("Layout", report.layout); printSection("Motion", report.motion); printContrastSection(report); printSnapshotSection(report); + if (golden) printGoldenSection(golden); console.log(); - const label = report.ok ? c.success("Check passed") : c.error("Check failed"); - console.log(`${report.ok ? c.success("◇") : c.error("◇")} ${label}`); + const ok = report.ok && (golden?.ok ?? true); + const label = ok ? c.success("Check passed") : c.error("Check failed"); + console.log(`${ok ? c.success("◇") : c.error("◇")} ${label}`); +} + +function printGoldenSection(golden: GoldenSummary): void { + console.log(); + console.log(c.bold("Golden")); + if (golden.updated) { + console.log( + ` ${c.success("◇")} ${golden.baselines.length} baseline(s) written for ${golden.compositionId}`, + ); + for (const baseline of golden.baselines) console.log(` ${c.dim(baseline)}`); + return; + } + if (golden.ok) { + console.log( + ` ${c.success("◇")} ${golden.compared}/${golden.compared} frame(s) match the committed baselines`, + ); + return; + } + for (const failure of golden.failed) { + const detail = + failure.reason === "pixel-diff" + ? `${(failure.diffRatio * 100).toFixed(3)}% pixels differ (max channel delta ${failure.maxDelta})` + : failure.reason; + console.log(` ${c.error("✗")} t=${failure.time}s ${detail}`); + } + console.log( + ` ${c.dim(`${golden.failed.length} of ${golden.compared} frame(s) regressed vs golden/${golden.compositionId}/`)}`, + ); + if (golden.diffSheet) { + console.log(` ${c.dim(`Diff sheet: ${golden.diffSheet}`)}`); + } + console.log( + ` ${c.dim("Intended change? Refresh baselines with hyperframes check --update-golden")}`, + ); } function printSection(title: string, section: CheckSection): void { diff --git a/packages/cli/src/commands/snapshot.ts b/packages/cli/src/commands/snapshot.ts index 767f446da0..f6f315824d 100644 --- a/packages/cli/src/commands/snapshot.ts +++ b/packages/cli/src/commands/snapshot.ts @@ -186,6 +186,7 @@ export const examples: Example[] = [ "Pair each frame with the reference footage at the same time", "snapshot --at 1.5,4.3,8.1 --against ref.mp4", ], + ["Refresh the committed golden baselines", "snapshot --update-golden"], ]; /** `--zoom-scale`: the deviceScaleFactor used for zoomed crops. Defaults to 3; @@ -704,6 +705,12 @@ export default defineCommand({ "Use hardware browser GPU capture; pass --no-browser-gpu for deterministic SwiftShader (default: auto-detect, PRODUCER_BROWSER_GPU_MODE overrides)", default: undefined, }, + "update-golden": { + type: "boolean", + description: + "Refresh the committed golden baselines (golden//.png) instead of capturing review snapshots; --at overrides the sample times", + default: false, + }, }, async run({ args }) { const project = resolveProject(args.dir); @@ -753,6 +760,32 @@ export default defineCommand({ ? null : String(args.describe); + if (args["update-golden"] === true) { + try { + const { runGoldenGate } = await import("../golden/baseline.js"); + const summary = await runGoldenGate(project, { + update: true, + at: atTimestamps, + timeoutMs: timeout, + autoProxy: args.proxy as boolean | undefined, + browserGpuMode: resolveLocalBrowserGpuMode(args["browser-gpu"] as boolean | undefined), + }); + console.log( + `${c.success("◇")} ${summary.baselines.length} golden baseline(s) written for ${c.accent(summary.compositionId)}`, + ); + for (const baseline of summary.baselines) console.log(` ${baseline}`); + console.log( + ` ${c.dim(`Commit golden/${summary.compositionId}/ and gate with hyperframes check --golden`)}`, + ); + } catch (err) { + console.error( + `\n${c.error("✗")} Golden baseline update failed: ${normalizeErrorMessage(err)}`, + ); + failCommand(); + } + return; + } + const camera = args.angle ? parseAngle(String(args.angle)) : undefined; const zoomTarget = args.zoom ? parseZoomTarget(String(args.zoom)) : undefined; const zoomScale = parseZoomScale(args["zoom-scale"]); diff --git a/packages/cli/src/golden/baseline.test.ts b/packages/cli/src/golden/baseline.test.ts new file mode 100644 index 0000000000..ce03afe757 --- /dev/null +++ b/packages/cli/src/golden/baseline.test.ts @@ -0,0 +1,172 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + evaluateGoldenDiff, + goldenBaselinePath, + goldenTimeFileName, + listGoldenBaselineTimesMs, + parseGoldenManifest, + parseGoldenTimeFileName, + readGoldenManifest, + resolveGoldenConfig, + resolveGoldenSampleTimes, + timeMsFromSeconds, +} from "./baseline.js"; +import type { PixelDiffResult } from "./pixelDiff.js"; + +function diffResult(overrides: Partial): PixelDiffResult { + return { + width: 4, + height: 4, + totalPixels: 16, + diffPixels: 0, + aaPixels: 0, + diffRatio: 0, + maxDelta: 0, + dimensionMismatch: false, + diff: new Uint8Array(64), + ...overrides, + }; +} + +describe("golden path conventions", () => { + it("encodes sample times as millisecond PNG filenames", () => { + expect(timeMsFromSeconds(1.5)).toBe(1500); + expect(timeMsFromSeconds(0)).toBe(0); + expect(timeMsFromSeconds(2.0004)).toBe(2000); + expect(goldenTimeFileName(1500)).toBe("1500.png"); + expect(goldenBaselinePath("/proj", "intro", 1500)).toBe( + join("/proj", "golden", "intro", "1500.png"), + ); + }); + + it("parses only bare .png names back into times", () => { + expect(parseGoldenTimeFileName("1500.png")).toBe(1500); + expect(parseGoldenTimeFileName("0.png")).toBe(0); + expect(parseGoldenTimeFileName("golden.json")).toBeNull(); + expect(parseGoldenTimeFileName("1500-diff.png")).toBeNull(); + expect(parseGoldenTimeFileName("1.5.png")).toBeNull(); + expect(parseGoldenTimeFileName("1500.PNG")).toBeNull(); + }); +}); + +describe("parseGoldenManifest", () => { + it("accepts a fully populated manifest", () => { + const manifest = parseGoldenManifest( + JSON.stringify({ + times: [0, 1.5, 3], + threshold: 0.05, + maxDiffRatio: 0.001, + ignoreAntialiasing: false, + }), + "golden.json", + ); + expect(manifest).toEqual({ + times: [0, 1.5, 3], + threshold: 0.05, + maxDiffRatio: 0.001, + ignoreAntialiasing: false, + }); + }); + + it("accepts an empty object and ignores unknown fields", () => { + expect(parseGoldenManifest("{}", "golden.json")).toEqual({}); + expect(parseGoldenManifest('{"note":"hi"}', "golden.json")).toEqual({}); + }); + + it("rejects malformed input with the offending field named", () => { + expect(() => parseGoldenManifest("not json", "golden.json")).toThrow(/not valid JSON/); + expect(() => parseGoldenManifest("[1,2]", "golden.json")).toThrow(/JSON object/); + expect(() => parseGoldenManifest('{"times":[]}', "golden.json")).toThrow(/"times"/); + expect(() => parseGoldenManifest('{"times":[-1]}', "golden.json")).toThrow(/"times"/); + expect(() => parseGoldenManifest('{"times":["a"]}', "golden.json")).toThrow(/"times"/); + expect(() => parseGoldenManifest('{"threshold":2}', "golden.json")).toThrow(/"threshold"/); + expect(() => parseGoldenManifest('{"maxDiffRatio":-0.1}', "golden.json")).toThrow( + /"maxDiffRatio"/, + ); + expect(() => parseGoldenManifest('{"ignoreAntialiasing":"yes"}', "golden.json")).toThrow( + /"ignoreAntialiasing"/, + ); + }); +}); + +describe("golden directory scanning", () => { + it("lists baseline times from filenames and reads the manifest beside them", () => { + const dir = mkdtempSync(join(tmpdir(), "hf-golden-test-")); + try { + const compDir = join(dir, "golden", "intro"); + mkdirSync(compDir, { recursive: true }); + writeFileSync(join(compDir, "1500.png"), "png"); + writeFileSync(join(compDir, "0.png"), "png"); + writeFileSync(join(compDir, "notes.txt"), "ignore me"); + writeFileSync(join(compDir, "golden.json"), JSON.stringify({ times: [0, 1.5] })); + + expect(listGoldenBaselineTimesMs(dir, "intro")).toEqual([0, 1500]); + expect(readGoldenManifest(dir, "intro")).toEqual({ times: [0, 1.5] }); + expect(listGoldenBaselineTimesMs(dir, "missing")).toEqual([]); + expect(readGoldenManifest(dir, "missing")).toBeNull(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +describe("resolveGoldenSampleTimes", () => { + it("prefers the explicit override, then the manifest, then baseline filenames", () => { + const input = { + atOverride: [1, 2], + manifestTimes: [3, 4], + baselineTimesMs: [5000, 6000], + duration: 10, + }; + expect(resolveGoldenSampleTimes(input)).toEqual([1, 2]); + expect(resolveGoldenSampleTimes({ ...input, atOverride: undefined })).toEqual([3, 4]); + expect( + resolveGoldenSampleTimes({ ...input, atOverride: undefined, manifestTimes: undefined }), + ).toEqual([5, 6]); + }); + + it("falls back to the snapshot command's default spread with a readable tail", () => { + const times = resolveGoldenSampleTimes({ duration: 10 }); + expect(times).toHaveLength(5); + expect(times[0]).toBe(0); + // The final sample backs off the exact duration so it is not a blank frame. + expect(times[4]).toBeLessThan(10); + expect(times[4]).toBeGreaterThan(9); + }); + + it("throws when no duration and no times are available", () => { + expect(() => resolveGoldenSampleTimes({ duration: 0 })).toThrow(/duration/); + }); +}); + +describe("resolveGoldenConfig", () => { + it("applies defaults, manifest values, and CLI overrides in that order", () => { + expect(resolveGoldenConfig(null)).toEqual({ + threshold: 0.1, + maxDiffRatio: 0, + ignoreAntialiasing: true, + }); + expect( + resolveGoldenConfig({ threshold: 0.02, maxDiffRatio: 0.005, ignoreAntialiasing: false }), + ).toEqual({ threshold: 0.02, maxDiffRatio: 0.005, ignoreAntialiasing: false }); + expect(resolveGoldenConfig({ threshold: 0.02 }, { threshold: 0.3 }).threshold).toBe(0.3); + }); +}); + +describe("evaluateGoldenDiff", () => { + it("fails on dimension changes regardless of ratio budget", () => { + expect(evaluateGoldenDiff(diffResult({ dimensionMismatch: true }), 1)).toBe( + "dimension-mismatch", + ); + }); + + it("passes at or under the allowed diff ratio and fails above it", () => { + expect(evaluateGoldenDiff(diffResult({ diffRatio: 0 }), 0)).toBe("pass"); + expect(evaluateGoldenDiff(diffResult({ diffRatio: 0.001 }), 0.001)).toBe("pass"); + expect(evaluateGoldenDiff(diffResult({ diffRatio: 0.0011 }), 0.001)).toBe("pixel-diff"); + expect(evaluateGoldenDiff(diffResult({ diffRatio: 0.0001 }), 0)).toBe("pixel-diff"); + }); +}); diff --git a/packages/cli/src/golden/baseline.ts b/packages/cli/src/golden/baseline.ts new file mode 100644 index 0000000000..e779066220 --- /dev/null +++ b/packages/cli/src/golden/baseline.ts @@ -0,0 +1,496 @@ +/** + * Golden baseline regression gate. + * + * Convention: committed reference frames live at + * `golden//.png`, with an optional + * `golden//golden.json` manifest describing the sample times + * and diff tuning. `runGoldenGate` re-captures the composition at those + * times and pixel-diffs against the baselines (fail-on-diff), or refreshes + * the baselines when `update` is set. + */ + +import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { isAbsolute, join, relative } from "node:path"; +import type { BrowserGpuMode } from "../browser/gpuPolicy.js"; +import { + openSettledCompositionPage, + seekCompositionTimeline, +} from "../capture/captureCompositionFrame.js"; +import type { ProjectDir } from "../utils/project.js"; +import { serveStaticProjectHtml } from "../utils/staticProjectServer.js"; +import { + DEFAULT_DIFF_THRESHOLD, + diffPngs, + writeRawImagePng, + type PixelDiffResult, +} from "./pixelDiff.js"; + +const GOLDEN_DIR_NAME = "golden"; +const GOLDEN_DIFF_DIR_NAME = "golden-diff"; +const GOLDEN_MANIFEST_NAME = "golden.json"; + +/** Frames captured when neither a manifest nor existing baselines pick the times. */ +const DEFAULT_GOLDEN_FRAMES = 5; + +/** Optional `golden//golden.json` tuning file. */ +export interface GoldenManifest { + /** Timeline sample times in seconds. */ + times?: number[]; + /** Per-channel pixel tolerance, fraction of 255 (default 0.1). */ + threshold?: number; + /** Fraction of differing pixels allowed before the gate fails (default 0: fail on any diff). */ + maxDiffRatio?: number; + /** Exclude 1px anti-aliasing edge shifts from the failure count (default true). */ + ignoreAntialiasing?: boolean; +} + +export interface ResolvedGoldenConfig { + threshold: number; + maxDiffRatio: number; + ignoreAntialiasing: boolean; +} + +export type GoldenFailureReason = "pixel-diff" | "dimension-mismatch" | "missing-baseline"; + +export interface GoldenFailure { + /** Composition id the baseline belongs to. */ + id: string; + /** Sample time in seconds. */ + time: number; + timeMs: number; + /** Max per-channel delta (0-255) among counted differing pixels. */ + maxDelta: number; + /** Fraction of pixels that differ beyond the threshold. */ + diffRatio: number; + reason: GoldenFailureReason; +} + +/** Agent-readable gate result (also embedded in `check --json` output). */ +export interface GoldenSummary { + ok: boolean; + updated: boolean; + compositionId: string; + compared: number; + times: number[]; + failed: GoldenFailure[]; + /** Contact sheet of baseline | current | diff rows, written on failure. */ + diffSheet: string | null; + /** Baseline PNGs written (update) or gated against (check), project-relative. */ + baselines: string[]; +} + +export interface GoldenGateOptions { + /** Refresh baselines from the current render instead of gating. */ + update?: boolean; + /** Explicit sample times in seconds; overrides manifest and baseline-derived times. */ + at?: number[]; + /** CLI override for the per-channel pixel tolerance (0-1). */ + threshold?: number; + timeoutMs?: number; + autoProxy?: boolean; + browserGpuMode?: BrowserGpuMode; +} + +export function timeMsFromSeconds(seconds: number): number { + return Math.round(seconds * 1000); +} + +export function goldenTimeFileName(timeMs: number): string { + return `${timeMs}.png`; +} + +/** `.png` → timeMs; anything else (manifest, diff artifacts) → null. */ +export function parseGoldenTimeFileName(fileName: string): number | null { + const match = /^(\d+)\.png$/.exec(fileName); + return match ? Number.parseInt(match[1]!, 10) : null; +} + +function goldenCompositionDir(projectDir: string, compositionId: string): string { + return join(projectDir, GOLDEN_DIR_NAME, compositionId); +} + +export function goldenBaselinePath( + projectDir: string, + compositionId: string, + timeMs: number, +): string { + return join(goldenCompositionDir(projectDir, compositionId), goldenTimeFileName(timeMs)); +} + +function manifestError(sourcePath: string, detail: string): Error { + return new Error(`Invalid golden manifest ${sourcePath}: ${detail}`); +} + +function manifestTimes(value: unknown, sourcePath: string): number[] | undefined { + if (value === undefined) return undefined; + const valid = + Array.isArray(value) && + value.length > 0 && + value.every((t) => typeof t === "number" && Number.isFinite(t) && t >= 0); + if (!valid) { + throw manifestError(sourcePath, '"times" must be a non-empty array of non-negative seconds'); + } + return value.filter((t): t is number => typeof t === "number"); +} + +function manifestFraction(value: unknown, field: string, sourcePath: string): number | undefined { + if (value === undefined) return undefined; + if (typeof value !== "number" || value < 0 || value > 1) { + throw manifestError(sourcePath, `"${field}" must be a number from 0 to 1`); + } + return value; +} + +function manifestBoolean(value: unknown, field: string, sourcePath: string): boolean | undefined { + if (value === undefined) return undefined; + if (typeof value !== "boolean") { + throw manifestError(sourcePath, `"${field}" must be a boolean`); + } + return value; +} + +function manifestRecord(raw: string, sourcePath: string): Record { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw manifestError(sourcePath, "not valid JSON"); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw manifestError(sourcePath, "expected a JSON object"); + } + return { ...parsed }; +} + +/** Parse and validate a golden.json manifest. Throws with the offending field on invalid input. */ +export function parseGoldenManifest(raw: string, sourcePath: string): GoldenManifest { + const record = manifestRecord(raw, sourcePath); + const manifest: GoldenManifest = {}; + const times = manifestTimes(record.times, sourcePath); + if (times) manifest.times = times; + const threshold = manifestFraction(record.threshold, "threshold", sourcePath); + if (threshold !== undefined) manifest.threshold = threshold; + const maxDiffRatio = manifestFraction(record.maxDiffRatio, "maxDiffRatio", sourcePath); + if (maxDiffRatio !== undefined) manifest.maxDiffRatio = maxDiffRatio; + const ignoreAntialiasing = manifestBoolean( + record.ignoreAntialiasing, + "ignoreAntialiasing", + sourcePath, + ); + if (ignoreAntialiasing !== undefined) manifest.ignoreAntialiasing = ignoreAntialiasing; + return manifest; +} + +export function readGoldenManifest( + projectDir: string, + compositionId: string, +): GoldenManifest | null { + const manifestPath = join(goldenCompositionDir(projectDir, compositionId), GOLDEN_MANIFEST_NAME); + if (!existsSync(manifestPath)) return null; + return parseGoldenManifest(readFileSync(manifestPath, "utf8"), manifestPath); +} + +/** Baseline sample times (ms) derived from committed `.png` files, sorted. */ +export function listGoldenBaselineTimesMs(projectDir: string, compositionId: string): number[] { + const dir = goldenCompositionDir(projectDir, compositionId); + if (!existsSync(dir)) return []; + return readdirSync(dir) + .map(parseGoldenTimeFileName) + .filter((timeMs): timeMs is number => timeMs !== null) + .sort((a, b) => a - b); +} + +/** + * Default sample spread: evenly spaced frames whose final point backs off the + * exact duration to a readable tail. Mirrors the snapshot command's + * `computeSnapshotTimes`/`tailFrameTime` policy — kept as a local copy so the + * golden module does not import the snapshot command (which dynamically + * imports this module back). + */ +function defaultGoldenTimes(duration: number): number[] { + const tail = Math.max(0, duration - Math.max(0.05, duration * 0.03)); + const round = (t: number) => Math.round(t * 1000) / 1000; + const times = Array.from( + { length: DEFAULT_GOLDEN_FRAMES }, + (_, i) => (i / (DEFAULT_GOLDEN_FRAMES - 1)) * duration, + ); + times[times.length - 1] = tail; + return times.map(round); +} + +/** + * Pick the sample times, in precedence order: explicit `--at` override, + * manifest `times`, times encoded in existing baseline filenames, then the + * default spread (evenly spaced with a readable tail). + */ +export function resolveGoldenSampleTimes(input: { + atOverride?: number[]; + manifestTimes?: number[]; + baselineTimesMs?: number[]; + duration: number; +}): number[] { + if (input.atOverride && input.atOverride.length > 0) return [...input.atOverride]; + if (input.manifestTimes && input.manifestTimes.length > 0) return [...input.manifestTimes]; + if (input.baselineTimesMs && input.baselineTimesMs.length > 0) { + return input.baselineTimesMs.map((timeMs) => timeMs / 1000); + } + if (!(input.duration > 0)) { + throw new Error( + "Could not determine composition duration and no golden sample times were provided — pass --at or add times to golden.json", + ); + } + return defaultGoldenTimes(input.duration); +} + +export function resolveGoldenConfig( + manifest: GoldenManifest | null, + overrides: { threshold?: number } = {}, +): ResolvedGoldenConfig { + return { + threshold: overrides.threshold ?? manifest?.threshold ?? DEFAULT_DIFF_THRESHOLD, + maxDiffRatio: manifest?.maxDiffRatio ?? 0, + ignoreAntialiasing: manifest?.ignoreAntialiasing !== false, + }; +} + +/** Gate decision for one compared frame. */ +export function evaluateGoldenDiff( + result: PixelDiffResult, + maxDiffRatio: number, +): "pass" | Extract { + if (result.dimensionMismatch) return "dimension-mismatch"; + return result.diffRatio > maxDiffRatio ? "pixel-diff" : "pass"; +} + +function relativeToProject(projectDir: string, path: string): string { + const rel = relative(projectDir, path); + return rel.startsWith("..") || isAbsolute(rel) ? path : rel; +} + +function roundTime(time: number): number { + return Math.round(time * 1000) / 1000; +} + +interface GoldenCapture { + time: number; + timeMs: number; + png: Uint8Array; +} + +interface CapturedComposition { + compositionId: string; + times: number[]; + captures: GoldenCapture[]; + manifest: GoldenManifest | null; +} + +/** + * Open the composition once, resolve the sample times, and screenshot each. + * Font-localized bundling and the settled-page capture path mirror + * `hyperframes snapshot`, so goldens match what agents already review. + */ +async function captureGoldenFrames( + project: ProjectDir, + opts: GoldenGateOptions, +): Promise { + const { bundleWithLocalizedFonts } = await import("../utils/bundleWithLocalizedFonts.js"); + const html = await bundleWithLocalizedFonts(project.dir); + const server = await serveStaticProjectHtml(project.dir, html, undefined, [], opts.autoProxy); + try { + const { browser: chromeBrowser, page } = await openSettledCompositionPage(html, server.url, { + renderReadyTimeoutMs: opts.timeoutMs ?? 5000, + renderReadyWarningSuffix: "golden captures may be inaccurate", + browserGpuMode: opts.browserGpuMode, + }); + try { + const meta = await page.evaluate(() => { + const root = document.querySelector("[data-composition-id]"); + // Serialized into the page: the runtime attaches __player without typings. + const win = window as unknown as { __player?: { getDuration?: () => number } }; + const candidates = [ + Number(win.__player?.getDuration?.()), + Number.parseFloat(root?.getAttribute("data-duration") ?? ""), + ]; + return { + id: root?.getAttribute("data-composition-id") ?? "", + duration: candidates.find((d) => Number.isFinite(d) && d > 0) ?? 0, + }; + }); + + const compositionId = meta.id || project.name; + const manifest = readGoldenManifest(project.dir, compositionId); + const baselineTimesMs = listGoldenBaselineTimesMs(project.dir, compositionId); + + if (!opts.update && !manifest?.times?.length && baselineTimesMs.length === 0) { + throw new Error( + `No golden baselines found for "${compositionId}" (${GOLDEN_DIR_NAME}/${compositionId}/). ` + + "Create them with `hyperframes check --update-golden` or `hyperframes snapshot --update-golden`.", + ); + } + + const times = resolveGoldenSampleTimes({ + atOverride: opts.at, + manifestTimes: manifest?.times, + baselineTimesMs, + duration: meta.duration, + }).map(roundTime); + + const captures: GoldenCapture[] = []; + for (const time of times) { + await seekCompositionTimeline(page, time); + const png = await page.screenshot({ type: "png" }); + captures.push({ time, timeMs: timeMsFromSeconds(time), png }); + } + return { compositionId, times, captures, manifest }; + } finally { + await chromeBrowser.close(); + } + } finally { + await server.close(); + } +} + +function writeGoldenBaselines( + project: ProjectDir, + captured: CapturedComposition, + opts: GoldenGateOptions, +): GoldenSummary { + const dir = goldenCompositionDir(project.dir, captured.compositionId); + mkdirSync(dir, { recursive: true }); + + // Drop stale baselines whose time is no longer sampled — they would gate + // nothing but still look authoritative in review. + const keep = new Set(captured.captures.map((capture) => goldenTimeFileName(capture.timeMs))); + for (const file of readdirSync(dir)) { + if (parseGoldenTimeFileName(file) !== null && !keep.has(file)) { + rmSync(join(dir, file), { force: true }); + } + } + + const baselines: string[] = []; + for (const capture of captured.captures) { + const path = join(dir, goldenTimeFileName(capture.timeMs)); + writeFileSync(path, capture.png); + baselines.push(relativeToProject(project.dir, path)); + } + + const manifest: GoldenManifest = { + ...(captured.manifest ?? {}), + times: captured.times, + }; + if (opts.threshold !== undefined) manifest.threshold = opts.threshold; + writeFileSync(join(dir, GOLDEN_MANIFEST_NAME), `${JSON.stringify(manifest, null, 2)}\n`); + + return { + ok: true, + updated: true, + compositionId: captured.compositionId, + compared: captured.captures.length, + times: captured.times, + failed: [], + diffSheet: null, + baselines, + }; +} + +async function compareAgainstBaselines( + project: ProjectDir, + captured: CapturedComposition, + opts: GoldenGateOptions, +): Promise { + const config = resolveGoldenConfig(captured.manifest, { threshold: opts.threshold }); + const diffDir = join(project.dir, GOLDEN_DIFF_DIR_NAME, captured.compositionId); + rmSync(diffDir, { recursive: true, force: true }); + + const failed: GoldenFailure[] = []; + const baselines: string[] = []; + const sheetRows: { label: string; paths: string[] }[] = []; + + for (const capture of captured.captures) { + const baselinePath = goldenBaselinePath(project.dir, captured.compositionId, capture.timeMs); + baselines.push(relativeToProject(project.dir, baselinePath)); + + if (!existsSync(baselinePath)) { + failed.push({ + id: captured.compositionId, + time: capture.time, + timeMs: capture.timeMs, + maxDelta: 255, + diffRatio: 1, + reason: "missing-baseline", + }); + mkdirSync(diffDir, { recursive: true }); + writeFileSync(join(diffDir, `${capture.timeMs}-current.png`), capture.png); + continue; + } + + const result = await diffPngs(baselinePath, capture.png, { + threshold: config.threshold, + ignoreAntialiasing: config.ignoreAntialiasing, + }); + const verdict = evaluateGoldenDiff(result, config.maxDiffRatio); + if (verdict === "pass") continue; + + failed.push({ + id: captured.compositionId, + time: capture.time, + timeMs: capture.timeMs, + maxDelta: result.maxDelta, + diffRatio: Math.round(result.diffRatio * 1e6) / 1e6, + reason: verdict, + }); + + mkdirSync(diffDir, { recursive: true }); + const currentPath = join(diffDir, `${capture.timeMs}-current.png`); + const diffPath = join(diffDir, `${capture.timeMs}-diff.png`); + writeFileSync(currentPath, capture.png); + await writeRawImagePng( + { width: result.width, height: result.height, data: result.diff }, + diffPath, + ); + sheetRows.push({ + label: `t=${capture.time}s`, + paths: [baselinePath, currentPath, diffPath], + }); + } + + let diffSheet: string | null = null; + if (sheetRows.length > 0) { + const { createGoldenDiffContactSheet } = await import("../capture/contactSheet.js"); + const sheets = await createGoldenDiffContactSheet( + sheetRows.map((row) => ({ + label: row.label, + baselinePath: row.paths[0]!, + currentPath: row.paths[1]!, + diffPath: row.paths[2]!, + })), + join(diffDir, "contact-sheet.jpg"), + ); + diffSheet = sheets[0] ? relativeToProject(project.dir, sheets[0]) : null; + } + + return { + ok: failed.length === 0, + updated: false, + compositionId: captured.compositionId, + compared: captured.captures.length, + times: captured.times, + failed, + diffSheet, + baselines, + }; +} + +/** + * Run the golden baseline gate for a project: capture at the manifest / + * baseline sample times and pixel-diff against `golden//`, + * or refresh those baselines when `update` is set. + */ +export async function runGoldenGate( + project: ProjectDir, + opts: GoldenGateOptions = {}, +): Promise { + const captured = await captureGoldenFrames(project, opts); + if (opts.update) return writeGoldenBaselines(project, captured, opts); + return compareAgainstBaselines(project, captured, opts); +} diff --git a/packages/cli/src/golden/pixelDiff.test.ts b/packages/cli/src/golden/pixelDiff.test.ts new file mode 100644 index 0000000000..52d598e54c --- /dev/null +++ b/packages/cli/src/golden/pixelDiff.test.ts @@ -0,0 +1,177 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import sharp from "sharp"; +import { describe, expect, it } from "vitest"; +import { + decodeRawImage, + diffPngs, + diffRawImages, + writeRawImagePng, + type RawImage, +} from "./pixelDiff.js"; + +type Rgba = [number, number, number, number]; + +function solid(width: number, height: number, rgba: Rgba): RawImage { + const data = new Uint8Array(width * height * 4); + for (let offset = 0; offset < data.length; offset += 4) { + data.set(rgba, offset); + } + return { width, height, data }; +} + +function setPixel(image: RawImage, x: number, y: number, rgba: Rgba): void { + image.data.set(rgba, (y * image.width + x) * 4); +} + +function getPixel(image: { width: number; data: Uint8Array }, x: number, y: number): Rgba { + const offset = (y * image.width + x) * 4; + return [ + image.data[offset]!, + image.data[offset + 1]!, + image.data[offset + 2]!, + image.data[offset + 3]!, + ]; +} + +const WHITE: Rgba = [255, 255, 255, 255]; +const BLACK: Rgba = [0, 0, 0, 255]; + +describe("diffRawImages", () => { + it("reports zero differences for identical images", () => { + const a = solid(8, 6, [12, 34, 56, 255]); + const b = solid(8, 6, [12, 34, 56, 255]); + const result = diffRawImages(a, b); + expect(result.diffPixels).toBe(0); + expect(result.aaPixels).toBe(0); + expect(result.diffRatio).toBe(0); + expect(result.maxDelta).toBe(0); + expect(result.dimensionMismatch).toBe(false); + expect(result.totalPixels).toBe(48); + }); + + it("counts an isolated changed pixel and reports its channel delta", () => { + const a = solid(8, 8, WHITE); + const b = solid(8, 8, WHITE); + setPixel(b, 3, 4, [255, 0, 0, 255]); + const result = diffRawImages(a, b); + expect(result.diffPixels).toBe(1); + expect(result.maxDelta).toBe(255); + expect(result.diffRatio).toBeCloseTo(1 / 64, 10); + }); + + it("absorbs sub-threshold channel drift", () => { + const a = solid(4, 4, [100, 100, 100, 255]); + const b = solid(4, 4, [120, 100, 100, 255]); + // threshold 0.1 → tolerance 26 per channel, drift of 20 passes... + expect(diffRawImages(a, b, { threshold: 0.1 }).diffPixels).toBe(0); + // ...but a zero threshold counts every drifted pixel. + const strict = diffRawImages(a, b, { threshold: 0 }); + expect(strict.diffPixels).toBe(16); + expect(strict.maxDelta).toBe(20); + }); + + it("classifies a one-pixel edge shift as anti-aliasing and can be told not to", () => { + // Vertical black/white edge, shifted right by one column in `current`. + const baseline = solid(10, 10, WHITE); + const current = solid(10, 10, WHITE); + for (let y = 0; y < 10; y++) { + for (let x = 0; x < 4; x++) setPixel(baseline, x, y, BLACK); + for (let x = 0; x < 5; x++) setPixel(current, x, y, BLACK); + } + const lenient = diffRawImages(baseline, current); + expect(lenient.diffPixels).toBe(0); + expect(lenient.aaPixels).toBe(10); + + const strict = diffRawImages(baseline, current, { ignoreAntialiasing: false }); + expect(strict.diffPixels).toBe(10); + expect(strict.aaPixels).toBe(0); + }); + + it("does not classify a genuinely new color as anti-aliasing", () => { + const baseline = solid(10, 10, WHITE); + const current = solid(10, 10, WHITE); + // 3×3 red block: no white-image neighborhood ever contains red. + for (let y = 4; y < 7; y++) { + for (let x = 4; x < 7; x++) setPixel(current, x, y, [255, 0, 0, 255]); + } + const result = diffRawImages(baseline, current); + expect(result.diffPixels).toBe(9); + expect(result.aaPixels).toBe(0); + }); + + it("treats mismatched dimensions as a total failure", () => { + const a = solid(8, 8, WHITE); + const b = solid(9, 8, WHITE); + const result = diffRawImages(a, b); + expect(result.dimensionMismatch).toBe(true); + expect(result.diffRatio).toBe(1); + expect(result.maxDelta).toBe(255); + expect(result.width).toBe(8); + expect(result.height).toBe(8); + }); + + it("paints counted diffs red and unchanged pixels as a pale backdrop", () => { + const a = solid(6, 6, WHITE); + const b = solid(6, 6, WHITE); + setPixel(b, 2, 2, BLACK); + const result = diffRawImages(a, b); + expect(getPixel({ width: result.width, data: result.diff }, 2, 2)).toEqual([255, 0, 64, 255]); + const [r, g, bChan] = getPixel({ width: result.width, data: result.diff }, 0, 0); + expect(r).toBe(g); + expect(g).toBe(bChan); + expect(r).toBeGreaterThan(180); + }); +}); + +describe("PNG round trip", () => { + it("diffs PNG files and re-encodes the visualization losslessly", async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-pixel-diff-test-")); + try { + const baselinePath = join(dir, "baseline.png"); + const currentPath = join(dir, "current.png"); + await sharp({ + create: { width: 12, height: 8, channels: 4, background: { r: 0, g: 0, b: 255, alpha: 1 } }, + }) + .png() + .toFile(baselinePath); + // Same canvas with a 4×4 red patch composited into the corner. + await sharp({ + create: { width: 12, height: 8, channels: 4, background: { r: 0, g: 0, b: 255, alpha: 1 } }, + }) + .composite([ + { + input: { + create: { + width: 4, + height: 4, + channels: 4, + background: { r: 255, g: 0, b: 0, alpha: 1 }, + }, + }, + left: 0, + top: 0, + }, + ]) + .png() + .toFile(currentPath); + + const result = await diffPngs(baselinePath, currentPath, { ignoreAntialiasing: false }); + expect(result.diffPixels).toBe(16); + expect(result.maxDelta).toBe(255); + + const diffPath = join(dir, "diff.png"); + await writeRawImagePng( + { width: result.width, height: result.height, data: result.diff }, + diffPath, + ); + const decoded = await decodeRawImage(diffPath); + expect(decoded.width).toBe(12); + expect(decoded.height).toBe(8); + expect(getPixel(decoded, 1, 1)).toEqual([255, 0, 64, 255]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/cli/src/golden/pixelDiff.ts b/packages/cli/src/golden/pixelDiff.ts new file mode 100644 index 0000000000..c7a8d4835d --- /dev/null +++ b/packages/cli/src/golden/pixelDiff.ts @@ -0,0 +1,236 @@ +/** + * Pixel-level PNG comparison for the golden baseline gate. + * + * Pure diff math on raw RGBA buffers (unit-testable without Chrome or + * fixtures) plus thin sharp-backed encode/decode helpers. No new + * dependencies: sharp is already a CLI dependency for contact sheets. + */ + +import sharp from "sharp"; + +/** Decoded RGBA image (4 channels, row-major). */ +export interface RawImage { + width: number; + height: number; + data: Uint8Array; +} + +export interface PixelDiffOptions { + /** + * Per-channel intensity tolerance as a fraction of 255 (default 0.1). + * A pixel only counts as different when at least one channel deviates + * by more than `threshold * 255`. + */ + threshold?: number; + /** + * Treat 1px edge shifts as anti-aliasing noise and exclude them from the + * failure count (default true). A differing pixel is classified as + * anti-aliasing when each side finds a matching pixel for the other + * side's color within its own 8-neighborhood — the signature of font / + * shape rasterization jitter, not a layout or color regression. + */ + ignoreAntialiasing?: boolean; +} + +export interface PixelDiffResult { + width: number; + height: number; + totalPixels: number; + /** Differing pixels that count against the gate (anti-aliasing excluded). */ + diffPixels: number; + /** Differing pixels classified as anti-aliasing noise and ignored. */ + aaPixels: number; + /** diffPixels / totalPixels. */ + diffRatio: number; + /** Max per-channel delta (0-255) among counted differing pixels. */ + maxDelta: number; + /** Baseline and current image dimensions do not match. */ + dimensionMismatch: boolean; + /** RGBA visualization: dimmed baseline, red = diff, amber = ignored AA. */ + diff: Uint8Array; +} + +export const DEFAULT_DIFF_THRESHOLD = 0.1; + +const DIFF_COLOR = { r: 255, g: 0, b: 64 }; +const AA_COLOR = { r: 255, g: 196, b: 0 }; + +function channelDelta(a: RawImage, b: RawImage, aOffset: number, bOffset: number): number { + let max = 0; + for (let channel = 0; channel < 4; channel++) { + const delta = Math.abs(a.data[aOffset + channel]! - b.data[bOffset + channel]!); + if (delta > max) max = delta; + } + return max; +} + +/** True when some pixel in `img`'s 8-neighborhood of (x, y) matches the RGBA at `targetOffset` in `target` within `tolerance`. */ +function neighborhoodMatches( + img: RawImage, + x: number, + y: number, + target: RawImage, + targetOffset: number, + tolerance: number, +): boolean { + const left = Math.max(0, x - 1); + const right = Math.min(img.width - 1, x + 1); + const top = Math.max(0, y - 1); + const bottom = Math.min(img.height - 1, y + 1); + for (let ny = top; ny <= bottom; ny++) { + for (let nx = left; nx <= right; nx++) { + if (nx === x && ny === y) continue; + const offset = (ny * img.width + nx) * 4; + if (channelDelta(img, target, offset, targetOffset) <= tolerance) return true; + } + } + return false; +} + +function paint(diff: Uint8Array, offset: number, color: { r: number; g: number; b: number }): void { + diff[offset] = color.r; + diff[offset + 1] = color.g; + diff[offset + 2] = color.b; + diff[offset + 3] = 255; +} + +/** Pale grayscale rendering of the baseline pixel, so diffs pop against context. */ +function paintBackground(diff: Uint8Array, baseline: RawImage, offset: number): void { + const luma = Math.round( + 0.299 * baseline.data[offset]! + + 0.587 * baseline.data[offset + 1]! + + 0.114 * baseline.data[offset + 2]!, + ); + const dimmed = Math.round(luma * 0.25 + 190); + diff[offset] = dimmed; + diff[offset + 1] = dimmed; + diff[offset + 2] = dimmed; + diff[offset + 3] = 255; +} + +function dimensionMismatchResult(baseline: RawImage): PixelDiffResult { + const totalPixels = baseline.width * baseline.height; + const diff = new Uint8Array(totalPixels * 4); + for (let offset = 0; offset < diff.length; offset += 4) { + paint(diff, offset, DIFF_COLOR); + } + return { + width: baseline.width, + height: baseline.height, + totalPixels, + diffPixels: totalPixels, + aaPixels: 0, + diffRatio: 1, + maxDelta: 255, + dimensionMismatch: true, + diff, + }; +} + +interface PixelComparison { + diff: Uint8Array; + diffPixels: number; + aaPixels: number; + maxDelta: number; +} + +function comparePixels( + baseline: RawImage, + current: RawImage, + tolerance: number, + ignoreAntialiasing: boolean, +): PixelComparison { + const diff = new Uint8Array(baseline.width * baseline.height * 4); + const result: PixelComparison = { diff, diffPixels: 0, aaPixels: 0, maxDelta: 0 }; + + for (let y = 0; y < baseline.height; y++) { + for (let x = 0; x < baseline.width; x++) { + const offset = (y * baseline.width + x) * 4; + const delta = channelDelta(baseline, current, offset, offset); + if (delta <= tolerance) { + paintBackground(diff, baseline, offset); + continue; + } + const isAntialiasing = + ignoreAntialiasing && + neighborhoodMatches(baseline, x, y, current, offset, tolerance) && + neighborhoodMatches(current, x, y, baseline, offset, tolerance); + if (isAntialiasing) { + result.aaPixels++; + paint(diff, offset, AA_COLOR); + continue; + } + result.diffPixels++; + result.maxDelta = Math.max(result.maxDelta, delta); + paint(diff, offset, DIFF_COLOR); + } + } + return result; +} + +/** + * Compare two RGBA images of equal dimensions. Baseline dimensions win for + * the visualization; mismatched dimensions are reported as a total failure + * (a resized canvas is always a regression, not a pixel drift). + */ +export function diffRawImages( + baseline: RawImage, + current: RawImage, + options: PixelDiffOptions = {}, +): PixelDiffResult { + if (baseline.width !== current.width || baseline.height !== current.height) { + return dimensionMismatchResult(baseline); + } + + const threshold = options.threshold ?? DEFAULT_DIFF_THRESHOLD; + const tolerance = Math.round(Math.max(0, Math.min(1, threshold)) * 255); + const totalPixels = baseline.width * baseline.height; + const compared = comparePixels( + baseline, + current, + tolerance, + options.ignoreAntialiasing !== false, + ); + + return { + width: baseline.width, + height: baseline.height, + totalPixels, + diffPixels: compared.diffPixels, + aaPixels: compared.aaPixels, + diffRatio: totalPixels === 0 ? 0 : compared.diffPixels / totalPixels, + maxDelta: compared.maxDelta, + dimensionMismatch: false, + diff: compared.diff, + }; +} + +/** Decode a PNG file path or buffer into flat RGBA. */ +export async function decodeRawImage(input: string | Uint8Array): Promise { + const { data, info } = await sharp(input).ensureAlpha().raw().toBuffer({ + resolveWithObject: true, + }); + return { width: info.width, height: info.height, data: new Uint8Array(data) }; +} + +/** Encode flat RGBA back into a PNG file. */ +export async function writeRawImagePng(image: RawImage, outputPath: string): Promise { + await sharp(Buffer.from(image.data.buffer, image.data.byteOffset, image.data.byteLength), { + raw: { width: image.width, height: image.height, channels: 4 }, + }) + .png() + .toFile(outputPath); +} + +/** Convenience wrapper: decode two PNGs (path or buffer) and diff them. */ +export async function diffPngs( + baseline: string | Uint8Array, + current: string | Uint8Array, + options: PixelDiffOptions = {}, +): Promise { + const [baselineRaw, currentRaw] = await Promise.all([ + decodeRawImage(baseline), + decodeRawImage(current), + ]); + return diffRawImages(baselineRaw, currentRaw, options); +}