From 7f31d315a07341c93e68ba04b18ddf8fde6c48bf Mon Sep 17 00:00:00 2001 From: Akshar Patel Date: Mon, 12 Oct 2026 22:34:12 -0400 Subject: [PATCH 1/3] fix(producer): preserve runtime media playback rates --- .../src/services/htmlCompiler.test.ts | 24 +++++ .../producer/src/services/htmlCompiler.ts | 26 ++++- .../render/stages/extractVideosStage.test.ts | 28 ++++++ .../render/stages/extractVideosStage.ts | 1 + .../services/render/stages/probeStage.test.ts | 94 +++++++++++++++++++ .../src/services/render/stages/probeStage.ts | 18 ++++ 6 files changed, 190 insertions(+), 1 deletion(-) diff --git a/packages/producer/src/services/htmlCompiler.test.ts b/packages/producer/src/services/htmlCompiler.test.ts index 85050835b85..a41b2c29040 100644 --- a/packages/producer/src/services/htmlCompiler.test.ts +++ b/packages/producer/src/services/htmlCompiler.test.ts @@ -36,6 +36,7 @@ describe("discoverMediaFromBrowser", () => { currentSrcById: Record, serializeCallback = false, intrinsicDurationById: Record = {}, + defaultPlaybackRateById: Record = {}, ) { const { document } = parseHTML(html); for (const [id, currentSrc] of Object.entries(currentSrcById)) { @@ -46,6 +47,13 @@ describe("discoverMediaFromBrowser", () => { const element = document.getElementById(id); if (element) Object.defineProperty(element, "duration", { value: duration }); } + for (const [id, defaultPlaybackRate] of Object.entries(defaultPlaybackRateById)) { + const element = document.getElementById(id); + if (element) { + Object.defineProperty(element, "defaultPlaybackRate", { value: defaultPlaybackRate }); + Object.defineProperty(element, "playbackRate", { value: defaultPlaybackRate * 2 }); + } + } const previousDocument = Reflect.get(globalThis, "document"); Reflect.set(globalThis, "document", document); try { @@ -129,6 +137,22 @@ describe("discoverMediaFromBrowser", () => { expect(media[0]).toMatchObject({ start: 0, end: 2, duration: 2, mediaStart: 1 }); }); + it("reads authored rates and native defaults after the browser callback is serialized", async () => { + const media = await discover( + ` + + + + `, + {}, + true, + {}, + { authored: 0.5, native: 0.5, invalid: 1.25 }, + ); + + expect(media.map((element) => element.playbackRate)).toEqual([1.5, 0.5, 1.25, 10, 1]); + }); + it("reports colliding empty-src videos by render id, not author id", async () => { const media = await discover( `` + diff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.ts index a810dc4d855..1579c042a24 100644 --- a/packages/producer/src/services/htmlCompiler.ts +++ b/packages/producer/src/services/htmlCompiler.ts @@ -36,6 +36,7 @@ import { type UnresolvedElement, } from "@hyperframes/core"; import { MAX_AUDIO_GAIN } from "@hyperframes/core/audio-gain"; +import { readPlaybackRate } from "@hyperframes/parsers/media-duration"; import { gsapCdnDist } from "@hyperframes/core/gsap-cdn"; import { assignBundledRuntimeCompositionIds, @@ -2235,6 +2236,7 @@ export interface BrowserMediaElement { /** True when compilation inferred duration from the fallback source. */ durationInferred: boolean; mediaStart: number; + playbackRate: number; loop: boolean; hasAudio: boolean; volume: number; @@ -2260,6 +2262,8 @@ export async function discoverMediaFromBrowser(page: Page): Promise ({ + ({ + endRaw, + durationRaw, + intrinsicDuration, + playbackStartRaw, + mediaStartRaw, + playbackRateRaw, + defaultPlaybackRate, + ...element + }) => ({ ...element, end: parseStrictFiniteTimingNumber(endRaw) ?? 0, duration: @@ -2353,6 +2373,10 @@ export async function discoverMediaFromBrowser(page: Page): Promise (name === "data-playback-rate" ? playbackRateRaw : null), + defaultPlaybackRate, + ), }), ); } diff --git a/packages/producer/src/services/render/stages/extractVideosStage.test.ts b/packages/producer/src/services/render/stages/extractVideosStage.test.ts index dc4d91598d2..97de3e87835 100644 --- a/packages/producer/src/services/render/stages/extractVideosStage.test.ts +++ b/packages/producer/src/services/render/stages/extractVideosStage.test.ts @@ -105,6 +105,34 @@ describe("appendAutoDetectedVideoAudio", () => { }); }); + it.each([ + 1.5, + { + target: "rate", + points: [ + { t: 0, v: 1 }, + { t: 3, v: 2 }, + ], + }, + ] satisfies NonNullable[])( + "keeps the video's rate and trim in its detected audio (%j)", + (playbackRate) => { + const composition: Parameters[0] = { + videos: [makeVideo({ start: 2, end: 5, mediaStart: 0.5, playbackRate })], + audios: [], + }; + + appendAutoDetectedVideoAudio(composition, [makeExtracted("v1", true)]); + + expect(composition.audios[0]).toMatchObject({ + start: 2, + end: 5, + mediaStart: 0.5, + playbackRate, + }); + }, + ); + it("skips a muted video even when the source file has audio", () => { const composition = { videos: [makeVideo({ hasAudio: false })], diff --git a/packages/producer/src/services/render/stages/extractVideosStage.ts b/packages/producer/src/services/render/stages/extractVideosStage.ts index 5208f5fac57..f774e3b7826 100644 --- a/packages/producer/src/services/render/stages/extractVideosStage.ts +++ b/packages/producer/src/services/render/stages/extractVideosStage.ts @@ -578,6 +578,7 @@ export function appendAutoDetectedVideoAudio( start: video.start, end: video.end, mediaStart: video.mediaStart, + playbackRate: video.playbackRate, layer: 0, volume: 1.0, type: "video", diff --git a/packages/producer/src/services/render/stages/probeStage.test.ts b/packages/producer/src/services/render/stages/probeStage.test.ts index 45bdddb1346..4dafea7140b 100644 --- a/packages/producer/src/services/render/stages/probeStage.test.ts +++ b/packages/producer/src/services/render/stages/probeStage.test.ts @@ -2,6 +2,7 @@ import { afterAll, describe, expect, it, mock } from "bun:test"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import type { RateSpec } from "@hyperframes/core"; const workDir = mkdtempSync(join(tmpdir(), "hf-stage-test-")); afterAll(() => rmSync(workDir, { recursive: true, force: true })); @@ -413,6 +414,99 @@ describe("runProbeStage — forceScreenshot threading", () => { }); // Scene in-point 5 and clip trim 1: the collector starts picture and sound at source 6. + it.each([ + { tagName: "video", existing: false }, + { tagName: "audio", existing: false }, + { tagName: "video", existing: true }, + { tagName: "audio", existing: true }, + ])("retains a runtime $tagName rate (existing: $existing)", async ({ tagName, existing }) => { + resetRetryMocks(); + browserMediaResults = [ + { + id: "clip", + tagName, + src: "clip.mp4", + start: 0, + end: 3, + duration: 3, + mediaStart: 0.5, + playbackRate: 1.5, + loop: false, + hasAudio: true, + volume: 1, + muted: false, + }, + ]; + const { runProbeStage } = await import("./probeStage.js"); + const input = makeProbeInput({}); + const clip = { + id: "clip", + src: "clip.mp4", + start: 0, + end: 3, + mediaStart: 0.5, + playbackRate: 1, + }; + if (existing && tagName === "video") { + input.composition.videos.push({ ...clip, loop: false, hasAudio: true }); + input.composition.audios.push({ ...clip, id: "clip-audio", layer: 0, type: "video" }); + } + if (existing && tagName === "audio") { + input.composition.audios.push({ ...clip, layer: 0, type: "audio" }); + } + + await runProbeStage(input); + + const media = tagName === "video" ? input.composition.videos : input.composition.audios; + expect(media[0]).toMatchObject({ playbackRate: 1.5, mediaStart: 0.5 }); + if (existing && tagName === "video") { + expect(input.composition.audios[0]).toMatchObject({ playbackRate: 1.5, mediaStart: 0.5 }); + } + }); + + it("preserves a collected speed ramp during scalar browser reconciliation", async () => { + resetRetryMocks(); + browserMediaResults = [ + { + id: "clip", + tagName: "video", + src: "clip.mp4", + start: 0, + end: 3, + duration: 3, + mediaStart: 0, + playbackRate: 1, + loop: false, + hasAudio: false, + volume: 1, + muted: false, + }, + ]; + const { runProbeStage } = await import("./probeStage.js"); + const input = makeProbeInput({}); + const playbackRate = { + target: "rate", + points: [ + { t: 0, v: 1 }, + { t: 3, v: 2 }, + ], + } satisfies RateSpec; + input.composition.videos.push({ + id: "clip", + src: "clip.mp4", + start: 0, + end: 3, + mediaStart: 0, + playbackRate, + loop: false, + hasAudio: false, + }); + + await runProbeStage(input); + + expect(input.composition.videos[0]?.playbackRate).toEqual(playbackRate); + }); + it.each([ { pageStart: 1, expected: 6 }, { pageStart: 3, expected: 8 }, diff --git a/packages/producer/src/services/render/stages/probeStage.ts b/packages/producer/src/services/render/stages/probeStage.ts index 5ecff9bd416..15ff02f3ba1 100644 --- a/packages/producer/src/services/render/stages/probeStage.ts +++ b/packages/producer/src/services/render/stages/probeStage.ts @@ -599,6 +599,19 @@ export async function runProbeStage(input: ProbeStageInput): Promise 0 && Math.abs((existing.volume ?? 1) - el.volume) > BROWSER_MEDIA_EPSILON @@ -658,6 +675,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise Date: Mon, 12 Oct 2026 23:36:43 -0400 Subject: [PATCH 2/3] fix(producer): preserve runtime-inserted speed ramps --- .../src/services/htmlCompiler.test.ts | 36 ++++ .../producer/src/services/htmlCompiler.ts | 16 +- .../services/render/stages/probeStage.test.ts | 168 ++++++++++-------- .../src/services/render/stages/probeStage.ts | 14 +- 4 files changed, 147 insertions(+), 87 deletions(-) diff --git a/packages/producer/src/services/htmlCompiler.test.ts b/packages/producer/src/services/htmlCompiler.test.ts index a41b2c29040..d90df840f49 100644 --- a/packages/producer/src/services/htmlCompiler.test.ts +++ b/packages/producer/src/services/htmlCompiler.test.ts @@ -153,6 +153,42 @@ describe("discoverMediaFromBrowser", () => { expect(media.map((element) => element.playbackRate)).toEqual([1.5, 0.5, 1.25, 10, 1]); }); + it.each(["video", "audio"])( + "discovers a %s rate lane after the browser callback is serialized", + async (tag) => { + const lane = { + target: "rate", + points: [ + { t: 0, v: 1 }, + { t: 1, v: 2 }, + ], + }; + const automation = JSON.stringify({ version: 1, lanes: [lane] }); + const media = await discover( + `<${tag} id="clip" src="clip.mp4" data-start="0" data-duration="1" data-playback-rate="3" data-automation='${automation}'>`, + {}, + true, + {}, + { clip: 0.5 }, + ); + + expect(media[0]?.playbackRate).toEqual(lane); + }, + ); + + it("uses scalar rates when automation is invalid or has no rate lane", async () => { + const media = await discover( + ` + `, + {}, + true, + {}, + { other: 0.5 }, + ); + + expect(media.map((element) => element.playbackRate)).toEqual([1.5, 0.5]); + }); + it("reports colliding empty-src videos by render id, not author id", async () => { const media = await discover( `` + diff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.ts index 1579c042a24..319a13cfd8b 100644 --- a/packages/producer/src/services/htmlCompiler.ts +++ b/packages/producer/src/services/htmlCompiler.ts @@ -37,6 +37,7 @@ import { } from "@hyperframes/core"; import { MAX_AUDIO_GAIN } from "@hyperframes/core/audio-gain"; import { readPlaybackRate } from "@hyperframes/parsers/media-duration"; +import { resolveRateSpec } from "@hyperframes/core/speed-ramp"; import { gsapCdnDist } from "@hyperframes/core/gsap-cdn"; import { assignBundledRuntimeCompositionIds, @@ -2236,7 +2237,7 @@ export interface BrowserMediaElement { /** True when compilation inferred duration from the fallback source. */ durationInferred: boolean; mediaStart: number; - playbackRate: number; + playbackRate: RateSpec; loop: boolean; hasAudio: boolean; volume: number; @@ -2263,6 +2264,7 @@ export async function discoverMediaFromBrowser(page: Page): Promise ({ @@ -2373,9 +2378,12 @@ export async function discoverMediaFromBrowser(page: Page): Promise (name === "data-playback-rate" ? playbackRateRaw : null), - defaultPlaybackRate, + playbackRate: resolveRateSpec( + automationRaw, + readPlaybackRate( + (name) => (name === "data-playback-rate" ? playbackRateRaw : null), + defaultPlaybackRate, + ), ), }), ); diff --git a/packages/producer/src/services/render/stages/probeStage.test.ts b/packages/producer/src/services/render/stages/probeStage.test.ts index 4dafea7140b..fd6611eaa4f 100644 --- a/packages/producer/src/services/render/stages/probeStage.test.ts +++ b/packages/producer/src/services/render/stages/probeStage.test.ts @@ -414,98 +414,116 @@ describe("runProbeStage — forceScreenshot threading", () => { }); // Scene in-point 5 and clip trim 1: the collector starts picture and sound at source 6. - it.each([ - { tagName: "video", existing: false }, - { tagName: "audio", existing: false }, - { tagName: "video", existing: true }, - { tagName: "audio", existing: true }, - ])("retains a runtime $tagName rate (existing: $existing)", async ({ tagName, existing }) => { - resetRetryMocks(); - browserMediaResults = [ - { + const runtimeRates = [ + 1.5, + { + target: "rate", + points: [ + { t: 0, v: 1 }, + { t: 3, v: 2 }, + ], + } satisfies RateSpec, + ]; + it.each( + runtimeRates.flatMap((playbackRate) => [ + { tagName: "video", existing: false, playbackRate }, + { tagName: "audio", existing: false, playbackRate }, + { tagName: "video", existing: true, playbackRate }, + { tagName: "audio", existing: true, playbackRate }, + ]), + )( + "retains a runtime $tagName rate (existing: $existing, rate: $playbackRate)", + async ({ tagName, existing, playbackRate }) => { + resetRetryMocks(); + browserMediaResults = [ + { + id: "clip", + tagName, + src: "clip.mp4", + start: 0, + end: 3, + duration: 3, + mediaStart: 0.5, + playbackRate, + loop: false, + hasAudio: true, + volume: 1, + muted: false, + }, + ]; + const { runProbeStage } = await import("./probeStage.js"); + const input = makeProbeInput({}); + const clip = { id: "clip", - tagName, src: "clip.mp4", start: 0, end: 3, - duration: 3, mediaStart: 0.5, - playbackRate: 1.5, - loop: false, - hasAudio: true, - volume: 1, - muted: false, - }, - ]; - const { runProbeStage } = await import("./probeStage.js"); - const input = makeProbeInput({}); - const clip = { - id: "clip", - src: "clip.mp4", - start: 0, - end: 3, - mediaStart: 0.5, - playbackRate: 1, - }; - if (existing && tagName === "video") { - input.composition.videos.push({ ...clip, loop: false, hasAudio: true }); - input.composition.audios.push({ ...clip, id: "clip-audio", layer: 0, type: "video" }); - } - if (existing && tagName === "audio") { - input.composition.audios.push({ ...clip, layer: 0, type: "audio" }); - } + playbackRate: 1, + }; + if (existing && tagName === "video") { + input.composition.videos.push({ ...clip, loop: false, hasAudio: true }); + input.composition.audios.push({ ...clip, id: "clip-audio", layer: 0, type: "video" }); + } + if (existing && tagName === "audio") { + input.composition.audios.push({ ...clip, layer: 0, type: "audio" }); + } - await runProbeStage(input); + await runProbeStage(input); - const media = tagName === "video" ? input.composition.videos : input.composition.audios; - expect(media[0]).toMatchObject({ playbackRate: 1.5, mediaStart: 0.5 }); - if (existing && tagName === "video") { - expect(input.composition.audios[0]).toMatchObject({ playbackRate: 1.5, mediaStart: 0.5 }); - } - }); + const media = tagName === "video" ? input.composition.videos : input.composition.audios; + expect(media[0]).toMatchObject({ playbackRate, mediaStart: 0.5 }); + if (existing && tagName === "video") { + expect(input.composition.audios[0]).toMatchObject({ playbackRate, mediaStart: 0.5 }); + } + }, + ); - it("preserves a collected speed ramp during scalar browser reconciliation", async () => { - resetRetryMocks(); - browserMediaResults = [ - { + it.each(runtimeRates)( + "preserves a collected speed ramp during browser rate %j reconciliation", + async (browserRate) => { + resetRetryMocks(); + browserMediaResults = [ + { + id: "clip", + tagName: "video", + src: "clip.mp4", + start: 0, + end: 3, + duration: 3, + mediaStart: 0, + playbackRate: browserRate, + loop: false, + hasAudio: false, + volume: 1, + muted: false, + }, + ]; + const { runProbeStage } = await import("./probeStage.js"); + const input = makeProbeInput({}); + const playbackRate = { + target: "rate", + points: [ + { t: 0, v: 1.25 }, + { t: 2, v: 2 }, + ], + } satisfies RateSpec; + input.composition.videos.push({ id: "clip", - tagName: "video", src: "clip.mp4", start: 0, end: 3, - duration: 3, mediaStart: 0, - playbackRate: 1, + playbackRate, loop: false, hasAudio: false, - volume: 1, - muted: false, - }, - ]; - const { runProbeStage } = await import("./probeStage.js"); - const input = makeProbeInput({}); - const playbackRate = { - target: "rate", - points: [ - { t: 0, v: 1 }, - { t: 3, v: 2 }, - ], - } satisfies RateSpec; - input.composition.videos.push({ - id: "clip", - src: "clip.mp4", - start: 0, - end: 3, - mediaStart: 0, - playbackRate, - loop: false, - hasAudio: false, - }); + }); - await runProbeStage(input); + await runProbeStage(input); - expect(input.composition.videos[0]?.playbackRate).toEqual(playbackRate); - }); + expect(input.composition.videos[0]?.playbackRate).toEqual(playbackRate); + }, + ); it.each([ { pageStart: 1, expected: 6 }, diff --git a/packages/producer/src/services/render/stages/probeStage.ts b/packages/producer/src/services/render/stages/probeStage.ts index 15ff02f3ba1..f1e89c44bc5 100644 --- a/packages/producer/src/services/render/stages/probeStage.ts +++ b/packages/producer/src/services/render/stages/probeStage.ts @@ -192,6 +192,10 @@ function authoredMediaStarts(html: string): Map { return starts; } +function isVideoAudio(audio: CompositionMetadata["audios"][number], videoId: string): boolean { + return audio.type === "video" && audio.id === `${videoId}-audio`; +} + /** * Take a start point the page changed by moving the collected one by the same amount, so the * in-point its scene added survives, and a video's sound track moves with its picture. @@ -207,9 +211,7 @@ function adoptRuntimeMediaStart( el.tagName === "video" ? [ ...composition.videos.filter((video) => video.id === el.id), - ...composition.audios.filter( - (audio) => audio.type === "video" && audio.id === `${el.id}-audio`, - ), + ...composition.audios.filter((audio) => isVideoAudio(audio, el.id)), ] : composition.audios.filter((audio) => audio.type !== "video" && audio.id === el.id); for (const entry of entries) entry.mediaStart += shift; @@ -603,11 +605,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise Date: Fri, 23 Oct 2026 22:11:43 -0400 Subject: [PATCH 3/3] fix(producer): retain scene trims for discovered speed ramps --- docs/reference/speed-ramps.mdx | 3 + packages/engine/src/index.ts | 1 + packages/engine/src/services/audioMixer.ts | 6 +- .../render/stages/extractVideosStage.ts | 3 +- .../services/render/stages/probeStage.test.ts | 131 +++++++++++++++--- .../src/services/render/stages/probeStage.ts | 50 ++++--- 6 files changed, 149 insertions(+), 45 deletions(-) diff --git a/docs/reference/speed-ramps.mdx b/docs/reference/speed-ramps.mdx index 13d4cd4902b..3cff09952ec 100644 --- a/docs/reference/speed-ramps.mdx +++ b/docs/reference/speed-ramps.mdx @@ -39,6 +39,9 @@ which the preview and the renderer both use, so the render shows the frames the preview showed. Set `data-duration` on the clip to fix how long it holds the timeline; without it the length follows from the source and the speed. +If a browser probe discovers a rate lane on an existing clip, export preserves +the scene's trim in both the ramp's position and the source media in-point. + Pitch is kept. The preview leaves the browser's default pitch correction on. The render time-stretches audio in short constant-speed slices (a quarter second, coarser on clips longer than a minute), which approximates the curve diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index e6fa67ca28b..455cd6d4fdd 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -262,6 +262,7 @@ export { parseAudioElements, parseMasterFxChain, processCompositionAudio, + videoAudioId, } from "./services/audioMixer.js"; export { audioFailureDiagnostics } from "./services/audioFailureDiagnostic.js"; export { cloneCaptureWarning, cloneCaptureWarnings } from "./services/captureWarning.js"; diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts index 2d6ebb92639..39433c16271 100644 --- a/packages/engine/src/services/audioMixer.ts +++ b/packages/engine/src/services/audioMixer.ts @@ -535,6 +535,10 @@ export function parseMasterFxChain(html: string): string | undefined { return readMasterFxChain(parseHTML(unwrapTemplate(html)).document); } +export function videoAudioId(videoId: string): string { + return `${videoId}-audio`; +} + export function parseAudioElements(html: string): AudioElement[] { const elements: AudioElement[] = []; const { document } = parseHTML(unwrapTemplate(html)); @@ -632,7 +636,7 @@ export function parseAudioElements(html: string): AudioElement[] { const src = resolveMediaElementSrc(el); if (!id || !src || isHidden(el) || memberGroupHidden(el)) continue; if (isKnownInactiveTimelineWindow(el, resolveStart(el))) continue; - elements.push(build(el, `${id}-audio`, src, "video", true)); + elements.push(build(el, videoAudioId(id), src, "video", true)); } return elements; diff --git a/packages/producer/src/services/render/stages/extractVideosStage.ts b/packages/producer/src/services/render/stages/extractVideosStage.ts index f774e3b7826..3230bfefc55 100644 --- a/packages/producer/src/services/render/stages/extractVideosStage.ts +++ b/packages/producer/src/services/render/stages/extractVideosStage.ts @@ -50,6 +50,7 @@ import { resolveProjectRelativeSrc, runVideoExtractionWithRetry, safeVideoExtractionSourceIdentity, + videoAudioId, } from "@hyperframes/engine"; import { collectVideoMetadataHints, @@ -573,7 +574,7 @@ export function appendAutoDetectedVideoAudio( const video = composition.videos.find((v) => v.id === ext.videoId); if (!video || !video.hasAudio || video.hidden || existingAudioSrcs.has(video.src)) continue; composition.audios.push({ - id: `${video.id}-audio`, + id: videoAudioId(video.id), src: video.src, start: video.start, end: video.end, diff --git a/packages/producer/src/services/render/stages/probeStage.test.ts b/packages/producer/src/services/render/stages/probeStage.test.ts index fd6611eaa4f..90492b9f252 100644 --- a/packages/producer/src/services/render/stages/probeStage.test.ts +++ b/packages/producer/src/services/render/stages/probeStage.test.ts @@ -3,6 +3,13 @@ import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { RateSpec } from "@hyperframes/core"; +import { parseAudioElements, videoAudioId } from "../../../../../engine/src/services/audioMixer.js"; +import { + parseVideoElements, + parseImageElements, +} from "../../../../../engine/src/services/videoFrameExtractor.js"; +import { resolveReferencedStart } from "../../../../../engine/src/services/referenceResolver.js"; +import { collectRenderMedia } from "../../renderMediaCollector.js"; const workDir = mkdtempSync(join(tmpdir(), "hf-stage-test-")); afterAll(() => rmSync(workDir, { recursive: true, force: true })); @@ -100,6 +107,11 @@ mock.module("../../assetMediaType.js", () => ({ })); mock.module("@hyperframes/engine", () => ({ + parseAudioElements, + parseVideoElements, + parseImageElements, + resolveReferencedStart, + videoAudioId, createCaptureSession: async ( _url: string, _dir: string, @@ -479,49 +491,124 @@ describe("runProbeStage — forceScreenshot threading", () => { }, ); - it.each(runtimeRates)( - "preserves a collected speed ramp during browser rate %j reconciliation", - async (browserRate) => { + it.each([ + { tagName: "video", pageStart: 1, expected: 3 }, + { tagName: "video", pageStart: 3, expected: 5 }, + { tagName: "audio", pageStart: 1, expected: 3 }, + { tagName: "audio", pageStart: 3, expected: 5 }, + ])( + "applies a scene trim when adopting a runtime $tagName lane (page start $pageStart)", + async ({ tagName, pageStart, expected }) => { resetRetryMocks(); + const playbackRate = { + target: "rate", + points: [ + { t: 0, v: 1 }, + { t: 2, v: 1 }, + { t: 4, v: 2 }, + ], + } satisfies RateSpec; browserMediaResults = [ { id: "clip", - tagName: "video", + tagName, src: "clip.mp4", start: 0, - end: 3, - duration: 3, - mediaStart: 0, - playbackRate: browserRate, + end: 5, + duration: 5, + mediaStart: pageStart, + playbackRate, loop: false, - hasAudio: false, + hasAudio: true, volume: 1, muted: false, }, ]; const { runProbeStage } = await import("./probeStage.js"); const input = makeProbeInput({}); - const playbackRate = { + input.compiled.html = `
+ <${tagName} id="clip" src="clip.mp4" data-start="0" data-duration="5" data-playback-start="1" data-playback-rate="2" data-has-audio="true"> +
`; + const collected = collectRenderMedia(input.compiled.html); + input.composition.videos = collected.videos; + input.composition.audios = collected.audios; + const before = tagName === "video" ? collected.videos : collected.audios; + expect(before[0]).toMatchObject({ start: 5, end: 8, mediaStart: 5, playbackRate: 2 }); + + await runProbeStage(input); + + const shiftedRate = { target: "rate", points: [ - { t: 0, v: 1.25 }, + { t: 0, v: 1 }, { t: 2, v: 2 }, ], - } satisfies RateSpec; - input.composition.videos.push({ - id: "clip", - src: "clip.mp4", - start: 0, - end: 3, - mediaStart: 0, - playbackRate, - loop: false, - hasAudio: false, + }; + const media = tagName === "video" ? input.composition.videos : input.composition.audios; + expect(media[0]).toMatchObject({ + start: 5, + end: 8, + mediaStart: expected, + playbackRate: shiftedRate, }); + if (tagName === "video") { + expect(input.composition.audios[0]).toMatchObject({ + mediaStart: expected, + playbackRate: shiftedRate, + }); + } + }, + ); + + it.each( + runtimeRates.flatMap((playbackRate) => [ + { tagName: "video", playbackRate }, + { tagName: "audio", playbackRate }, + ]), + )( + "preserves a collected $tagName speed ramp during reconciliation (browser rate: $playbackRate)", + async ({ tagName, playbackRate }) => { + resetRetryMocks(); + browserMediaResults = [ + { + id: "clip", + tagName, + src: "clip.mp4", + start: 0, + end: 5, + duration: 5, + mediaStart: 1, + playbackRate, + loop: false, + hasAudio: true, + volume: 1, + muted: false, + }, + ]; + const { runProbeStage } = await import("./probeStage.js"); + const input = makeProbeInput({}); + const authoredRate = { + target: "rate", + points: [ + { t: 0, v: 1 }, + { t: 2, v: 1 }, + { t: 4, v: 2 }, + ], + } satisfies RateSpec; + input.compiled.html = `
+ <${tagName} id="clip" src="clip.mp4" data-start="0" data-duration="5" data-playback-start="1" data-has-audio="true" data-automation='${JSON.stringify({ version: 1, lanes: [authoredRate] })}'> +
`; + const collected = collectRenderMedia(input.compiled.html); + input.composition.videos = collected.videos; + input.composition.audios = collected.audios; + const before = structuredClone({ videos: collected.videos, audios: collected.audios }); + const media = tagName === "video" ? collected.videos : collected.audios; + expect(media[0]).toMatchObject({ mediaStart: 3 }); await runProbeStage(input); - expect(input.composition.videos[0]?.playbackRate).toEqual(playbackRate); + expect(input.composition.videos).toEqual(before.videos); + expect(input.composition.audios).toEqual(before.audios); }, ); diff --git a/packages/producer/src/services/render/stages/probeStage.ts b/packages/producer/src/services/render/stages/probeStage.ts index f1e89c44bc5..bd9cdb0faf5 100644 --- a/packages/producer/src/services/render/stages/probeStage.ts +++ b/packages/producer/src/services/render/stages/probeStage.ts @@ -43,8 +43,17 @@ import { initializeSession, isTransientBrowserError, probeBeginFrameLiveness, + videoAudioId, } from "@hyperframes/engine"; -import { MEDIA_RENDER_ID_ATTR, fpsToNumber, readMediaStart } from "@hyperframes/core"; +import { + MEDIA_RENDER_ID_ATTR, + fpsToNumber, + normalizeRateSpec, + readMediaStart, + shiftRateLane, + sourceTimeAt, + timeAtSourceTime, +} from "@hyperframes/core"; import { extractMediaSrcMutations } from "@hyperframes/parsers"; import type { BrowserMediaElement, CompiledComposition } from "../../htmlCompiler.js"; import { @@ -193,20 +202,18 @@ function authoredMediaStarts(html: string): Map { } function isVideoAudio(audio: CompositionMetadata["audios"][number], videoId: string): boolean { - return audio.type === "video" && audio.id === `${videoId}-audio`; + return audio.type === "video" && audio.id === videoAudioId(videoId); } /** - * Take a start point the page changed by moving the collected one by the same amount, so the - * in-point its scene added survives, and a video's sound track moves with its picture. + * Adopt browser timing without losing the scene trim the collector applied to picture and sound. */ -function adoptRuntimeMediaStart( +function adoptRuntimeMediaTiming( composition: Pick, el: BrowserMediaElement, authored: number, ): void { const shift = el.mediaStart - authored; - if (el.mediaStart <= 0 || Math.abs(shift) <= BROWSER_MEDIA_EPSILON) return; const entries = el.tagName === "video" ? [ @@ -214,7 +221,20 @@ function adoptRuntimeMediaStart( ...composition.audios.filter((audio) => isVideoAudio(audio, el.id)), ] : composition.audios.filter((audio) => audio.type !== "video" && audio.id === el.id); - for (const entry of entries) entry.mediaStart += shift; + for (const entry of entries) { + // Collected lanes already carry their scene trim. Only newly discovered lanes need it. + if (typeof entry.playbackRate !== "object") { + if (typeof el.playbackRate === "object") { + const consumed = Math.max(0, entry.mediaStart - authored); + const head = timeAtSourceTime(normalizeRateSpec(entry.playbackRate), consumed); + entry.mediaStart += sourceTimeAt(el.playbackRate, head) - consumed; + entry.playbackRate = shiftRateLane(el.playbackRate, head); + } else { + entry.playbackRate = el.playbackRate; + } + } + if (el.mediaStart > 0 && Math.abs(shift) > BROWSER_MEDIA_EPSILON) entry.mediaStart += shift; + } } function reconcileBrowserMediaEnd( @@ -596,20 +616,11 @@ export async function runProbeStage(input: ProbeStageInput): Promise 0 && Math.abs((existing.volume ?? 1) - el.volume) > BROWSER_MEDIA_EPSILON