diff --git a/docs/reference/speed-ramps.mdx b/docs/reference/speed-ramps.mdx index 13d4cd4902b..3cff09952ec 100644 --- a/docs/reference/speed-ramps.mdx +++ b/docs/reference/speed-ramps.mdx @@ -39,6 +39,9 @@ which the preview and the renderer both use, so the render shows the frames the preview showed. Set `data-duration` on the clip to fix how long it holds the timeline; without it the length follows from the source and the speed. +If a browser probe discovers a rate lane on an existing clip, export preserves +the scene's trim in both the ramp's position and the source media in-point. + Pitch is kept. The preview leaves the browser's default pitch correction on. The render time-stretches audio in short constant-speed slices (a quarter second, coarser on clips longer than a minute), which approximates the curve diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index e6fa67ca28b..455cd6d4fdd 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -262,6 +262,7 @@ export { parseAudioElements, parseMasterFxChain, processCompositionAudio, + videoAudioId, } from "./services/audioMixer.js"; export { audioFailureDiagnostics } from "./services/audioFailureDiagnostic.js"; export { cloneCaptureWarning, cloneCaptureWarnings } from "./services/captureWarning.js"; diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts index 2d6ebb92639..39433c16271 100644 --- a/packages/engine/src/services/audioMixer.ts +++ b/packages/engine/src/services/audioMixer.ts @@ -535,6 +535,10 @@ export function parseMasterFxChain(html: string): string | undefined { return readMasterFxChain(parseHTML(unwrapTemplate(html)).document); } +export function videoAudioId(videoId: string): string { + return `${videoId}-audio`; +} + export function parseAudioElements(html: string): AudioElement[] { const elements: AudioElement[] = []; const { document } = parseHTML(unwrapTemplate(html)); @@ -632,7 +636,7 @@ export function parseAudioElements(html: string): AudioElement[] { const src = resolveMediaElementSrc(el); if (!id || !src || isHidden(el) || memberGroupHidden(el)) continue; if (isKnownInactiveTimelineWindow(el, resolveStart(el))) continue; - elements.push(build(el, `${id}-audio`, src, "video", true)); + elements.push(build(el, videoAudioId(id), src, "video", true)); } return elements; diff --git a/packages/producer/src/services/htmlCompiler.test.ts b/packages/producer/src/services/htmlCompiler.test.ts index 85050835b85..d90df840f49 100644 --- a/packages/producer/src/services/htmlCompiler.test.ts +++ b/packages/producer/src/services/htmlCompiler.test.ts @@ -36,6 +36,7 @@ describe("discoverMediaFromBrowser", () => { currentSrcById: Record, serializeCallback = false, intrinsicDurationById: Record = {}, + defaultPlaybackRateById: Record = {}, ) { const { document } = parseHTML(html); for (const [id, currentSrc] of Object.entries(currentSrcById)) { @@ -46,6 +47,13 @@ describe("discoverMediaFromBrowser", () => { const element = document.getElementById(id); if (element) Object.defineProperty(element, "duration", { value: duration }); } + for (const [id, defaultPlaybackRate] of Object.entries(defaultPlaybackRateById)) { + const element = document.getElementById(id); + if (element) { + Object.defineProperty(element, "defaultPlaybackRate", { value: defaultPlaybackRate }); + Object.defineProperty(element, "playbackRate", { value: defaultPlaybackRate * 2 }); + } + } const previousDocument = Reflect.get(globalThis, "document"); Reflect.set(globalThis, "document", document); try { @@ -129,6 +137,58 @@ describe("discoverMediaFromBrowser", () => { expect(media[0]).toMatchObject({ start: 0, end: 2, duration: 2, mediaStart: 1 }); }); + it("reads authored rates and native defaults after the browser callback is serialized", async () => { + const media = await discover( + ` + + + + `, + {}, + true, + {}, + { authored: 0.5, native: 0.5, invalid: 1.25 }, + ); + + expect(media.map((element) => element.playbackRate)).toEqual([1.5, 0.5, 1.25, 10, 1]); + }); + + it.each(["video", "audio"])( + "discovers a %s rate lane after the browser callback is serialized", + async (tag) => { + const lane = { + target: "rate", + points: [ + { t: 0, v: 1 }, + { t: 1, v: 2 }, + ], + }; + const automation = JSON.stringify({ version: 1, lanes: [lane] }); + const media = await discover( + `<${tag} id="clip" src="clip.mp4" data-start="0" data-duration="1" data-playback-rate="3" data-automation='${automation}'>`, + {}, + true, + {}, + { clip: 0.5 }, + ); + + expect(media[0]?.playbackRate).toEqual(lane); + }, + ); + + it("uses scalar rates when automation is invalid or has no rate lane", async () => { + const media = await discover( + ` + `, + {}, + true, + {}, + { other: 0.5 }, + ); + + expect(media.map((element) => element.playbackRate)).toEqual([1.5, 0.5]); + }); + it("reports colliding empty-src videos by render id, not author id", async () => { const media = await discover( `` + diff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.ts index a810dc4d855..319a13cfd8b 100644 --- a/packages/producer/src/services/htmlCompiler.ts +++ b/packages/producer/src/services/htmlCompiler.ts @@ -36,6 +36,8 @@ import { type UnresolvedElement, } from "@hyperframes/core"; import { MAX_AUDIO_GAIN } from "@hyperframes/core/audio-gain"; +import { readPlaybackRate } from "@hyperframes/parsers/media-duration"; +import { resolveRateSpec } from "@hyperframes/core/speed-ramp"; import { gsapCdnDist } from "@hyperframes/core/gsap-cdn"; import { assignBundledRuntimeCompositionIds, @@ -2235,6 +2237,7 @@ export interface BrowserMediaElement { /** True when compilation inferred duration from the fallback source. */ durationInferred: boolean; mediaStart: number; + playbackRate: RateSpec; loop: boolean; hasAudio: boolean; volume: number; @@ -2260,6 +2263,9 @@ export async function discoverMediaFromBrowser(page: Page): Promise ({ + ({ + endRaw, + durationRaw, + intrinsicDuration, + playbackStartRaw, + mediaStartRaw, + playbackRateRaw, + automationRaw, + defaultPlaybackRate, + ...element + }) => ({ ...element, end: parseStrictFiniteTimingNumber(endRaw) ?? 0, duration: @@ -2353,6 +2378,13 @@ export async function discoverMediaFromBrowser(page: Page): Promise (name === "data-playback-rate" ? playbackRateRaw : null), + defaultPlaybackRate, + ), + ), }), ); } diff --git a/packages/producer/src/services/render/stages/extractVideosStage.test.ts b/packages/producer/src/services/render/stages/extractVideosStage.test.ts index dc4d91598d2..97de3e87835 100644 --- a/packages/producer/src/services/render/stages/extractVideosStage.test.ts +++ b/packages/producer/src/services/render/stages/extractVideosStage.test.ts @@ -105,6 +105,34 @@ describe("appendAutoDetectedVideoAudio", () => { }); }); + it.each([ + 1.5, + { + target: "rate", + points: [ + { t: 0, v: 1 }, + { t: 3, v: 2 }, + ], + }, + ] satisfies NonNullable[])( + "keeps the video's rate and trim in its detected audio (%j)", + (playbackRate) => { + const composition: Parameters[0] = { + videos: [makeVideo({ start: 2, end: 5, mediaStart: 0.5, playbackRate })], + audios: [], + }; + + appendAutoDetectedVideoAudio(composition, [makeExtracted("v1", true)]); + + expect(composition.audios[0]).toMatchObject({ + start: 2, + end: 5, + mediaStart: 0.5, + playbackRate, + }); + }, + ); + it("skips a muted video even when the source file has audio", () => { const composition = { videos: [makeVideo({ hasAudio: false })], diff --git a/packages/producer/src/services/render/stages/extractVideosStage.ts b/packages/producer/src/services/render/stages/extractVideosStage.ts index 5208f5fac57..3230bfefc55 100644 --- a/packages/producer/src/services/render/stages/extractVideosStage.ts +++ b/packages/producer/src/services/render/stages/extractVideosStage.ts @@ -50,6 +50,7 @@ import { resolveProjectRelativeSrc, runVideoExtractionWithRetry, safeVideoExtractionSourceIdentity, + videoAudioId, } from "@hyperframes/engine"; import { collectVideoMetadataHints, @@ -573,11 +574,12 @@ export function appendAutoDetectedVideoAudio( const video = composition.videos.find((v) => v.id === ext.videoId); if (!video || !video.hasAudio || video.hidden || existingAudioSrcs.has(video.src)) continue; composition.audios.push({ - id: `${video.id}-audio`, + id: videoAudioId(video.id), src: video.src, start: video.start, end: video.end, mediaStart: video.mediaStart, + playbackRate: video.playbackRate, layer: 0, volume: 1.0, type: "video", diff --git a/packages/producer/src/services/render/stages/probeStage.test.ts b/packages/producer/src/services/render/stages/probeStage.test.ts index 45bdddb1346..90492b9f252 100644 --- a/packages/producer/src/services/render/stages/probeStage.test.ts +++ b/packages/producer/src/services/render/stages/probeStage.test.ts @@ -2,6 +2,14 @@ import { afterAll, describe, expect, it, mock } from "bun:test"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import type { RateSpec } from "@hyperframes/core"; +import { parseAudioElements, videoAudioId } from "../../../../../engine/src/services/audioMixer.js"; +import { + parseVideoElements, + parseImageElements, +} from "../../../../../engine/src/services/videoFrameExtractor.js"; +import { resolveReferencedStart } from "../../../../../engine/src/services/referenceResolver.js"; +import { collectRenderMedia } from "../../renderMediaCollector.js"; const workDir = mkdtempSync(join(tmpdir(), "hf-stage-test-")); afterAll(() => rmSync(workDir, { recursive: true, force: true })); @@ -99,6 +107,11 @@ mock.module("../../assetMediaType.js", () => ({ })); mock.module("@hyperframes/engine", () => ({ + parseAudioElements, + parseVideoElements, + parseImageElements, + resolveReferencedStart, + videoAudioId, createCaptureSession: async ( _url: string, _dir: string, @@ -413,6 +426,192 @@ describe("runProbeStage — forceScreenshot threading", () => { }); // Scene in-point 5 and clip trim 1: the collector starts picture and sound at source 6. + const runtimeRates = [ + 1.5, + { + target: "rate", + points: [ + { t: 0, v: 1 }, + { t: 3, v: 2 }, + ], + } satisfies RateSpec, + ]; + it.each( + runtimeRates.flatMap((playbackRate) => [ + { tagName: "video", existing: false, playbackRate }, + { tagName: "audio", existing: false, playbackRate }, + { tagName: "video", existing: true, playbackRate }, + { tagName: "audio", existing: true, playbackRate }, + ]), + )( + "retains a runtime $tagName rate (existing: $existing, rate: $playbackRate)", + async ({ tagName, existing, playbackRate }) => { + resetRetryMocks(); + browserMediaResults = [ + { + id: "clip", + tagName, + src: "clip.mp4", + start: 0, + end: 3, + duration: 3, + mediaStart: 0.5, + playbackRate, + loop: false, + hasAudio: true, + volume: 1, + muted: false, + }, + ]; + const { runProbeStage } = await import("./probeStage.js"); + const input = makeProbeInput({}); + const clip = { + id: "clip", + src: "clip.mp4", + start: 0, + end: 3, + mediaStart: 0.5, + playbackRate: 1, + }; + if (existing && tagName === "video") { + input.composition.videos.push({ ...clip, loop: false, hasAudio: true }); + input.composition.audios.push({ ...clip, id: "clip-audio", layer: 0, type: "video" }); + } + if (existing && tagName === "audio") { + input.composition.audios.push({ ...clip, layer: 0, type: "audio" }); + } + + await runProbeStage(input); + + const media = tagName === "video" ? input.composition.videos : input.composition.audios; + expect(media[0]).toMatchObject({ playbackRate, mediaStart: 0.5 }); + if (existing && tagName === "video") { + expect(input.composition.audios[0]).toMatchObject({ playbackRate, mediaStart: 0.5 }); + } + }, + ); + + it.each([ + { tagName: "video", pageStart: 1, expected: 3 }, + { tagName: "video", pageStart: 3, expected: 5 }, + { tagName: "audio", pageStart: 1, expected: 3 }, + { tagName: "audio", pageStart: 3, expected: 5 }, + ])( + "applies a scene trim when adopting a runtime $tagName lane (page start $pageStart)", + async ({ tagName, pageStart, expected }) => { + resetRetryMocks(); + const playbackRate = { + target: "rate", + points: [ + { t: 0, v: 1 }, + { t: 2, v: 1 }, + { t: 4, v: 2 }, + ], + } satisfies RateSpec; + browserMediaResults = [ + { + id: "clip", + tagName, + src: "clip.mp4", + start: 0, + end: 5, + duration: 5, + mediaStart: pageStart, + playbackRate, + loop: false, + hasAudio: true, + volume: 1, + muted: false, + }, + ]; + const { runProbeStage } = await import("./probeStage.js"); + const input = makeProbeInput({}); + input.compiled.html = `
+ <${tagName} id="clip" src="clip.mp4" data-start="0" data-duration="5" data-playback-start="1" data-playback-rate="2" data-has-audio="true"> +
`; + const collected = collectRenderMedia(input.compiled.html); + input.composition.videos = collected.videos; + input.composition.audios = collected.audios; + const before = tagName === "video" ? collected.videos : collected.audios; + expect(before[0]).toMatchObject({ start: 5, end: 8, mediaStart: 5, playbackRate: 2 }); + + await runProbeStage(input); + + const shiftedRate = { + target: "rate", + points: [ + { t: 0, v: 1 }, + { t: 2, v: 2 }, + ], + }; + const media = tagName === "video" ? input.composition.videos : input.composition.audios; + expect(media[0]).toMatchObject({ + start: 5, + end: 8, + mediaStart: expected, + playbackRate: shiftedRate, + }); + if (tagName === "video") { + expect(input.composition.audios[0]).toMatchObject({ + mediaStart: expected, + playbackRate: shiftedRate, + }); + } + }, + ); + + it.each( + runtimeRates.flatMap((playbackRate) => [ + { tagName: "video", playbackRate }, + { tagName: "audio", playbackRate }, + ]), + )( + "preserves a collected $tagName speed ramp during reconciliation (browser rate: $playbackRate)", + async ({ tagName, playbackRate }) => { + resetRetryMocks(); + browserMediaResults = [ + { + id: "clip", + tagName, + src: "clip.mp4", + start: 0, + end: 5, + duration: 5, + mediaStart: 1, + playbackRate, + loop: false, + hasAudio: true, + volume: 1, + muted: false, + }, + ]; + const { runProbeStage } = await import("./probeStage.js"); + const input = makeProbeInput({}); + const authoredRate = { + target: "rate", + points: [ + { t: 0, v: 1 }, + { t: 2, v: 1 }, + { t: 4, v: 2 }, + ], + } satisfies RateSpec; + input.compiled.html = `
+ <${tagName} id="clip" src="clip.mp4" data-start="0" data-duration="5" data-playback-start="1" data-has-audio="true" data-automation='${JSON.stringify({ version: 1, lanes: [authoredRate] })}'> +
`; + const collected = collectRenderMedia(input.compiled.html); + input.composition.videos = collected.videos; + input.composition.audios = collected.audios; + const before = structuredClone({ videos: collected.videos, audios: collected.audios }); + const media = tagName === "video" ? collected.videos : collected.audios; + expect(media[0]).toMatchObject({ mediaStart: 3 }); + + await runProbeStage(input); + + expect(input.composition.videos).toEqual(before.videos); + expect(input.composition.audios).toEqual(before.audios); + }, + ); + it.each([ { pageStart: 1, expected: 6 }, { pageStart: 3, expected: 8 }, diff --git a/packages/producer/src/services/render/stages/probeStage.ts b/packages/producer/src/services/render/stages/probeStage.ts index 5ecff9bd416..bd9cdb0faf5 100644 --- a/packages/producer/src/services/render/stages/probeStage.ts +++ b/packages/producer/src/services/render/stages/probeStage.ts @@ -43,8 +43,17 @@ import { initializeSession, isTransientBrowserError, probeBeginFrameLiveness, + videoAudioId, } from "@hyperframes/engine"; -import { MEDIA_RENDER_ID_ATTR, fpsToNumber, readMediaStart } from "@hyperframes/core"; +import { + MEDIA_RENDER_ID_ATTR, + fpsToNumber, + normalizeRateSpec, + readMediaStart, + shiftRateLane, + sourceTimeAt, + timeAtSourceTime, +} from "@hyperframes/core"; import { extractMediaSrcMutations } from "@hyperframes/parsers"; import type { BrowserMediaElement, CompiledComposition } from "../../htmlCompiler.js"; import { @@ -192,27 +201,40 @@ function authoredMediaStarts(html: string): Map { return starts; } +function isVideoAudio(audio: CompositionMetadata["audios"][number], videoId: string): boolean { + return audio.type === "video" && audio.id === videoAudioId(videoId); +} + /** - * Take a start point the page changed by moving the collected one by the same amount, so the - * in-point its scene added survives, and a video's sound track moves with its picture. + * Adopt browser timing without losing the scene trim the collector applied to picture and sound. */ -function adoptRuntimeMediaStart( +function adoptRuntimeMediaTiming( composition: Pick, el: BrowserMediaElement, authored: number, ): void { const shift = el.mediaStart - authored; - if (el.mediaStart <= 0 || Math.abs(shift) <= BROWSER_MEDIA_EPSILON) return; const entries = el.tagName === "video" ? [ ...composition.videos.filter((video) => video.id === el.id), - ...composition.audios.filter( - (audio) => audio.type === "video" && audio.id === `${el.id}-audio`, - ), + ...composition.audios.filter((audio) => isVideoAudio(audio, el.id)), ] : composition.audios.filter((audio) => audio.type !== "video" && audio.id === el.id); - for (const entry of entries) entry.mediaStart += shift; + for (const entry of entries) { + // Collected lanes already carry their scene trim. Only newly discovered lanes need it. + if (typeof entry.playbackRate !== "object") { + if (typeof el.playbackRate === "object") { + const consumed = Math.max(0, entry.mediaStart - authored); + const head = timeAtSourceTime(normalizeRateSpec(entry.playbackRate), consumed); + entry.mediaStart += sourceTimeAt(el.playbackRate, head) - consumed; + entry.playbackRate = shiftRateLane(el.playbackRate, head); + } else { + entry.playbackRate = el.playbackRate; + } + } + if (el.mediaStart > 0 && Math.abs(shift) > BROWSER_MEDIA_EPSILON) entry.mediaStart += shift; + } } function reconcileBrowserMediaEnd( @@ -594,7 +616,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise