From ae5c571833f97e28324f51e727f1f9303eb62742 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 14 Oct 2026 12:31:20 -0700 Subject: [PATCH 1/5] fix(audio): preserve subprocess evidence through CLI diagnostics --- packages/cli/README.md | 14 ++ packages/cli/src/commands/render.test.ts | 61 +++++ packages/cli/src/commands/render.ts | 2 + packages/cli/src/ui/renderDiagnostic.test.ts | 233 ++++++++++++++++++ packages/cli/src/ui/renderDiagnostic.ts | 31 +++ packages/engine/src/index.ts | 3 + .../src/services/audioFailureDiagnostic.ts | 120 +++++++++ packages/engine/src/services/audioMixer.ts | 19 +- .../engine/src/services/audioMixer.types.ts | 13 + .../src/services/captureWarning.test.ts | 31 +++ .../engine/src/services/captureWarning.ts | 4 + packages/engine/src/types.ts | 2 + .../producer/src/services/distributed/plan.ts | 2 + .../services/render/encoderInterruption.ts | 10 +- .../services/render/stages/audioStage.test.ts | 21 +- .../src/services/render/stages/audioStage.ts | 1 + .../src/services/renderOrchestrator.ts | 2 + 17 files changed, 557 insertions(+), 12 deletions(-) create mode 100644 packages/cli/src/ui/renderDiagnostic.test.ts create mode 100644 packages/cli/src/ui/renderDiagnostic.ts create mode 100644 packages/engine/src/services/audioFailureDiagnostic.ts diff --git a/packages/cli/README.md b/packages/cli/README.md index c9eab88f07b..ba8f0be874f 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -75,6 +75,20 @@ A render stops when the process that started it exits. To keep one running after its launcher exits, set `HYPERFRAMES_RENDER_DETACHED=1` and start it in its own process group, e.g. `bash -c 'set -m; HYPERFRAMES_RENDER_DETACHED=1 npx hyperframes render -o out.mp4 < /dev/null > render.log 2>&1 &'`. +Piped, non-quiet renders emit a single `@hf-diagnostic ` JSON line when an audio +failure reaches the CLI error handler. Version 1 has `operation: "render"` and +at most 16 `audioFailures`: allowlisted stage, reason, owner, retryable, and +optional subprocess exit code, signal, termination reason and OS code. It omits +messages, stderr, source paths/URLs, element IDs and command arguments. Consumers +must ignore unknown versions and fields. This is local CLI output; uploading it +requires the caller's existing consent and privacy checks. Quiet mode (including +batch JSON mode) emits no diagnostic line. + +This covers audio preparation and mix failures, including the typed external +interruption path. Probe failures retain their existing stage/reason but do not +gain subprocess evidence here; browser/FX failures remain classified as internal. +Desktop ingestion and live report delivery are separate changes. + ### `publish` Upload a project directory and get a hosted URL that keeps working after the CLI exits. diff --git a/packages/cli/src/commands/render.test.ts b/packages/cli/src/commands/render.test.ts index 033cce32e9f..552f2f9f9c1 100644 --- a/packages/cli/src/commands/render.test.ts +++ b/packages/cli/src/commands/render.test.ts @@ -617,6 +617,67 @@ describe("renderLocal browser GPU config", () => { vi.restoreAllMocks(); }); + it.each([ + { quiet: false, interrupted: false, emits: true }, + { quiet: false, interrupted: true, emits: true }, + { quiet: true, interrupted: false, emits: false }, + ])("writes the failure diagnostic only for piped non-JSON output: %j", async (mode) => { + const failures = [ + { + stage: "prepare", + reason: "ffmpeg_failed", + owner: "system", + retryable: false, + subprocess: { exitCode: 7, terminationReason: "exit" }, + detail: "private-detail", + elementId: "private-id", + }, + ]; + producerState.executeImpl = async (job) => { + if (mode.interrupted) throw Object.assign(new Error("failed"), { audioFailures: failures }); + job.warnings = [ + { + code: "audio_processing_failed", + message: "private-warning", + details: { audioFailures: failures }, + }, + ]; + throw new Error("failed"); + }; + const originalIsTTY = process.stdout.isTTY; + Object.defineProperty(process.stdout, "isTTY", { value: false, configurable: true }); + let output = ""; + vi.spyOn(process.stdout, "write").mockImplementation((chunk) => { + output += String(chunk); + return true; + }); + try { + await expect( + renderLocal("/tmp/project", "/tmp/out.mp4", { + fps: { num: 30, den: 1 }, + quality: "standard", + format: "mp4", + gpu: false, + browserGpuMode: "software", + hdrMode: "auto", + quiet: mode.quiet, + throwOnError: true, + }), + ).rejects.toThrow("failed"); + const lines = output.split("\n").filter((line) => line.startsWith("@hf-diagnostic ")); + expect(lines).toHaveLength(mode.emits ? 1 : 0); + if (mode.emits) { + expect( + JSON.parse((lines[0] ?? "").slice("@hf-diagnostic ".length)).audioFailures[0].subprocess + .exitCode, + ).toBe(7); + expect(lines[0] ?? "").not.toMatch(/private-/); + } + } finally { + Object.defineProperty(process.stdout, "isTTY", { value: originalIsTTY, configurable: true }); + } + }); + it("logs only warnings and errors from the producer under --quiet", async () => { const options = { fps: { num: 30, den: 1 }, diff --git a/packages/cli/src/commands/render.ts b/packages/cli/src/commands/render.ts index 36d508644dc..ee31d8bc513 100644 --- a/packages/cli/src/commands/render.ts +++ b/packages/cli/src/commands/render.ts @@ -74,6 +74,7 @@ import { errorBox, } from "../ui/format.js"; import { warnIfWebmAlphaDropped } from "../utils/webmAlphaCheck.js"; +import { renderFailureDiagnostic } from "../ui/renderDiagnostic.js"; import { renderProgress, renderMachineProgress } from "../ui/progress.js"; import { trackRenderComplete, @@ -1705,6 +1706,7 @@ function handleRenderError( job?: RenderJob, ): never { const message = normalizeErrorMessage(error); + renderFailureDiagnostic(job, error, options.quiet); trackRenderError({ fps: fpsToNumber(options.fps), quality: options.quality, diff --git a/packages/cli/src/ui/renderDiagnostic.test.ts b/packages/cli/src/ui/renderDiagnostic.test.ts new file mode 100644 index 00000000000..4278f484661 --- /dev/null +++ b/packages/cli/src/ui/renderDiagnostic.test.ts @@ -0,0 +1,233 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { processCompositionAudio, type AudioFailureDiagnostic } from "@hyperframes/engine"; +import { applyDistributedAudioWarningPolicy } from "../../../producer/src/services/distributed/plan.js"; +import { buildSyntheticRenderJob } from "../../../producer/src/services/distributed/shared.js"; +import { createRenderJob, executeRenderJob, type RenderJob } from "@hyperframes/producer"; +import { renderMachineDiagnostic } from "./renderDiagnostic.js"; + +const roots: string[] = []; +const originalIsTTY = process.stdout.isTTY; +afterEach(() => { + vi.restoreAllMocks(); + Object.defineProperty(process.stdout, "isTTY", { value: originalIsTTY, configurable: true }); + vi.unstubAllEnvs(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function capture(values: readonly unknown[], terminal = false): string { + Object.defineProperty(process.stdout, "isTTY", { value: terminal, configurable: true }); + let output = ""; + vi.spyOn(process.stdout, "write").mockImplementation((chunk) => { + output += String(chunk); + return true; + }); + renderMachineDiagnostic(values); + return output; +} + +function captureJob(job: Pick): string { + return capture(job.warnings.flatMap((warning) => warning.details?.audioFailures ?? [])); +} + +const PRIVATE = + "/private/customer/project?token=secret-token element-PRIVATE https://private.test/prompt"; +const failure = { + stage: "prepare", + reason: "ffmpeg_failed", + owner: "system", + retryable: false, + detail: PRIVATE, + elementId: PRIVATE, + subprocess: { exitCode: 7, terminationReason: "exit", stderr: PRIVATE, args: [PRIVATE] }, +}; + +describe("render diagnostic wire", () => { + it("re-projects every field and bounds the record count at the final writer", () => { + const line = capture(Array.from({ length: 100 }, () => failure)); + const parsed = JSON.parse(line.slice("@hf-diagnostic ".length)); + expect(parsed.version).toBe(1); + expect(parsed.operation).toBe("render"); + expect(parsed.audioFailures).toHaveLength(16); + expect(parsed.audioFailures[0]).toEqual({ + stage: "prepare", + reason: "ffmpeg_failed", + owner: "system", + retryable: false, + subprocess: { exitCode: 7, terminationReason: "exit" }, + }); + expect(line).not.toContain(PRIVATE); + expect(line.trim().split("\n")).toHaveLength(1); + }); + + it.each(["stage", "reason", "owner", "retryable"])( + "rejects private text substituted for %s", + (key) => { + expect(capture([{ ...failure, [key]: PRIVATE }])).toBe(""); + }, + ); + + it("drops non-allowlisted subprocess strings and unbounded exit codes", () => { + const line = capture([ + { + ...failure, + subprocess: { + exitCode: Infinity, + signal: PRIVATE, + terminationReason: PRIVATE, + code: PRIVATE, + message: PRIVATE, + error: { code: PRIVATE, message: PRIVATE }, + }, + }, + ]); + expect(JSON.parse(line.slice("@hf-diagnostic ".length)).audioFailures[0].subprocess).toEqual( + {}, + ); + expect(line).not.toContain(PRIVATE); + }); + + it("stays silent for a terminal or a container attached to a terminal", () => { + expect(capture([failure], true)).toBe(""); + vi.unstubAllEnvs(); + vi.stubEnv("HYPERFRAMES_STDOUT_IS_TTY", "1"); + expect(capture([failure])).toBe(""); + }); +}); + +const cases: Array<{ name: string; script?: string; expected: AudioFailureDiagnostic }> = [ + { + name: "missing executable", + expected: { + stage: "prepare", + reason: "ffmpeg_unavailable", + owner: "system", + retryable: true, + subprocess: { terminationReason: "spawn_error", code: "ENOENT" }, + }, + }, + { + name: "ordinary exit", + script: 'process.stderr.write("private-customer-stderr"); process.exit(7);', + expected: { + stage: "prepare", + reason: "ffmpeg_failed", + owner: "system", + retryable: false, + subprocess: { exitCode: 7, terminationReason: "exit" }, + }, + }, + { + name: "deadline", + script: "setInterval(() => {}, 1000);", + expected: { + stage: "prepare", + reason: "ffmpeg_timeout", + owner: "system", + retryable: true, + subprocess: { signal: "SIGTERM", terminationReason: "deadline" }, + }, + }, + { + name: "group submix", + script: + 'const fs = require("node:fs"); if (process.argv.includes("-filter_complex_script")) process.exit(9); fs.writeFileSync(process.argv.at(-1), "audio");', + expected: { + stage: "mix", + reason: "ffmpeg_failed", + owner: "system", + retryable: false, + subprocess: { exitCode: 9, terminationReason: "exit" }, + }, + }, +]; + +// The executable stand-ins use a POSIX shebang; field validation above runs on every OS. +for (const scenario of cases) { + it.skipIf(process.platform === "win32")( + `carries a real ${scenario.name} through the mixer, producer policy and final CLI line`, + async () => { + const root = mkdtempSync(join(tmpdir(), "hf-diagnostic-private-")); + roots.push(root); + const executable = join(root, "ffmpeg-private"); + if (scenario.script) { + writeFileSync(executable, `#!${process.execPath}\n${scenario.script}\n`); + chmodSync(executable, 0o755); + } + vi.stubEnv("HYPERFRAMES_FFMPEG_PATH", executable); + writeFileSync(join(root, "private.wav"), "RIFF0000WAVEfmt "); + const result = await processCompositionAudio( + [ + { + id: "private-element", + src: "private.wav", + type: "audio", + start: 0, + end: 1, + mediaStart: 0, + layer: 0, + volume: 1, + ...(scenario.name === "group submix" + ? { groupId: "private-group", groupVolume: 1 } + : {}), + }, + ], + root, + join(root, "work"), + join(root, "out.m4a"), + 1, + undefined, + { ffmpegProcessTimeout: scenario.name === "deadline" ? 200 : 5_000 }, + ); + expect(result.success).toBe(false); + const job = buildSyntheticRenderJob({ + fps: { num: 30, den: 1 }, + format: "mp4", + quality: "high", + hdrMode: "force-sdr", + entryFile: "index.html", + }); + expect(() => + applyDistributedAudioWarningPolicy(job, result.error ?? "failed", result.failures), + ).toThrow(); + const line = captureJob(job); + expect(JSON.parse(line.slice("@hf-diagnostic ".length)).audioFailures).toEqual([ + scenario.expected, + ]); + for (const secret of [ + root, + "private.wav", + "private-element", + "private-group", + "private-customer-stderr", + ]) + expect(line).not.toContain(secret); + }, + ); +} + +it("carries a local render audio failure through the real orchestration warning policy", async () => { + const root = mkdtempSync(join(tmpdir(), "hf-local-audio-diagnostic-")); + roots.push(root); + vi.stubEnv("HYPERFRAMES_FFMPEG_PATH", process.execPath); + writeFileSync(join(root, "private.wav"), "RIFF0000WAVEfmt "); + writeFileSync( + join(root, "index.html"), + `
`, + ); + const job = createRenderJob({ fps: 30, quality: "standard", hdrMode: "force-sdr" }); + await expect(executeRenderJob(job, root, join(root, "out.mp4"))).rejects.toThrow(); + const line = captureJob(job); + expect(JSON.parse(line.slice("@hf-diagnostic ".length)).audioFailures).toEqual([ + { + stage: "prepare", + reason: "ffmpeg_failed", + owner: "system", + retryable: false, + subprocess: { exitCode: 9, terminationReason: "exit" }, + }, + ]); + for (const secret of [root, "private.wav", "private-cue"]) expect(line).not.toContain(secret); +}); diff --git a/packages/cli/src/ui/renderDiagnostic.ts b/packages/cli/src/ui/renderDiagnostic.ts new file mode 100644 index 00000000000..cfc2b17b7cd --- /dev/null +++ b/packages/cli/src/ui/renderDiagnostic.ts @@ -0,0 +1,31 @@ +import type { RenderJob } from "@hyperframes/producer"; +import { audioFailureDiagnostics } from "@hyperframes/engine"; + +/** Versioned, content-free failure line for Desktop's piped CLI, separate from human error output. */ +export function renderMachineDiagnostic(values: readonly unknown[]): void { + if (process.stdout.isTTY || process.env.HYPERFRAMES_STDOUT_IS_TTY === "1") return; + const failures = audioFailureDiagnostics(values); + if (failures.length === 0) return; + process.stdout.write( + `@hf-diagnostic ${JSON.stringify({ version: 1, operation: "render", audioFailures: failures })}\n`, + ); +} + +export function renderFailureDiagnostic( + job: RenderJob | undefined, + error: unknown, + quiet: boolean, +): void { + if (quiet) return; + const interruptedFailures = + error !== null && + typeof error === "object" && + "audioFailures" in error && + Array.isArray(error.audioFailures) + ? error.audioFailures + : []; + renderMachineDiagnostic([ + ...interruptedFailures, + ...(job?.warnings.flatMap((warning) => warning.details?.audioFailures ?? []) ?? []), + ]); +} diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index cd2325ed3c0..e6fa67ca28b 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -263,12 +263,15 @@ export { parseMasterFxChain, processCompositionAudio, } from "./services/audioMixer.js"; +export { audioFailureDiagnostics } from "./services/audioFailureDiagnostic.js"; export { cloneCaptureWarning, cloneCaptureWarnings } from "./services/captureWarning.js"; export type { AudioElement, AudioFailureReason, AudioFailureStage, AudioProcessingFailure, + AudioFailureDiagnostic, + AudioSubprocessEvidence, AudioTrack, AudioVolumeKeyframe, MixResult, diff --git a/packages/engine/src/services/audioFailureDiagnostic.ts b/packages/engine/src/services/audioFailureDiagnostic.ts new file mode 100644 index 00000000000..d55bce3e6bb --- /dev/null +++ b/packages/engine/src/services/audioFailureDiagnostic.ts @@ -0,0 +1,120 @@ +import type { AudioFailureDiagnostic, AudioSubprocessEvidence } from "./audioMixer.types.js"; + +const STAGES = new Set([ + "source", + "download", + "probe", + "extract", + "prepare", + "mix", + "silence", + "cancelled", + "internal", +]); +const REASONS = new Set([ + "source_not_found", + "download_failed", + "probe_failed", + "invalid_media", + "ffmpeg_unsupported", + "ffmpeg_timeout", + "ffmpeg_unavailable", + "external_interruption", + "ffmpeg_failed", + "cancelled", + "internal", +]); +const TERMINATIONS = new Set(["exit", "abort", "deadline", "inactivity", "spawn_error"]); +const SIGNALS = new Set([ + "SIGABRT", + "SIGBUS", + "SIGFPE", + "SIGHUP", + "SIGILL", + "SIGINT", + "SIGKILL", + "SIGPIPE", + "SIGQUIT", + "SIGSEGV", + "SIGTERM", + "SIGTRAP", + "SIGXCPU", + "SIGXFSZ", +]); +const CODES = new Set([ + "EACCES", + "EAGAIN", + "EBUSY", + "EIO", + "EISDIR", + "EMFILE", + "ENAMETOOLONG", + "ENFILE", + "ENOENT", + "ENOEXEC", + "ENOMEM", + "ENOSPC", + "ENOTDIR", + "EPERM", + "ETIMEDOUT", +]); + +function isStage(value: unknown): value is AudioFailureDiagnostic["stage"] { + return typeof value === "string" && STAGES.has(value); +} + +function isReason(value: unknown): value is AudioFailureDiagnostic["reason"] { + return typeof value === "string" && REASONS.has(value); +} + +function isTermination( + value: unknown, +): value is NonNullable { + return typeof value === "string" && TERMINATIONS.has(value); +} + +function record(value: unknown): value is Record { + return value !== null && typeof value === "object"; +} + +export function audioSubprocessEvidence(value: unknown): AudioSubprocessEvidence { + if (!record(value)) return {}; + const evidence: AudioSubprocessEvidence = {}; + if ( + typeof value.exitCode === "number" && + Number.isInteger(value.exitCode) && + value.exitCode >= -2147483648 && + value.exitCode <= 4294967295 + ) + evidence.exitCode = value.exitCode; + if (typeof value.signal === "string" && SIGNALS.has(value.signal)) evidence.signal = value.signal; + if (isTermination(value.terminationReason)) evidence.terminationReason = value.terminationReason; + const error = record(value.error) ? value.error : value; + if (typeof error.code === "string" && CODES.has(error.code)) evidence.code = error.code; + return evidence; +} + +/** Re-project at the wire boundary: neither typed callers nor warning details are a privacy boundary. */ +export function audioFailureDiagnostics(values: readonly unknown[]): AudioFailureDiagnostic[] { + const result: AudioFailureDiagnostic[] = []; + for (const value of values.slice(0, 16)) { + if ( + !record(value) || + !isStage(value.stage) || + !isReason(value.reason) || + (value.owner !== "user" && value.owner !== "system") || + typeof value.retryable !== "boolean" + ) + continue; + result.push({ + stage: value.stage, + reason: value.reason, + owner: value.owner, + retryable: value.retryable, + ...(record(value.subprocess) + ? { subprocess: audioSubprocessEvidence(value.subprocess) } + : {}), + }); + } + return result; +} diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts index 585c3f182d0..2d6ebb92639 100644 --- a/packages/engine/src/services/audioMixer.ts +++ b/packages/engine/src/services/audioMixer.ts @@ -15,6 +15,7 @@ import { } from "../utils/urlDownloader.js"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; import { formatFfmpegError, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js"; +import { audioSubprocessEvidence } from "./audioFailureDiagnostic.js"; import { unwrapTemplate } from "../utils/htmlTemplate.js"; import { resolveMediaElementSrc, resolveProjectRelativeSrc } from "./videoFrameExtractor.js"; import { resolveReferencedStart, type RefResolverEl } from "./referenceResolver.js"; @@ -466,6 +467,7 @@ function ffmpegFailure( owner, retryable, elementId, + subprocess: audioSubprocessEvidence(result), detail: boundedDetail( result.error?.message ? `${formatFfmpegError(result.exitCode, stderr)}: ${result.error.message}` @@ -953,7 +955,10 @@ async function mixGroupMembers( totalDuration: number, signal?: AbortSignal, config?: Partial>, -): Promise<{ success: boolean; error?: string; degradedAutomation?: boolean }> { +): Promise< + | { success: true; degradedAutomation?: boolean } + | { success: false; error: string; failure: AudioProcessingFailure } +> { const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout; const outputDir = dirname(outputPath); if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true }); @@ -1046,9 +1051,10 @@ async function mixGroupMembers( } } - if (signal?.aborted) return { success: false, error: "Group sub-mix cancelled" }; - if (!result.success) - return { success: false, error: formatFfmpegError(result.exitCode, result.stderr) }; + if (signal?.aborted || !result.success) { + const failure = ffmpegFailure("mix", result); + return { success: false, error: failure.detail, failure }; + } return { success: true, degradedAutomation }; } @@ -1508,10 +1514,7 @@ export async function processCompositionAudio( ); if (!subMix.success) { failures.push({ - stage: "mix", - reason: "ffmpeg_failed", - owner: "system", - retryable: false, + ...subMix.failure, elementId: groupId, detail: boundedDetail( `Group sub-mix failed for group ${groupId}: ${subMix.error ?? "unknown"}`, diff --git a/packages/engine/src/services/audioMixer.types.ts b/packages/engine/src/services/audioMixer.types.ts index 6383e390253..bf2aca08f74 100644 --- a/packages/engine/src/services/audioMixer.types.ts +++ b/packages/engine/src/services/audioMixer.types.ts @@ -82,11 +82,24 @@ export type AudioFailureReason = | "cancelled" | "internal"; +export interface AudioSubprocessEvidence { + exitCode?: number; + signal?: string; + terminationReason?: "exit" | "abort" | "deadline" | "inactivity" | "spawn_error"; + code?: string; +} + +export type AudioFailureDiagnostic = Pick< + AudioProcessingFailure, + "stage" | "reason" | "owner" | "retryable" | "subprocess" +>; + export interface AudioProcessingFailure { stage: AudioFailureStage; reason: AudioFailureReason; owner: "user" | "system"; retryable: boolean; + subprocess?: AudioSubprocessEvidence; elementId?: string; /** Bounded diagnostic text; never includes the authored source URL/path. */ detail: string; diff --git a/packages/engine/src/services/captureWarning.test.ts b/packages/engine/src/services/captureWarning.test.ts index 2f5931b9e40..8a1b7ad1f9d 100644 --- a/packages/engine/src/services/captureWarning.test.ts +++ b/packages/engine/src/services/captureWarning.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import type { CaptureWarning } from "../types.js"; import { cloneCaptureWarning } from "./captureWarning.js"; describe("cloneCaptureWarning", () => { @@ -25,3 +26,33 @@ describe("cloneCaptureWarning", () => { }); }); }); + +it("isolates nested audio subprocess evidence in warning snapshots", () => { + const source: CaptureWarning = { + code: "audio_processing_failed", + message: "failed", + details: { + audioFailures: [ + { + stage: "prepare", + reason: "ffmpeg_failed", + owner: "system", + retryable: false, + subprocess: { exitCode: 7, terminationReason: "exit" }, + }, + ], + }, + }; + const copy = cloneCaptureWarning(source); + const evidence = copy.details?.audioFailures?.[0]?.subprocess; + if (!evidence) throw new Error("missing copied evidence"); + evidence.exitCode = 8; + copy.details?.audioFailures?.push({ + stage: "mix", + reason: "internal", + owner: "system", + retryable: false, + }); + expect(source.details?.audioFailures).toHaveLength(1); + expect(source.details?.audioFailures?.[0]?.subprocess?.exitCode).toBe(7); +}); diff --git a/packages/engine/src/services/captureWarning.ts b/packages/engine/src/services/captureWarning.ts index 844b3602f32..94e394e5903 100644 --- a/packages/engine/src/services/captureWarning.ts +++ b/packages/engine/src/services/captureWarning.ts @@ -7,6 +7,10 @@ export function cloneCaptureWarning(warning: T): T { details: warning.details ? { ...warning.details, + audioFailures: warning.details.audioFailures?.map((failure) => ({ + ...failure, + ...(failure.subprocess ? { subprocess: { ...failure.subprocess } } : {}), + })), sources: warning.details.sources ? [...warning.details.sources] : undefined, failureReasons: warning.details.failureReasons ? [...warning.details.failureReasons] diff --git a/packages/engine/src/types.ts b/packages/engine/src/types.ts index 5b5815137a8..ef71e03e5a4 100644 --- a/packages/engine/src/types.ts +++ b/packages/engine/src/types.ts @@ -4,6 +4,7 @@ * The engine's page contract. Any web page that wants to be rendered * as video must expose `window.__hf` implementing the HfProtocol interface. */ +import type { AudioFailureDiagnostic } from "./services/audioMixer.types.js"; import type { Fps } from "@hyperframes/core"; import type { ChromeMemoryStats } from "./services/chromeMemorySampler.js"; import type { MotionBlurOptions } from "./services/motionBlur.js"; @@ -41,6 +42,7 @@ export interface CaptureWarning { sources?: string[]; pendingCompositionIds?: string[]; timeoutMs?: number; + audioFailures?: AudioFailureDiagnostic[]; failureReasons?: string[]; failureStages?: string[]; failureOwner?: "user" | "system"; diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.ts index d6c5e1408c0..6b09a4693bf 100644 --- a/packages/producer/src/services/distributed/plan.ts +++ b/packages/producer/src/services/distributed/plan.ts @@ -1,3 +1,4 @@ +import { audioFailureDiagnostics } from "@hyperframes/engine"; /** * Activity A of the distributed render pipeline. * @@ -337,6 +338,7 @@ export function applyDistributedAudioWarningPolicy( message: `Audio mix failed; output would be video-only: ${audioError}`, details: { mediaType: "audio", + audioFailures: audioFailureDiagnostics(audioFailures), failureReasons: [...new Set(audioFailures.map((failure) => failure.reason))], failureStages: [...new Set(audioFailures.map((failure) => failure.stage))], failureOwner, diff --git a/packages/producer/src/services/render/encoderInterruption.ts b/packages/producer/src/services/render/encoderInterruption.ts index 6b3f62def2f..38cb07219ff 100644 --- a/packages/producer/src/services/render/encoderInterruption.ts +++ b/packages/producer/src/services/render/encoderInterruption.ts @@ -1,8 +1,11 @@ +import { audioFailureDiagnostics, type AudioFailureDiagnostic } from "@hyperframes/engine"; + const ENCODER_INTERRUPTED = "ENCODER_INTERRUPTED" as const; interface EncoderFailureResult { error?: string; failureReason?: "external_interruption"; + audioFailures?: readonly unknown[]; } /** A host/process lifecycle interruption that is safe to retry on a fresh producer. */ @@ -14,16 +17,19 @@ export class EncoderInterruptedError extends Error { /** Retained inside the producer for diagnostics; never copied to the wire envelope. */ readonly diagnosticMessage: string; - constructor(prefix: string, diagnosticMessage: string) { + readonly audioFailures: AudioFailureDiagnostic[]; + + constructor(prefix: string, diagnosticMessage: string, audioFailures: readonly unknown[] = []) { super(`${prefix}: encoder process was interrupted by the host lifecycle`); this.name = "EncoderInterruptedError"; this.diagnosticMessage = diagnosticMessage; + this.audioFailures = audioFailureDiagnostics(audioFailures); } } export function encoderFailureError(prefix: string, result: EncoderFailureResult): Error { const message = `${prefix}: ${result.error ?? "unknown encoder failure"}`; return result.failureReason === "external_interruption" - ? new EncoderInterruptedError(prefix, message) + ? new EncoderInterruptedError(prefix, message, result.audioFailures) : new Error(message); } diff --git a/packages/producer/src/services/render/stages/audioStage.test.ts b/packages/producer/src/services/render/stages/audioStage.test.ts index a04db699cfc..03632ad04c1 100644 --- a/packages/producer/src/services/render/stages/audioStage.test.ts +++ b/packages/producer/src/services/render/stages/audioStage.test.ts @@ -177,12 +177,29 @@ describe("runAudioStage", () => { reason: "external_interruption", owner: "system", retryable: true, - detail: "ffmpeg handled signal 15", + detail: "private ffmpeg handled signal 15", + elementId: "private-element", + subprocess: { exitCode: 255, terminationReason: "exit" }, }, ], }); - await expect(runAudioStage(makeInput())).rejects.toBeInstanceOf(EncoderInterruptedError); + const pending = runAudioStage(makeInput()); + await expect(pending).rejects.toBeInstanceOf(EncoderInterruptedError); + await expect(pending).rejects.toMatchObject({ + audioFailures: [ + { + stage: "mix", + reason: "external_interruption", + owner: "system", + retryable: true, + subprocess: { exitCode: 255, terminationReason: "exit" }, + }, + ], + }); + await pending.catch((error: EncoderInterruptedError) => { + expect(JSON.stringify(error.audioFailures)).not.toContain("private"); + }); }); it("does not set audioError when there is no audio to mix", async () => { diff --git a/packages/producer/src/services/render/stages/audioStage.ts b/packages/producer/src/services/render/stages/audioStage.ts index dae085046d3..aed20e66a74 100644 --- a/packages/producer/src/services/render/stages/audioStage.ts +++ b/packages/producer/src/services/render/stages/audioStage.ts @@ -140,6 +140,7 @@ export async function runAudioStage(input: AudioStageInput): Promise failure.reason))], failureStages: [...new Set(audioFailures.map((failure) => failure.stage))], failureOwner, From 20fe1457cbdbea60e350eb19709d8efdaa91371b Mon Sep 17 00:00:00 2001 From: James Date: Wed, 14 Oct 2026 12:38:15 -0700 Subject: [PATCH 2/5] test(producer): retain real engine exports in assembly mock --- .../services/render/stages/assembleStage.test.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/producer/src/services/render/stages/assembleStage.test.ts b/packages/producer/src/services/render/stages/assembleStage.test.ts index faa18b14f70..613ba91b65b 100644 --- a/packages/producer/src/services/render/stages/assembleStage.test.ts +++ b/packages/producer/src/services/render/stages/assembleStage.test.ts @@ -9,11 +9,15 @@ const { applyFaststartMock, muxVideoWithAudioMock, packageHlsMock, padOrTrimAudi padOrTrimAudioMock: vi.fn(), })); -vi.mock("@hyperframes/engine", () => ({ - applyFaststart: applyFaststartMock, - muxVideoWithAudio: muxVideoWithAudioMock, - packageHls: packageHlsMock, -})); +vi.mock("@hyperframes/engine", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + applyFaststart: applyFaststartMock, + muxVideoWithAudio: muxVideoWithAudioMock, + packageHls: packageHlsMock, + }; +}); vi.mock("../audioPadTrim.js", () => ({ AAC_DELIVERY_TRUE_PEAK_DBFS: -1, From fb27200bdd0b5d17f77c4d1482e61fc8106eef7d Mon Sep 17 00:00:00 2001 From: James Date: Wed, 14 Oct 2026 12:41:26 -0700 Subject: [PATCH 3/5] fix(audio): retain triggering interruption within diagnostic cap --- packages/cli/src/ui/renderDiagnostic.test.ts | 69 ++++++++++++++++++- .../src/services/render/stages/audioStage.ts | 9 ++- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/ui/renderDiagnostic.test.ts b/packages/cli/src/ui/renderDiagnostic.test.ts index 4278f484661..a6b9d4b3eb9 100644 --- a/packages/cli/src/ui/renderDiagnostic.test.ts +++ b/packages/cli/src/ui/renderDiagnostic.test.ts @@ -6,7 +6,8 @@ import { processCompositionAudio, type AudioFailureDiagnostic } from "@hyperfram import { applyDistributedAudioWarningPolicy } from "../../../producer/src/services/distributed/plan.js"; import { buildSyntheticRenderJob } from "../../../producer/src/services/distributed/shared.js"; import { createRenderJob, executeRenderJob, type RenderJob } from "@hyperframes/producer"; -import { renderMachineDiagnostic } from "./renderDiagnostic.js"; +import { runAudioStage } from "../../../producer/src/services/render/stages/audioStage.js"; +import { renderFailureDiagnostic, renderMachineDiagnostic } from "./renderDiagnostic.js"; const roots: string[] = []; const originalIsTTY = process.stdout.isTTY; @@ -17,14 +18,15 @@ afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); -function capture(values: readonly unknown[], terminal = false): string { +function capture(values: readonly unknown[] | (() => void), terminal = false): string { Object.defineProperty(process.stdout, "isTTY", { value: terminal, configurable: true }); let output = ""; vi.spyOn(process.stdout, "write").mockImplementation((chunk) => { output += String(chunk); return true; }); - renderMachineDiagnostic(values); + if (typeof values === "function") values(); + else renderMachineDiagnostic(values); return output; } @@ -231,3 +233,64 @@ it("carries a local render audio failure through the real orchestration warning ]); for (const secret of [root, "private.wav", "private-cue"]) expect(line).not.toContain(secret); }); + +it.skipIf(process.platform === "win32")( + "keeps the triggering seventeenth interruption in the final bounded record", + async () => { + const root = mkdtempSync(join(tmpdir(), "hf-audio-mixed-failure-")); + roots.push(root); + const executable = join(root, "ffmpeg"); + writeFileSync(executable, `#!${process.execPath}\nprocess.kill(process.pid, "SIGTERM");\n`); + chmodSync(executable, 0o755); + vi.stubEnv("HYPERFRAMES_FFMPEG_PATH", executable); + writeFileSync(join(root, "private.wav"), "RIFF0000WAVEfmt "); + const track = { + id: "private-cue", + src: "private.wav", + type: "audio" as const, + start: 0, + end: 1, + mediaStart: 0, + volume: 1, + layer: 0, + }; + let interrupted: unknown; + try { + await runAudioStage({ + projectDir: root, + compiledDir: root, + workDir: join(root, "work"), + duration: 1, + ffmpegProcessTimeout: 5_000, + audioGain: 1, + abortSignal: undefined, + assertNotAborted: () => {}, + audios: [ + ...Array.from({ length: 16 }, (_, i) => ({ + ...track, + id: `private-missing-${i}`, + src: `private-missing-${i}.wav`, + })), + track, + ], + }); + } catch (error) { + interrupted = error; + } + expect(interrupted).toBeInstanceOf(Error); + const line = capture(() => renderFailureDiagnostic(undefined, interrupted, false)); + const records = JSON.parse(line.slice("@hf-diagnostic ".length)).audioFailures; + expect(records).toHaveLength(16); + expect(records[0]).toEqual({ + stage: "prepare", + reason: "external_interruption", + owner: "system", + retryable: true, + subprocess: { signal: "SIGTERM", terminationReason: "exit" }, + }); + expect( + records.slice(1).every((entry: { reason: string }) => entry.reason === "source_not_found"), + ).toBe(true); + expect(line).not.toMatch(/private-|secret/); + }, +); diff --git a/packages/producer/src/services/render/stages/audioStage.ts b/packages/producer/src/services/render/stages/audioStage.ts index aed20e66a74..7e4fcb8d9fd 100644 --- a/packages/producer/src/services/render/stages/audioStage.ts +++ b/packages/producer/src/services/render/stages/audioStage.ts @@ -64,6 +64,13 @@ export interface AudioStageResult { audioFailures?: AudioProcessingFailure[]; } +function triggerFirst( + failures: readonly AudioProcessingFailure[] | undefined, + trigger: AudioProcessingFailure, +): AudioProcessingFailure[] { + return [trigger, ...(failures ?? []).filter((failure) => failure !== trigger)]; +} + export async function runAudioStage(input: AudioStageInput): Promise { const { projectDir, @@ -140,7 +147,7 @@ export async function runAudioStage(input: AudioStageInput): Promise Date: Wed, 14 Oct 2026 12:49:58 -0700 Subject: [PATCH 4/5] fix(producer): sanitize audio evidence before encoder error construction --- .../src/services/render/encoderInterruption.ts | 12 ++++++++---- .../services/render/stages/assembleStage.test.ts | 14 +++++--------- .../src/services/render/stages/audioStage.ts | 3 ++- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/packages/producer/src/services/render/encoderInterruption.ts b/packages/producer/src/services/render/encoderInterruption.ts index 38cb07219ff..dc559f27e3f 100644 --- a/packages/producer/src/services/render/encoderInterruption.ts +++ b/packages/producer/src/services/render/encoderInterruption.ts @@ -1,11 +1,11 @@ -import { audioFailureDiagnostics, type AudioFailureDiagnostic } from "@hyperframes/engine"; +import type { AudioFailureDiagnostic } from "@hyperframes/engine"; const ENCODER_INTERRUPTED = "ENCODER_INTERRUPTED" as const; interface EncoderFailureResult { error?: string; failureReason?: "external_interruption"; - audioFailures?: readonly unknown[]; + audioFailures?: AudioFailureDiagnostic[]; } /** A host/process lifecycle interruption that is safe to retry on a fresh producer. */ @@ -19,11 +19,15 @@ export class EncoderInterruptedError extends Error { readonly audioFailures: AudioFailureDiagnostic[]; - constructor(prefix: string, diagnosticMessage: string, audioFailures: readonly unknown[] = []) { + constructor( + prefix: string, + diagnosticMessage: string, + audioFailures: AudioFailureDiagnostic[] = [], + ) { super(`${prefix}: encoder process was interrupted by the host lifecycle`); this.name = "EncoderInterruptedError"; this.diagnosticMessage = diagnosticMessage; - this.audioFailures = audioFailureDiagnostics(audioFailures); + this.audioFailures = audioFailures; } } diff --git a/packages/producer/src/services/render/stages/assembleStage.test.ts b/packages/producer/src/services/render/stages/assembleStage.test.ts index 613ba91b65b..faa18b14f70 100644 --- a/packages/producer/src/services/render/stages/assembleStage.test.ts +++ b/packages/producer/src/services/render/stages/assembleStage.test.ts @@ -9,15 +9,11 @@ const { applyFaststartMock, muxVideoWithAudioMock, packageHlsMock, padOrTrimAudi padOrTrimAudioMock: vi.fn(), })); -vi.mock("@hyperframes/engine", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - applyFaststart: applyFaststartMock, - muxVideoWithAudio: muxVideoWithAudioMock, - packageHls: packageHlsMock, - }; -}); +vi.mock("@hyperframes/engine", () => ({ + applyFaststart: applyFaststartMock, + muxVideoWithAudio: muxVideoWithAudioMock, + packageHls: packageHlsMock, +})); vi.mock("../audioPadTrim.js", () => ({ AAC_DELIVERY_TRUE_PEAK_DBFS: -1, diff --git a/packages/producer/src/services/render/stages/audioStage.ts b/packages/producer/src/services/render/stages/audioStage.ts index 7e4fcb8d9fd..0cbf2bea1f3 100644 --- a/packages/producer/src/services/render/stages/audioStage.ts +++ b/packages/producer/src/services/render/stages/audioStage.ts @@ -20,6 +20,7 @@ import { join } from "node:path"; import { MIXED_AUDIO_FILENAME, + audioFailureDiagnostics, processCompositionAudio, type AudioProcessingFailure, } from "@hyperframes/engine"; @@ -147,7 +148,7 @@ export async function runAudioStage(input: AudioStageInput): Promise Date: Wed, 14 Oct 2026 13:05:12 -0700 Subject: [PATCH 5/5] test(cli): isolate diagnostic render temporary project --- packages/cli/src/commands/render.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/commands/render.test.ts b/packages/cli/src/commands/render.test.ts index 552f2f9f9c1..1da6f7e30dc 100644 --- a/packages/cli/src/commands/render.test.ts +++ b/packages/cli/src/commands/render.test.ts @@ -622,6 +622,7 @@ describe("renderLocal browser GPU config", () => { { quiet: false, interrupted: true, emits: true }, { quiet: true, interrupted: false, emits: false }, ])("writes the failure diagnostic only for piped non-JSON output: %j", async (mode) => { + const projectDir = mkdtempSync(join(tmpdir(), "hf-render-diagnostic-")); const failures = [ { stage: "prepare", @@ -653,7 +654,7 @@ describe("renderLocal browser GPU config", () => { }); try { await expect( - renderLocal("/tmp/project", "/tmp/out.mp4", { + renderLocal(projectDir, join(projectDir, "out.mp4"), { fps: { num: 30, den: 1 }, quality: "standard", format: "mp4", @@ -675,6 +676,7 @@ describe("renderLocal browser GPU config", () => { } } finally { Object.defineProperty(process.stdout, "isTTY", { value: originalIsTTY, configurable: true }); + rmSync(projectDir, { recursive: true, force: true }); } });