diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.ts index e7b1fb5ac7..cc04191f93 100644 --- a/packages/engine/src/services/audioMixer.test.ts +++ b/packages/engine/src/services/audioMixer.test.ts @@ -1,6 +1,6 @@ // fallow-ignore-file code-duplication import { afterEach, describe, expect, it, vi } from "vitest"; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; @@ -861,6 +861,149 @@ describe("processCompositionAudio", () => { ]); }); + it("preserves an external interruption from a group sub-mix", async () => { + const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); + tempDirs.push(baseDir, workDir); + writeFileSync(join(baseDir, "voice-a.wav"), "stub"); + writeFileSync(join(baseDir, "voice-b.wav"), "stub"); + + runFfmpegMock.mockImplementation(async (args: string[]) => { + if (String(args.at(-1)).includes("group-voiceover")) { + return { + success: false, + durationMs: 1, + stderr: "normalize: Option not found\nffmpeg exited after signal 15", + exitCode: 0, + terminationReason: "signal" as const, + failureReason: "external_interruption" as const, + }; + } + return { success: true, durationMs: 1, stderr: "", exitCode: 0 }; + }); + + const result = await processCompositionAudio( + [ + { + id: "voice-a", + src: "voice-a.wav", + start: 0, + end: 2, + mediaStart: 0, + layer: 0, + volume: 1, + volumeKeyframes: [ + { time: 0, volume: 1 }, + { time: 2, volume: 0.5 }, + ], + groupId: "voiceover", + type: "audio", + }, + { + id: "voice-b", + src: "voice-b.wav", + start: 0, + end: 2, + mediaStart: 0, + layer: 1, + volume: 1, + groupId: "voiceover", + type: "audio", + }, + ], + baseDir, + workDir, + join(baseDir, "out.m4a"), + 2, + ); + + expect(result.success).toBe(false); + expect(result.failures).toEqual([ + expect.objectContaining({ + stage: "mix", + reason: "external_interruption", + owner: "system", + retryable: true, + elementId: "voiceover", + }), + ]); + expect( + runFfmpegMock.mock.calls.filter(([args]) => String(args.at(-1)).includes("group-voiceover")), + ).toHaveLength(1); + expect(existsSync(workDir)).toBe(false); + }); + + it("preserves a managed deadline from a group sub-mix without degradation retries", async () => { + const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); + tempDirs.push(baseDir, workDir); + writeFileSync(join(baseDir, "sfx-a.wav"), "stub"); + writeFileSync(join(baseDir, "sfx-b.wav"), "stub"); + + runFfmpegMock.mockImplementation(async (args: string[]) => { + if (String(args.at(-1)).includes("group-sfx")) { + return { + success: false, + durationMs: 300_000, + stderr: "managed deadline reached", + exitCode: null, + terminationReason: "deadline" as const, + }; + } + return { success: true, durationMs: 1, stderr: "", exitCode: 0 }; + }); + + const result = await processCompositionAudio( + [ + { + id: "sfx-a", + src: "sfx-a.wav", + start: 0, + end: 2, + mediaStart: 0, + layer: 0, + volume: 1, + volumeKeyframes: [ + { time: 0, volume: 1 }, + { time: 2, volume: 0.5 }, + ], + groupId: "sfx", + type: "audio", + }, + { + id: "sfx-b", + src: "sfx-b.wav", + start: 0, + end: 2, + mediaStart: 0, + layer: 1, + volume: 1, + groupId: "sfx", + type: "audio", + }, + ], + baseDir, + workDir, + join(baseDir, "out.m4a"), + 2, + ); + + expect(result.success).toBe(false); + expect(result.failures).toEqual([ + expect.objectContaining({ + stage: "mix", + reason: "ffmpeg_timeout", + owner: "system", + retryable: true, + elementId: "sfx", + }), + ]); + expect( + runFfmpegMock.mock.calls.filter(([args]) => String(args.at(-1)).includes("group-sfx")), + ).toHaveLength(1); + expect(existsSync(workDir)).toBe(false); + }); + it("bounds per-cause details and the aggregate error across many authored IDs", async () => { const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts index 2c2b8c6884..a0d5801142 100644 --- a/packages/engine/src/services/audioMixer.ts +++ b/packages/engine/src/services/audioMixer.ts @@ -955,7 +955,12 @@ async function mixGroupMembers( totalDuration: number, signal?: AbortSignal, config?: Partial>, -): Promise<{ success: boolean; error?: string; degradedAutomation?: boolean }> { +): Promise<{ + success: boolean; + error?: string; + degradedAutomation?: boolean; + failure?: AudioProcessingFailure; +}> { const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout; const outputDir = dirname(outputPath); if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true }); @@ -1034,7 +1039,9 @@ async function mixGroupMembers( let useNormalize = true; let result = await runOnce(useNormalize); - if (!result.success && groupNormalizeOptionUnsupported(result.stderr)) { + const canRetrySubmix = () => + !result.success && !signal?.aborted && !ffmpegFailure("mix", result).retryable; + if (canRetrySubmix() && groupNormalizeOptionUnsupported(result.stderr)) { useNormalize = false; result = await runOnce(useNormalize); } @@ -1046,7 +1053,7 @@ async function mixGroupMembers( // grouped, it took the entire composition's audio down with it. let degradedAutomation = false; const hasAutomation = memberTracks.some((track) => (track.volumeKeyframes?.length ?? 0) > 0); - if (!result.success && !signal?.aborted && hasAutomation) { + if (canRetrySubmix() && hasAutomation) { const retry = await runOnce(useNormalize, true); if (retry.success) { result = retry; @@ -1056,7 +1063,11 @@ async function mixGroupMembers( if (signal?.aborted) return { success: false, error: "Group sub-mix cancelled" }; if (!result.success) - return { success: false, error: formatFfmpegError(result.exitCode, result.stderr) }; + return { + success: false, + error: formatFfmpegError(result.exitCode, result.stderr), + failure: ffmpegFailure("mix", result), + }; return { success: true, degradedAutomation }; } @@ -1435,11 +1446,15 @@ export async function processCompositionAudio( config, ); if (!subMix.success) { - failures.push({ + const failure = subMix.failure ?? { stage: "mix", reason: "ffmpeg_failed", owner: "system", retryable: false, + detail: subMix.error ?? "unknown", + }; + failures.push({ + ...failure, elementId: groupId, detail: boundedDetail( `Group sub-mix failed for group ${groupId}: ${subMix.error ?? "unknown"}`, diff --git a/packages/engine/src/services/captureFailure.test.ts b/packages/engine/src/services/captureFailure.test.ts index 4cb720813f..150c6dfb20 100644 --- a/packages/engine/src/services/captureFailure.test.ts +++ b/packages/engine/src/services/captureFailure.test.ts @@ -4,6 +4,8 @@ import { CaptureFailure, classifyCaptureFailure, isFatalCaptureFailure } from ". describe("classifyCaptureFailure", () => { it.each([ ["Target closed", "transient_browser"], + ["connect ETIMEDOUT 127.0.0.1:49152", "transient_browser"], + ["net::ERR_TIMED_OUT at http://localhost:49152/index.html", "transient_browser"], ["Runtime.callFunctionOn timed out after 30000ms", "protocol_timeout"], ["Runtime.evaluate timed out", "protocol_timeout"], [ @@ -15,6 +17,7 @@ describe("classifyCaptureFailure", () => { ["JavaScript heap out of memory", "memory_exhaustion"], ["drawElement self-verify failed", "verification"], ["Composition has zero duration. Runtime ready: true", "authoring"], + ["connect ETIMEDOUT 203.0.113.10:443", "authoring"], ] as const)("classifies %s as %s", (message, kind) => { expect(classifyCaptureFailure(new Error(message)).kind).toBe(kind); }); @@ -60,6 +63,16 @@ describe("classifyCaptureFailure", () => { expect(Object.isFrozen(failure.workerDiagnostics[0]?.lines)).toBe(true); }); + it("retains loopback endpoint provenance for capture diagnostics", () => { + expect(classifyCaptureFailure(new Error("connect ETIMEDOUT 127.0.0.1:49152"))).toMatchObject({ + kind: "transient_browser", + endpoint: { host: "127.0.0.1", port: 49152 }, + }); + expect( + classifyCaptureFailure(new Error("net::ERR_TIMED_OUT at http://localhost:4173/index.html")), + ).toMatchObject({ endpoint: { host: "localhost", port: 4173 } }); + }); + it("classifies repeated operation text in linear time", () => { const repeatedCopy = "copy".repeat(25_000); diff --git a/packages/engine/src/services/captureFailure.ts b/packages/engine/src/services/captureFailure.ts index a2428a4a58..e6f3a95139 100644 --- a/packages/engine/src/services/captureFailure.ts +++ b/packages/engine/src/services/captureFailure.ts @@ -15,16 +15,23 @@ export interface CaptureWorkerDiagnostic { lines: readonly string[]; } +export interface CaptureEndpointDiagnostic { + host: string; + port: number; +} + export class CaptureFailure extends Error { readonly kind: CaptureFailureKind; readonly cause: unknown; readonly workerDiagnostics: readonly CaptureWorkerDiagnostic[]; + readonly endpoint?: Readonly; constructor(input: { kind: CaptureFailureKind; message: string; cause?: unknown; workerDiagnostics?: readonly CaptureWorkerDiagnostic[]; + endpoint?: CaptureEndpointDiagnostic; }) { super(input.message); this.name = "CaptureFailure"; @@ -35,6 +42,7 @@ export class CaptureFailure extends Error { Object.freeze({ ...diagnostic, lines: Object.freeze([...diagnostic.lines]) }), ), ); + this.endpoint = input.endpoint ? Object.freeze({ ...input.endpoint }) : undefined; if (input.cause instanceof Error && input.cause.stack) this.stack = input.cause.stack; } } @@ -101,6 +109,17 @@ function messageOf(error: unknown): string { return error instanceof Error ? error.message : String(error); } +function loopbackTimeoutEndpoint(message: string): CaptureEndpointDiagnostic | undefined { + const match = + /connect ETIMEDOUT (127\.0\.0\.1|localhost|\[::1\]):(\d+)/i.exec(message) ?? + /net::ERR_TIMED_OUT at https?:\/\/(127\.0\.0\.1|localhost|\[::1\]):(\d+)/i.exec(message); + if (!match?.[1] || !match[2]) return undefined; + const port = Number(match[2]); + return Number.isInteger(port) && port > 0 && port <= 65_535 + ? { host: match[1], port } + : undefined; +} + function matchesAny(message: string, patterns: readonly RegExp[]): boolean { return patterns.some((pattern) => pattern.test(message)); } @@ -138,6 +157,8 @@ export function classifyCaptureFailure( return error; } const message = messageOf(error); + const timeoutEndpoint = loopbackTimeoutEndpoint(message); + const endpoint = error instanceof CaptureFailure ? error.endpoint : timeoutEndpoint; let kind: CaptureFailureKind; if (options.signal?.aborted || /(?:render|capture)?_?cancelled|AbortError/i.test(message)) { kind = "cancelled"; @@ -151,7 +172,7 @@ export function classifyCaptureFailure( kind = "verification"; } else if (matchesAny(message, PROTOCOL_TIMEOUT_PATTERNS)) { kind = "protocol_timeout"; - } else if (matchesAny(message, TRANSIENT_BROWSER_ERROR_PATTERNS)) { + } else if (timeoutEndpoint || matchesAny(message, TRANSIENT_BROWSER_ERROR_PATTERNS)) { kind = "transient_browser"; } else if (matchesAny(message, AUTHORING_ERROR_PATTERNS)) { kind = "authoring"; @@ -165,6 +186,7 @@ export function classifyCaptureFailure( workerDiagnostics: options.workerDiagnostics ?? (error instanceof CaptureFailure ? error.workerDiagnostics : undefined), + endpoint, }); } diff --git a/packages/producer/src/services/fileServer.test.ts b/packages/producer/src/services/fileServer.test.ts index 6101bd1ba0..912eecccdb 100644 --- a/packages/producer/src/services/fileServer.test.ts +++ b/packages/producer/src/services/fileServer.test.ts @@ -5,12 +5,14 @@ import { tmpdir } from "node:os"; import { closeFileServerSafely, createFileServer, + FILE_SERVER_HEALTH_PATH, RENDER_CAPTURE_MODE_SHIM, HF_BRIDGE_SCRIPT, HF_EARLY_STUB, injectScriptsAtHeadStart, isPathInside, parseRangeHeader, + probeFileServerHealth, VIRTUAL_TIME_SHIM, } from "./fileServer.js"; @@ -74,6 +76,52 @@ function writeEmptyIndex(projectDir: string): void { writeFileSync(join(projectDir, "index.html"), ""); } +describe("file server health", () => { + it("serves a dedicated loopback health endpoint", async () => { + const projectDir = mkdtempSync(join(tmpdir(), "hf-file-server-health-")); + try { + writeEmptyIndex(projectDir); + await withFileServer(projectDir, async (server) => { + const response = await fetch(`${server.url}${FILE_SERVER_HEALTH_PATH}`); + + expect(response.status).toBe(200); + expect(await response.text()).toBe("ok"); + expect(await probeFileServerHealth(server)).toMatchObject({ + healthy: true, + status: 200, + }); + }); + } finally { + rmSync(projectDir, { recursive: true, force: true }); + } + }); + + it("reports an unreachable endpoint without throwing", async () => { + const health = await probeFileServerHealth({ url: "http://127.0.0.1:1" }, 100); + + expect(health.healthy).toBe(false); + expect(health.error).toBeTruthy(); + expect(health.durationMs).toBeLessThan(1_000); + }); + + it("bounds a health endpoint that never responds", async () => { + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch: () => new Promise(() => {}), + }); + try { + const health = await probeFileServerHealth({ url: `http://127.0.0.1:${server.port}` }, 25); + + expect(health.healthy).toBe(false); + expect(health.error).toBeTruthy(); + expect(health.durationMs).toBeLessThan(500); + } finally { + server.stop(true); + } + }); +}); + async function expectTextResponse( url: string, options: { contentType?: string; bodyIncludes: string }, diff --git a/packages/producer/src/services/fileServer.ts b/packages/producer/src/services/fileServer.ts index 33ceade0b9..c537b80e21 100644 --- a/packages/producer/src/services/fileServer.ts +++ b/packages/producer/src/services/fileServer.ts @@ -682,6 +682,47 @@ export interface FileServerHandle { addPreHeadScript: (script: string) => void; } +export interface FileServerHealth { + healthy: boolean; + status?: number; + durationMs: number; + error?: string; +} + +export const FILE_SERVER_HEALTH_PATH = "/__hyperframes_health"; +const FILE_SERVER_HEALTH_HEADER = "x-hyperframes-file-server"; + +export async function probeFileServerHealth( + fileServer: Pick, + timeoutMs = 1_000, +): Promise { + const startedAt = Date.now(); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + if (typeof timer.unref === "function") timer.unref(); + try { + const response = await fetch(`${fileServer.url}${FILE_SERVER_HEALTH_PATH}`, { + signal: controller.signal, + cache: "no-store", + }); + const healthy = response.ok && response.headers.get(FILE_SERVER_HEALTH_HEADER) === "healthy"; + await response.body?.cancel(); + return { + healthy, + status: response.status, + durationMs: Date.now() - startedAt, + }; + } catch (error) { + return { + healthy: false, + durationMs: Date.now() - startedAt, + error: error instanceof Error ? error.message : String(error), + }; + } finally { + clearTimeout(timer); + } +} + /** * Set before the Hyperframes runtime executes so render/probe pages can avoid * preview-only initialization work that mutates the live visual timeline. @@ -734,6 +775,11 @@ export function createFileServer(options: FileServerOptions): Promise { + c.header(FILE_SERVER_HEALTH_HEADER, "healthy"); + return c.text("ok"); + }); + app.get("/*", async (c) => { let requestPath = c.req.path; if (requestPath === "/") requestPath = "/index.html"; diff --git a/packages/producer/src/services/render/capturePlan.test.ts b/packages/producer/src/services/render/capturePlan.test.ts index 8e8352de42..5d11a34a56 100644 --- a/packages/producer/src/services/render/capturePlan.test.ts +++ b/packages/producer/src/services/render/capturePlan.test.ts @@ -69,6 +69,17 @@ describe("CapturePlan", () => { }); }); + it("replans an ordinary one-worker capture failure onto a fresh screenshot stream", () => { + expect( + replanAfterFailure(streaming(), { kind: "capture_failure", memoryExhaustion: false }), + ).toMatchObject({ + kind: "sdr_streaming", + workerCount: 1, + forceScreenshot: true, + routing: { kind: "default" }, + }); + }); + it("atomically restores the pre-inversion disk route after verification failure", () => { const initial = streaming({ kind: "worker_inversion", diff --git a/packages/producer/src/services/renderOrchestrator.test.ts b/packages/producer/src/services/renderOrchestrator.test.ts index 5ce124b887..21ccfde808 100644 --- a/packages/producer/src/services/renderOrchestrator.test.ts +++ b/packages/producer/src/services/renderOrchestrator.test.ts @@ -2508,6 +2508,30 @@ describe("shouldRetryViaPinnedFallback (widen the self-verify retry to generic c ).toBe(false); }); + it("retries a transient browser death on an ordinary one-worker streaming route", () => { + expect( + shouldRetryViaPinnedFallback({ + isVerifyError: false, + isCancellation: false, + isTransientSingleWorkerFailure: true, + deWorkerInversion: undefined, + deParallelRouter: undefined, + }), + ).toBe(true); + }); + + it("does not widen the transient streaming retry to an unpinned multi-worker route", () => { + expect( + shouldRetryViaPinnedFallback({ + isVerifyError: false, + isCancellation: false, + isTransientSingleWorkerFailure: false, + deWorkerInversion: undefined, + deParallelRouter: undefined, + }), + ).toBe(false); + }); + it("retries OOM too when the router pinned the worker count (fallback's Chrome processes are already dead by the time this runs, and the fallback is pooled/lighter than the pinned path)", () => { expect( shouldRetryViaPinnedFallback({ @@ -2569,6 +2593,15 @@ describe("shouldRetryViaPinnedFallback (widen the self-verify retry to generic c deParallelRouter: undefined, }), ).toBe(false); + expect( + shouldRetryViaPinnedFallback({ + isVerifyError: false, + isCancellation: true, + isTransientSingleWorkerFailure: true, + deWorkerInversion: undefined, + deParallelRouter: undefined, + }), + ).toBe(false); }); it("never hides an encoder host interruption behind the same-host pinned fallback", () => { diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index c7a373846d..321cb58bd1 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -99,6 +99,7 @@ import { fileURLToPath } from "url"; import { closeFileServerSafely, createFileServer, + probeFileServerHealth, type FileServerHandle, HF_PAGE_SIDE_COMPOSITING_STUB, VIRTUAL_TIME_SHIM, @@ -1810,10 +1811,8 @@ export function resolveParallelRouterRetryPlan(args: { } /** - * Should a capture-stage error retry via the pinned-worker-count fallback - * (the same "well-tested parallel-disk / single-worker screenshot" path - * `resolveInversionRetryPlan`/`resolveParallelRouterRetryPlan` reroute to) - * instead of failing the render outright? + * Should a streaming capture-stage error use the bounded screenshot recovery + * path instead of failing the render outright? * * True for the drawElement self-verify failures this retry path was * originally built for (blank frame / PSNR breach), AND for any OTHER @@ -1822,6 +1821,12 @@ export function resolveParallelRouterRetryPlan(args: { * of calibration, so a generic capture failure on that pinned count is * exactly the scenario the pin itself introduced risk for. * + * An ordinary one-worker stream also gets one retry when Chrome itself dies. + * There is no worker count to reduce, but the failed stage has already closed + * its session and encoder; replanning forces screenshot capture and the second + * invoke creates fresh resources. The surrounding catch performs this at most + * once, so a deterministically dying composition still fails. + * * Includes OOM (previously excluded — see PR history): every worker's * `executeWorkerTask` closes its capture session in a `finally` that awaits * `closeCaptureSession` → `releaseBrowser`, which SIGKILLs the Chrome process @@ -1847,16 +1852,18 @@ export function shouldRetryViaPinnedFallback(args: { isVerifyError: boolean; isCancellation: boolean; isEncoderInterrupted?: boolean; + isTransientSingleWorkerFailure?: boolean; deWorkerInversion: "inverted" | "reverted" | undefined; deParallelRouter: "routed" | "reverted" | undefined; }): boolean { if (args.isCancellation || args.isEncoderInterrupted) return false; if (args.isVerifyError) return true; + if (args.isTransientSingleWorkerFailure) return true; return args.deWorkerInversion === "inverted" || args.deParallelRouter === "routed"; } /** - * When a self-verify (or pinned-fallback) retry is triggered mid-capture, the + * When a self-verify or capture retry is triggered mid-capture, the * caller may still hold a live probe session that the failed stage was passed * but did not (or could not) close in its own `finally` before it threw. Left * behind, that session's Chrome process orphans until the containing render @@ -2637,11 +2644,10 @@ async function executeRenderPipeline(input: { } else { observability.checkpoint("file_server", "reused probe file server"); } - const activeFileServer = fileServer; + let activeFileServer = fileServer; if (!activeFileServer) { throw new Error("File server failed to initialize before frame capture"); } - const framesDir = join(workDir, "captured-frames"); if (!existsSync(framesDir)) mkdirSync(framesDir, { recursive: true }); @@ -3312,6 +3318,21 @@ async function executeRenderPipeline(input: { "screenshot per output frame.", ); } + const restartCaptureFileServer = async (): Promise => { + closeFileServerSafely(activeFileServer, "capture retry", log); + fileServer = null; + activeFileServer = await createFileServer({ + projectDir, + compiledDir: join(workDir, "compiled"), + port: 0, + preHeadScripts: [ + VIRTUAL_TIME_SHIM, + ...(usePageSideCompositingForTransitions ? [HF_PAGE_SIDE_COMPOSITING_STUB] : []), + ], + fps: job.config.fps, + }); + fileServer = activeFileServer; + }; const useLayeredComposite = !usePageSideCompositingForTransitions && shouldUseLayeredComposite({ @@ -3590,10 +3611,9 @@ async function executeRenderPipeline(input: { streamingRes = await invokeStreaming(); } catch (err) { // drawElement self-verification tripped (blank frame or PSNR breach - // vs the pre-injection ground truth), OR — when the inversion/router - // pinned a fixed worker count regardless of calibration — any other - // capture-stage failure (host contention timeout, worker crash, OOM) - // on that pinned path. Both restart the whole render on the same + // vs the pre-injection ground truth), an ordinary single-worker + // stream lost its browser, OR a pinned inversion/router path failed. + // Each restarts the whole render on the same // tested screenshot/parallel-SS baseline: slower, never wrong. The // failed attempt's session was closed by the stage's finally; // probeSession (if any) was consumed by it, so a fresh session @@ -3602,11 +3622,16 @@ async function executeRenderPipeline(input: { const isVerifyError = isDrawElementVerificationError(err); const isCancellation = err instanceof RenderCancelledError || executionSignal?.aborted === true; + const captureFailure = classifyCaptureFailure(err, { signal: executionSignal }); + const isTransientBrowserFailure = captureFailure.kind === "transient_browser"; + const isTransientSingleWorkerFailure = + isTransientBrowserFailure && capturePlan.workerCount === 1; if ( !shouldRetryViaPinnedFallback({ isVerifyError, isCancellation, isEncoderInterrupted: err instanceof EncoderInterruptedError, + isTransientSingleWorkerFailure, deWorkerInversion, deParallelRouter, }) @@ -3626,14 +3651,18 @@ async function executeRenderPipeline(input: { log.warn( isVerifyError ? "[Render] drawElement self-verification failed; re-rendering via screenshot" - : "[Render] capture failed on the pinned worker count; re-rendering via screenshot", + : isTransientSingleWorkerFailure + ? "[Render] transient single-worker browser failure; retrying with a fresh screenshot session" + : "[Render] capture failed on the pinned worker count; re-rendering via screenshot", { error: err instanceof Error ? err.message : String(err) }, ); observability.checkpoint( "capture_streaming", isVerifyError ? "drawElement self-verify failed; retrying with forceScreenshot" - : "capture failed on pinned worker count; retrying with forceScreenshot", + : isTransientSingleWorkerFailure + ? "transient single-worker browser failure; retrying with a fresh screenshot session" + : "capture failed on pinned worker count; retrying with forceScreenshot", ); const failedRouting = capturePlan.routing.kind; capturePlan = replanAfterFailure( @@ -3655,6 +3684,10 @@ async function executeRenderPipeline(input: { deWorkerInversion, deParallelRouter, }); + const preFrameHealthPromise = + isTransientSingleWorkerFailure && job.framesRendered === 0 + ? probeFileServerHealth(activeFileServer) + : null; // Streaming stage aims to close the probe in its own finally; if it // threw before doing so, the Chrome process would orphan through the // pinned-fallback retry. Close defensively before we release the @@ -3665,6 +3698,32 @@ async function executeRenderPipeline(input: { probeSession = null; await closeOrphanedProbeForRetry(orphaned, closeCaptureSession, log, "streaming"); } + if (preFrameHealthPromise) { + const health = await preFrameHealthPromise; + const endpointOwner = + captureFailure.endpoint?.port === activeFileServer.port + ? "file_server" + : captureFailure.endpoint + ? "browser_or_unknown" + : "unknown"; + log.warn("[Render] Pre-frame capture endpoint health", { + reportedEndpoint: captureFailure.endpoint + ? `${captureFailure.endpoint.host}:${captureFailure.endpoint.port}` + : undefined, + endpointOwner, + fileServerEndpoint: activeFileServer.url, + fileServerHealthy: health.healthy, + fileServerStatus: health.status, + healthProbeMs: health.durationMs, + healthProbeError: health.error, + }); + if (!health.healthy) { + await restartCaptureFileServer(); + log.warn("[Render] Recreated unhealthy file server before bounded capture retry", { + fileServerEndpoint: activeFileServer.url, + }); + } + } if (failedRouting === "worker_inversion") { // The inversion bet on drawElement and lost — re-render on the // pre-inversion parallel screenshot path instead of single-worker