From 00db19ea1f116a1721cddfa0d3eca518d6bd27e8 Mon Sep 17 00:00:00 2001 From: amide-init Date: Fri, 25 Sep 2026 01:06:08 +0530 Subject: [PATCH 01/35] feat(server): add TranscriptionJob model for background transcription Long podcast episodes take minutes to transcribe, too long to hold an API request open (claude.md section 14). A job row mirroring RenderJob lets the work run in the background while the client polls status and per-chunk progress. The finished transcript still lives in Transcript; this row only tracks the run. Co-Authored-By: Claude Opus 5.5 --- .../migration.sql | 15 +++++++++++ server/prisma/schema.prisma | 26 +++++++++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 server/prisma/migrations/20260924192520_add_transcription_job/migration.sql diff --git a/server/prisma/migrations/20260924192520_add_transcription_job/migration.sql b/server/prisma/migrations/20260924192520_add_transcription_job/migration.sql new file mode 100644 index 0000000..6b6250c --- /dev/null +++ b/server/prisma/migrations/20260924192520_add_transcription_job/migration.sql @@ -0,0 +1,15 @@ +-- CreateTable +CREATE TABLE "TranscriptionJob" ( + "id" TEXT NOT NULL PRIMARY KEY, + "projectId" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'queued', + "totalChunks" INTEGER NOT NULL DEFAULT 0, + "completedChunks" INTEGER NOT NULL DEFAULT 0, + "error" TEXT, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL, + CONSTRAINT "TranscriptionJob_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project" ("id") ON DELETE CASCADE ON UPDATE CASCADE +); + +-- CreateIndex +CREATE INDEX "TranscriptionJob_projectId_idx" ON "TranscriptionJob"("projectId"); diff --git a/server/prisma/schema.prisma b/server/prisma/schema.prisma index 3748fd5..d98f07f 100644 --- a/server/prisma/schema.prisma +++ b/server/prisma/schema.prisma @@ -35,8 +35,9 @@ model Project { assets Asset[] transcript Transcript? - editOperations EditOperation[] - renderJobs RenderJob[] + editOperations EditOperation[] + renderJobs RenderJob[] + transcriptionJobs TranscriptionJob[] } model Asset { @@ -83,6 +84,27 @@ model EditOperation { @@index([projectId]) } +/// Background transcription run (audio extraction -> chunked Whisper calls -> +/// merge), tracked the same way as RenderJob so long episodes never block +/// an API request (claude.md section 14). The finished result is written to +/// Transcript; this row only carries status/progress. +model TranscriptionJob { + id String @id @default(cuid()) + projectId String + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) + + /// "queued" | "processing" | "completed" | "failed" + status String @default("queued") + totalChunks Int @default(0) + completedChunks Int @default(0) + error String? + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@index([projectId]) +} + model RenderJob { id String @id @default(cuid()) projectId String From fee758328dd91da0b0337186c14b8b88fea78109 Mon Sep 17 00:00:00 2001 From: amide-init Date: Fri, 25 Sep 2026 01:06:14 +0530 Subject: [PATCH 02/35] feat(server): add ffmpeg builders for audio extraction, chunking and proxy Deterministic argv builders for the long-episode pipeline: extract a mono 16kHz 32kbps speech track (~14MB/hour, well under Whisper's 25MB cap per chunk), detect silences to choose chunk boundaries, slice chunks with an exact re-encoded seek so merged timestamps stay precise, and build a 720p editing proxy (claude.md section 16). runFfmpegCapturingStderr exists because silencedetect reports its results only as log lines, and probeDuration is needed to plan chunks. Co-Authored-By: Claude Opus 5.5 --- server/src/lib/ffmpeg/audio.test.ts | 45 ++++++++++ server/src/lib/ffmpeg/audio.ts | 122 ++++++++++++++++++++++++++++ server/src/lib/ffmpeg/probe.ts | 33 ++++++++ server/src/lib/ffmpeg/run.ts | 15 +++- 4 files changed, 212 insertions(+), 3 deletions(-) create mode 100644 server/src/lib/ffmpeg/audio.test.ts create mode 100644 server/src/lib/ffmpeg/audio.ts diff --git a/server/src/lib/ffmpeg/audio.test.ts b/server/src/lib/ffmpeg/audio.test.ts new file mode 100644 index 0000000..529c365 --- /dev/null +++ b/server/src/lib/ffmpeg/audio.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; +import { buildAudioChunkArgs, buildProxyArgs, needsProxy, parseSilenceDetectOutput } from "@/lib/ffmpeg/audio"; + +describe("parseSilenceDetectOutput", () => { + it("pairs silence_start/silence_end lines into spans", () => { + const stderr = [ + "Input #0, mp3, from 'speech.mp3':", + "[silencedetect @ 0x1] silence_start: 12.5", + "[silencedetect @ 0x1] silence_end: 13.25 | silence_duration: 0.75", + "size=N/A time=00:01:00.00", + "[silencedetect @ 0x1] silence_start: -0.01", + "[silencedetect @ 0x1] silence_end: 0.6 | silence_duration: 0.61", + ].join("\n"); + expect(parseSilenceDetectOutput(stderr)).toEqual([ + { start: 12.5, end: 13.25 }, + { start: 0, end: 0.6 }, + ]); + }); + + it("drops a trailing silence_start with no matching end", () => { + expect(parseSilenceDetectOutput("[silencedetect @ 0x1] silence_start: 50")).toEqual([]); + }); +}); + +describe("buildAudioChunkArgs", () => { + it("seeks before the input and re-encodes to the speech format", () => { + const args = buildAudioChunkArgs("/in.mp3", "/out.mp3", 600.5, 598.25); + expect(args.slice(0, 7)).toEqual(["-y", "-ss", "600.500", "-t", "598.250", "-i", "/in.mp3"]); + expect(args).toContain("libmp3lame"); + expect(args[args.length - 1]).toBe("/out.mp3"); + }); +}); + +describe("proxy", () => { + it("is only needed for sources above 720p or large files", () => { + expect(needsProxy({ height: 720, sizeBytes: 50 * 1024 * 1024 })).toBe(false); + expect(needsProxy({ height: 1080, sizeBytes: 50 * 1024 * 1024 })).toBe(true); + expect(needsProxy({ height: 720, sizeBytes: 2 * 1024 * 1024 * 1024 })).toBe(true); + }); + + it("scales to 720p with an even width", () => { + const args = buildProxyArgs("/in.mov", "/out.mp4"); + expect(args[args.indexOf("-vf") + 1]).toBe("scale=-2:720"); + }); +}); diff --git a/server/src/lib/ffmpeg/audio.ts b/server/src/lib/ffmpeg/audio.ts new file mode 100644 index 0000000..c352adb --- /dev/null +++ b/server/src/lib/ffmpeg/audio.ts @@ -0,0 +1,122 @@ +/** + * Deterministic ffmpeg argv builders for the transcription pipeline + * (audio extraction, silence detection, chunk slicing) and the playback + * proxy. Every builder returns an argv array for execFile -- never a shell + * string -- and only ever interpolates app-controlled paths and numbers + * (claude.md sections 6, 12, 18). + */ + +/** + * Mono, 16kHz, 32kbps MP3: roughly 14MB per hour of audio. Whisper resamples + * to 16kHz internally anyway, so this loses nothing it would have used, and + * it keeps even a 10-minute chunk far below Whisper's 25MB upload cap. + */ +const SPEECH_AUDIO_ARGS = ["-vn", "-ac", "1", "-ar", "16000", "-c:a", "libmp3lame", "-b:a", "32k"]; + +/** Extracts the source's audio track as a small speech-quality MP3. */ +export function buildExtractAudioArgs(inputPath: string, outputPath: string): string[] { + return ["-y", "-i", inputPath, ...SPEECH_AUDIO_ARGS, outputPath]; +} + +/** Silences quieter than this, for at least SILENCE_MIN_SECONDS, are candidate chunk boundaries. */ +const SILENCE_NOISE_DB = -35; +const SILENCE_MIN_SECONDS = 0.4; + +/** Analysis-only pass: ffmpeg logs silence_start/silence_end lines to stderr, writes no output. */ +export function buildSilenceDetectArgs(inputPath: string): string[] { + return [ + "-i", + inputPath, + "-af", + `silencedetect=noise=${SILENCE_NOISE_DB}dB:d=${SILENCE_MIN_SECONDS}`, + "-f", + "null", + "-", + ]; +} + +export type SilenceSpan = { start: number; end: number }; + +/** Parses silencedetect's stderr log into [start, end] spans, in order. */ +export function parseSilenceDetectOutput(stderr: string): SilenceSpan[] { + const spans: SilenceSpan[] = []; + let pendingStart: number | null = null; + for (const line of stderr.split("\n")) { + const startMatch = line.match(/silence_start:\s*(-?[\d.]+)/); + if (startMatch) { + pendingStart = Math.max(0, Number(startMatch[1])); + continue; + } + const endMatch = line.match(/silence_end:\s*([\d.]+)/); + if (endMatch && pendingStart !== null) { + spans.push({ start: pendingStart, end: Number(endMatch[1]) }); + pendingStart = null; + } + } + return spans; +} + +/** + * Slices [start, start + duration) out of the extracted audio into its own + * file. Re-encodes rather than stream-copying so the cut lands exactly on + * `start` (a stream copy snaps to the nearest MP3 frame), which keeps each + * chunk's timestamp offset exact when the transcripts are merged. + */ +export function buildAudioChunkArgs( + inputPath: string, + outputPath: string, + start: number, + duration: number +): string[] { + return [ + "-y", + "-ss", + start.toFixed(3), + "-t", + duration.toFixed(3), + "-i", + inputPath, + ...SPEECH_AUDIO_ARGS, + outputPath, + ]; +} + +/** Proxy height (claude.md section 16). Sources at or below this, and small enough, skip the proxy. */ +export const PROXY_HEIGHT = 720; +export const PROXY_MIN_SOURCE_BYTES = 300 * 1024 * 1024; + +export function needsProxy(source: { height: number; sizeBytes: number }): boolean { + return source.height > PROXY_HEIGHT || source.sizeBytes > PROXY_MIN_SOURCE_BYTES; +} + +/** + * 720p H.264 editing proxy. A short GOP (-g 48) keeps seeks snappy in the + * browser, and +faststart puts the moov atom first so playback can begin + * before the whole file has been range-requested. + */ +export function buildProxyArgs(inputPath: string, outputPath: string): string[] { + return [ + "-y", + "-i", + inputPath, + "-vf", + `scale=-2:${PROXY_HEIGHT}`, + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "28", + "-g", + "48", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-b:a", + "96k", + "-movflags", + "+faststart", + outputPath, + ]; +} diff --git a/server/src/lib/ffmpeg/probe.ts b/server/src/lib/ffmpeg/probe.ts index f202a58..359c6f4 100644 --- a/server/src/lib/ffmpeg/probe.ts +++ b/server/src/lib/ffmpeg/probe.ts @@ -57,3 +57,36 @@ export function probeVideoDimensions(inputPath: string): Promise<{ width: number }); }); } + +/** + * Reads a media file's container duration in seconds via ffprobe -- used to + * plan transcription chunks over the extracted audio track. + */ +export function probeDuration(inputPath: string): Promise { + const binary = process.env.FFPROBE_PATH || "ffprobe"; + const args = ["-v", "error", "-show_entries", "format=duration", "-of", "json", inputPath]; + return new Promise((resolve, reject) => { + execFile(/* turbopackIgnore: true */ binary, args, { maxBuffer: 1024 * 1024 }, (error, stdout, stderr) => { + if (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + reject(new FfprobeNotFoundError()); + return; + } + const tail = stderr.trim().split("\n").slice(-20).join("\n"); + reject(new Error(`ffprobe exited with an error:\n${tail}`)); + return; + } + try { + const parsed = JSON.parse(stdout) as { format?: { duration?: string } }; + const duration = Number(parsed.format?.duration); + if (!Number.isFinite(duration) || duration <= 0) { + reject(new Error(`ffprobe returned no duration for ${inputPath}.`)); + return; + } + resolve(duration); + } catch { + reject(new Error(`Could not parse ffprobe output for ${inputPath}.`)); + } + }); + }); +} diff --git a/server/src/lib/ffmpeg/run.ts b/server/src/lib/ffmpeg/run.ts index f6a5c05..4470b80 100644 --- a/server/src/lib/ffmpeg/run.ts +++ b/server/src/lib/ffmpeg/run.ts @@ -20,14 +20,23 @@ export class FfmpegNotFoundError extends Error { * ffmpeg build lacks libass and can't burn in captions, but a full build * (e.g. `ffmpeg-full`) is installed alongside it. */ -export function runFfmpeg(args: string[]): Promise { +export async function runFfmpeg(args: string[]): Promise { + await runFfmpegCapturingStderr(args); +} + +/** + * Same as runFfmpeg, but resolves with ffmpeg's stderr on success -- for + * analysis passes (e.g. silencedetect) whose results ffmpeg only reports as + * log lines, never as an output file. + */ +export function runFfmpegCapturingStderr(args: string[]): Promise { const binary = process.env.FFMPEG_PATH || "ffmpeg"; return new Promise((resolve, reject) => { // turbopackIgnore: binary is an external system executable (resolved via // PATH or FFMPEG_PATH), never a project file -- it must not be traced. - execFile(/* turbopackIgnore: true */ binary, args, { maxBuffer: 1024 * 1024 * 10 }, (error, _stdout, stderr) => { + execFile(/* turbopackIgnore: true */ binary, args, { maxBuffer: 1024 * 1024 * 50 }, (error, _stdout, stderr) => { if (!error) { - resolve(); + resolve(stderr); return; } if ((error as NodeJS.ErrnoException).code === "ENOENT") { From 4cde01d26d96f9236e5fc1f03843f89e2c22ae14 Mon Sep 17 00:00:00 2001 From: amide-init Date: Fri, 25 Sep 2026 01:06:39 +0530 Subject: [PATCH 03/35] feat: transcribe long episodes in background chunks Podcast episodes run 30-120 minutes and several GB, which broke both the 500MB upload cap and Whisper's 25MB file limit, and held the transcribe request open for the whole run. Transcription is now a background job (lib/ai/transcription-job.ts): extract a small speech track, split it into ~10-minute chunks at detected silences so no word is cut in half, transcribe 3 chunks at a time, then shift and merge them onto the source timeline. The merge also drops Whisper's end-of-clip repeats: a copy of the previous sentence made of zero-length words, which chunking would otherwise produce at every chunk end. Genuine repeated phrases with real timing are kept. POST /transcribe now returns 202 + jobId, and the upload screen polls it and shows chunk progress. Uploads use XHR so they can report progress, and the upload cap is 10GB by default, configurable with MAX_UPLOAD_MB. Jobs still running at startup were interrupted by a restart, so they are marked failed; otherwise they would block retries forever. Co-Authored-By: Claude Opus 5.5 --- client/src/components/editor/UploadScreen.tsx | 92 +++++++++++-- server/.env.example | 4 + server/src/index.ts | 18 ++- server/src/lib/ai/chunking.test.ts | 129 ++++++++++++++++++ server/src/lib/ai/chunking.ts | 113 +++++++++++++++ server/src/lib/ai/transcription-job.ts | 115 ++++++++++++++++ server/src/lib/limits.ts | 13 ++ server/src/routes/transcribe.ts | 66 +++++---- server/src/routes/upload.ts | 13 +- 9 files changed, 510 insertions(+), 53 deletions(-) create mode 100644 server/src/lib/ai/chunking.test.ts create mode 100644 server/src/lib/ai/chunking.ts create mode 100644 server/src/lib/ai/transcription-job.ts create mode 100644 server/src/lib/limits.ts diff --git a/client/src/components/editor/UploadScreen.tsx b/client/src/components/editor/UploadScreen.tsx index 8bf3aff..fa69350 100644 --- a/client/src/components/editor/UploadScreen.tsx +++ b/client/src/components/editor/UploadScreen.tsx @@ -18,10 +18,65 @@ async function postJson(url: string, body: unknown): Promise { return res.json(); } +type TranscriptionJob = { + status: "queued" | "processing" | "completed" | "failed"; + totalChunks: number; + completedChunks: number; + error: string | null; +}; + +const POLL_INTERVAL_MS = 1500; + +/** + * XHR rather than fetch: fetch has no upload-progress events, and a + * multi-GB podcast upload with no progress bar looks hung. + */ +function uploadWithProgress( + url: string, + file: File, + onProgress: (fraction: number) => void +): Promise<{ success: true } | ApiError> { + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.open("POST", url); + xhr.setRequestHeader("Content-Type", file.type); + xhr.upload.onprogress = (e) => { + if (e.lengthComputable) onProgress(e.loaded / e.total); + }; + xhr.onload = () => { + try { + resolve(JSON.parse(xhr.responseText)); + } catch { + reject(new Error("Invalid upload response")); + } + }; + xhr.onerror = () => reject(new Error("Upload failed")); + xhr.send(file); + }); +} + +/** Polls a background transcription job until it finishes, reporting chunk progress along the way. */ +async function waitForTranscription( + projectId: string, + jobId: string, + onProgress: (job: TranscriptionJob) => void +): Promise { + for (;;) { + const res = await fetch(`/api/projects/${projectId}/transcribe/${jobId}`); + const data: { success: true; job: TranscriptionJob } | ApiError = await res.json(); + if (!data.success) throw new Error(data.error.message); + onProgress(data.job); + if (data.job.status === "completed" || data.job.status === "failed") return data.job; + await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS)); + } +} + export function UploadScreen() { const navigate = useNavigate(); const fileInputRef = useRef(null); const [nameDraft, setNameDraft] = useState("Untitled Project"); + const [uploadFraction, setUploadFraction] = useState(0); + const [chunkProgress, setChunkProgress] = useState<{ done: number; total: number } | null>(null); const status = useProjectStore((s) => s.status); const error = useProjectStore((s) => s.error); @@ -41,16 +96,13 @@ export function UploadScreen() { const projectId = created.project.id; setStatus("uploading"); + setUploadFraction(0); try { - const uploadRes = await fetch( + const uploaded = await uploadWithProgress( `/api/projects/${projectId}/upload?filename=${encodeURIComponent(file.name)}`, - { - method: "POST", - headers: { "Content-Type": file.type }, - body: file, - } + file, + setUploadFraction ); - const uploaded: { success: true } | ApiError = await uploadRes.json(); if (!uploaded.success) { setErrorState(uploaded.error.message); return; @@ -61,11 +113,21 @@ export function UploadScreen() { } setStatus("transcribing"); + setChunkProgress(null); try { - const transcribeRes = await fetch(`/api/projects/${projectId}/transcribe`, { method: "POST" }); - const transcribed: { success: true } | ApiError = await transcribeRes.json(); - if (!transcribed.success) { - setErrorState(transcribed.error.message); + const started = await postJson<{ success: true; jobId: string }>( + `/api/projects/${projectId}/transcribe`, + {} + ); + if (!started.success) { + setErrorState(started.error.message); + return; + } + const job = await waitForTranscription(projectId, started.jobId, (j) => + setChunkProgress(j.totalChunks > 0 ? { done: j.completedChunks, total: j.totalChunks } : null) + ); + if (job.status === "failed") { + setErrorState("The video could not be transcribed. Please try again."); return; } } catch { @@ -82,9 +144,11 @@ export function UploadScreen() { status === "creating" ? "Creating project…" : status === "uploading" - ? "Uploading…" + ? `Uploading… ${Math.round(uploadFraction * 100)}%` : status === "transcribing" - ? "Transcribing…" + ? chunkProgress && chunkProgress.total > 1 + ? `Transcribing… ${chunkProgress.done}/${chunkProgress.total}` + : "Transcribing…" : "Choose video"; return ( @@ -140,7 +204,7 @@ export function UploadScreen() {

{error}

) : (

- mp4, mov, webm, m4a — up to 25MB for now + mp4, mov, webm — long episodes welcome; transcription runs in chunks

)} diff --git a/server/.env.example b/server/.env.example index eebab08..5ae9797 100644 --- a/server/.env.example +++ b/server/.env.example @@ -28,3 +28,7 @@ CORS_ORIGIN="http://localhost:5173" # "Burn in captions" will fail with "No such filter: 'subtitles'" if your # default ffmpeg build doesn't include libass. # FFMPEG_PATH="/opt/homebrew/opt/ffmpeg-full/bin/ffmpeg" + +# Optional. Largest accepted video upload, in MB (default 10240 = 10GB). +# Uploads stream straight to disk, so this is about disk space, not memory. +# MAX_UPLOAD_MB=10240 diff --git a/server/src/index.ts b/server/src/index.ts index 2c806c7..33a8e50 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -15,6 +15,8 @@ import { captionsRoute } from "@/routes/captions"; import { logoRoute } from "@/routes/logo"; import { renderRoute } from "@/routes/render"; import { settingsRoute } from "@/routes/settings"; +import { prisma } from "@/lib/db/client"; +import { getMaxUploadBytes } from "@/lib/limits"; const app = new Hono(); @@ -50,14 +52,22 @@ if (existsSync(clientDistDir)) { app.get("*", serveStatic({ root: clientDistDir, path: "index.html" })); } +// Jobs run in-process, so any still "queued"/"processing" at startup were +// interrupted by a restart and will never finish -- mark them failed so the +// client stops polling and a retry isn't blocked as "already running". +await prisma.transcriptionJob.updateMany({ + where: { status: { in: ["queued", "processing"] } }, + data: { status: "failed", error: "Interrupted by a server restart. Please try again." }, +}); + const port = Number(process.env.PORT ?? 3001); logger.info(`server listening on :${port}`); export default { port, fetch: app.fetch, - // Video uploads can be hundreds of MB -- raise Bun's default request body - // size ceiling to match the upload route's own 500MB cap (see - // src/routes/upload.ts) instead of relying on Bun's built-in default. - maxRequestBodySize: 500 * 1024 * 1024, + // Video uploads can be many GB -- raise Bun's default request body size + // ceiling to match the upload route's own cap (see src/lib/limits.ts) + // instead of relying on Bun's built-in default. + maxRequestBodySize: getMaxUploadBytes(), }; diff --git a/server/src/lib/ai/chunking.test.ts b/server/src/lib/ai/chunking.test.ts new file mode 100644 index 0000000..f2aacce --- /dev/null +++ b/server/src/lib/ai/chunking.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it } from "vitest"; +import { isDegenerateRepeat, mergeChunkTranscripts, planChunks } from "@/lib/ai/chunking"; +import type { Transcript } from "@/types/transcript"; + +const opts = { targetSeconds: 600, searchWindowSeconds: 90 }; + +describe("planChunks", () => { + it("returns a single chunk for audio shorter than a chunk", () => { + expect(planChunks(120, [], opts)).toEqual([{ index: 0, start: 0, end: 120 }]); + }); + + it("returns nothing for zero-length audio", () => { + expect(planChunks(0, [], opts)).toEqual([]); + }); + + it("folds a short tail into the last chunk instead of making a sliver", () => { + expect(planChunks(700, [], opts)).toEqual([{ index: 0, start: 0, end: 700 }]); + }); + + it("hard-cuts at the target when there's no silence nearby", () => { + expect(planChunks(1500, [], opts)).toEqual([ + { index: 0, start: 0, end: 600 }, + { index: 1, start: 600, end: 1200 }, + { index: 2, start: 1200, end: 1500 }, + ]); + }); + + it("puts the boundary mid-silence, picking the silence closest to the target", () => { + const silences = [ + { start: 540, end: 541 }, // 59.5s early + { start: 620, end: 622 }, // 21s late -- closest + { start: 900, end: 901 }, + ]; + const chunks = planChunks(1500, silences, opts); + expect(chunks[0]).toEqual({ index: 0, start: 0, end: 621 }); + expect(chunks[1].start).toBe(621); + }); + + it("ignores silences outside the search window", () => { + const chunks = planChunks(1500, [{ start: 300, end: 301 }], opts); + expect(chunks[0].end).toBe(600); + }); + + it("covers the whole duration with contiguous chunks", () => { + const silences = Array.from({ length: 40 }, (_, i) => ({ start: i * 173 + 50, end: i * 173 + 51 })); + const chunks = planChunks(7200, silences, opts); + expect(chunks[0].start).toBe(0); + expect(chunks[chunks.length - 1].end).toBe(7200); + for (let i = 1; i < chunks.length; i++) { + expect(chunks[i].start).toBe(chunks[i - 1].end); + expect(chunks[i].index).toBe(i); + } + }); +}); + +function part(offset: number, words: [string, number, number][]): { offset: number; transcript: Transcript } { + return { + offset, + transcript: { + id: "x", + segments: [ + { + id: "segment-0", + start: words[0][1], + end: words[words.length - 1][2], + text: words.map((w) => w[0]).join(" "), + words: words.map(([text, start, end], i) => ({ id: `word-${i}`, text, start, end })), + }, + ], + }, + }; +} + +describe("mergeChunkTranscripts", () => { + it("shifts timestamps by each chunk's offset and orders by time", () => { + const merged = mergeChunkTranscripts([ + part(600, [["world", 1, 1.5]]), + part(0, [["hello", 2, 2.4]]), + ]); + expect(merged.segments.map((s) => [s.text, s.start, s.end])).toEqual([ + ["hello", 2, 2.4], + ["world", 601, 601.5], + ]); + expect(merged.segments[1].words[0]).toMatchObject({ start: 601, end: 601.5 }); + }); + + it("re-numbers segment and word ids so they're unique across chunks", () => { + const merged = mergeChunkTranscripts([ + part(0, [["a", 0, 1], ["b", 1, 2]]), + part(600, [["c", 0, 1]]), + ]); + expect(merged.segments.map((s) => s.id)).toEqual(["segment-0", "segment-1"]); + expect(merged.segments.flatMap((s) => s.words.map((w) => w.id))).toEqual(["word-0", "word-1", "word-2"]); + }); + + it("handles chunks with no speech", () => { + const merged = mergeChunkTranscripts([{ offset: 0, transcript: { id: "x", segments: [] } }, part(600, [["hi", 0, 1]])]); + expect(merged.segments).toHaveLength(1); + expect(merged.segments[0].start).toBe(600); + }); +}); + +describe("isDegenerateRepeat", () => { + const seg = (text: string, times: [number, number][]) => ({ + id: "s", + start: times[0][0], + end: times[times.length - 1][1], + text, + words: times.map(([start, end], i) => ({ id: `w${i}`, text: "x", start, end })), + }); + + it("flags a repeat of the previous sentence made of zero-length words", () => { + const previous = seg("This is a test.", [[1, 1.3], [1.3, 1.5], [1.5, 1.6], [1.6, 2]]); + const repeat = seg("this is a test", [[2, 2.2], [2.2, 2.2], [2.2, 2.2], [2.2, 2.2]]); + expect(isDegenerateRepeat(previous, repeat)).toBe(true); + }); + + it("keeps a genuinely repeated phrase with real timing", () => { + const previous = seg("Yeah.", [[1, 1.4]]); + const repeat = seg("Yeah.", [[1.6, 2]]); + expect(isDegenerateRepeat(previous, repeat)).toBe(false); + }); + + it("keeps zero-length words when the text differs", () => { + const previous = seg("Hello there.", [[1, 1.3], [1.3, 1.6]]); + const other = seg("General Kenobi.", [[2, 2], [2, 2]]); + expect(isDegenerateRepeat(previous, other)).toBe(false); + }); +}); diff --git a/server/src/lib/ai/chunking.ts b/server/src/lib/ai/chunking.ts new file mode 100644 index 0000000..64ce498 --- /dev/null +++ b/server/src/lib/ai/chunking.ts @@ -0,0 +1,113 @@ +import type { SilenceSpan } from "@/lib/ffmpeg/audio"; +import type { Transcript, TranscriptSegment } from "@/types/transcript"; + +/** + * Splitting long audio for Whisper, whose API rejects uploads over 25MB. + * Pure functions only -- the ffmpeg/OpenAI side lives in + * lib/ai/transcription-job.ts -- so the boundary and offset logic is + * unit-testable (claude.md section 25). + */ + +export type AudioChunk = { index: number; start: number; end: number }; + +export type ChunkPlanOptions = { + /** Preferred chunk length in seconds. */ + targetSeconds: number; + /** How far from the target a silence may be and still be used as the boundary. */ + searchWindowSeconds: number; +}; + +/** + * Ten minutes at the extracted 32kbps is ~2.4MB, far under Whisper's cap; + * the length is chosen for progress granularity and retry cost, not size. + */ +export const DEFAULT_CHUNK_PLAN: ChunkPlanOptions = { targetSeconds: 600, searchWindowSeconds: 90 }; + +/** + * Splits [0, duration] into ~targetSeconds chunks, placing each boundary in + * the middle of the silence nearest the target, so no word is cut in half + * and transcribed twice or not at all. Falls back to a hard cut at the + * target when there's no silence within the search window. + */ +export function planChunks( + duration: number, + silences: SilenceSpan[], + options: ChunkPlanOptions = DEFAULT_CHUNK_PLAN +): AudioChunk[] { + const { targetSeconds, searchWindowSeconds } = options; + if (duration <= 0) return []; + + const chunks: AudioChunk[] = []; + let cursor = 0; + // A trailing sliver shorter than this is folded into the last chunk instead. + const minTail = targetSeconds / 4; + + while (duration - cursor > targetSeconds + minTail) { + const ideal = cursor + targetSeconds; + let boundary = ideal; + let bestDistance = Infinity; + for (const silence of silences) { + const mid = (silence.start + silence.end) / 2; + const distance = Math.abs(mid - ideal); + const leavesRealChunk = mid - cursor >= minTail && duration - mid >= minTail; + if (distance <= searchWindowSeconds && distance < bestDistance && leavesRealChunk) { + bestDistance = distance; + boundary = mid; + } + } + chunks.push({ index: chunks.length, start: cursor, end: boundary }); + cursor = boundary; + } + + chunks.push({ index: chunks.length, start: cursor, end: duration }); + return chunks; +} + +/** + * Whisper sometimes repeats a clip's last sentence into trailing silence, + * with every repeated word collapsed to (near) zero duration -- and chunking + * gives it one clip end per chunk to do this at. Only a segment that both + * repeats the previous one's text *and* is mostly zero-length words is + * dropped, so a genuinely repeated phrase ("yeah. yeah.") with real timing + * survives. + */ +export function isDegenerateRepeat(previous: TranscriptSegment | undefined, segment: TranscriptSegment): boolean { + if (!previous || segment.words.length === 0) return false; + const normalize = (text: string) => text.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim(); + if (normalize(previous.text) !== normalize(segment.text)) return false; + const zeroLength = segment.words.filter((w) => w.end - w.start < 0.02).length; + return zeroLength / segment.words.length >= 0.6; +} + +/** + * Stitches per-chunk transcripts (each timed from 0 within its own chunk) + * into one transcript on the source timeline: shifts every timestamp by its + * chunk's start, and re-numbers segment/word ids so they stay unique across + * chunks. Drops Whisper's end-of-clip repeats (see isDegenerateRepeat). + * Input order doesn't matter; output is ordered by time. + */ +export function mergeChunkTranscripts(parts: { offset: number; transcript: Transcript }[]): Transcript { + const ordered = [...parts].sort((a, b) => a.offset - b.offset); + const segments: TranscriptSegment[] = []; + let wordCounter = 0; + + for (const { offset, transcript } of ordered) { + for (const segment of transcript.segments) { + if (isDegenerateRepeat(segments[segments.length - 1], segment)) continue; + segments.push({ + ...segment, + id: `segment-${segments.length}`, + start: segment.start + offset, + end: segment.end + offset, + words: segment.words.map((word) => ({ + ...word, + id: `word-${wordCounter++}`, + start: word.start + offset, + end: word.end + offset, + })), + }); + } + } + + return { id: crypto.randomUUID(), segments }; +} diff --git a/server/src/lib/ai/transcription-job.ts b/server/src/lib/ai/transcription-job.ts new file mode 100644 index 0000000..ac1afc0 --- /dev/null +++ b/server/src/lib/ai/transcription-job.ts @@ -0,0 +1,115 @@ +import { mkdir, readFile, rm } from "node:fs/promises"; +import path from "node:path"; +import { logger } from "@/lib/logger"; +import { prisma } from "@/lib/db/client"; +import { resolveInDataDir, statAsset } from "@/lib/storage/local"; +import { runFfmpeg, runFfmpegCapturingStderr } from "@/lib/ffmpeg/run"; +import { probeDuration } from "@/lib/ffmpeg/probe"; +import { + buildAudioChunkArgs, + buildExtractAudioArgs, + buildSilenceDetectArgs, + parseSilenceDetectOutput, +} from "@/lib/ffmpeg/audio"; +import { mergeChunkTranscripts, planChunks, type AudioChunk } from "@/lib/ai/chunking"; +import { transcribeFile } from "@/lib/ai/transcribe"; +import type { Transcript } from "@/types/transcript"; + +/** Parallel Whisper calls per job -- enough to cut a long episode's wait, few enough to stay clear of rate limits. */ +const CHUNK_CONCURRENCY = 3; + +async function mapWithConcurrency(items: T[], limit: number, fn: (item: T) => Promise): Promise { + const results = new Array(items.length); + let next = 0; + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + while (next < items.length) { + const i = next++; + results[i] = await fn(items[i]); + } + }); + await Promise.all(workers); + return results; +} + +/** + * Runs one transcription job to completion, updating its TranscriptionJob + * row along the way (queued -> processing -> completed/failed). Called + * fire-and-forget from the transcribe route, like runRenderJob, so a + * 2-hour episode never holds an API request open (claude.md section 14). + * + * Pipeline: extract a small speech-quality audio track (kept as the + * project's "audio" asset -- the timeline waveform reads it too), find + * silences, split into ~10-minute chunks at those silences, transcribe the + * chunks, then shift and merge them onto the source timeline. + */ +export async function runTranscriptionJob(jobId: string): Promise { + const chunkPaths: string[] = []; + try { + const job = await prisma.transcriptionJob.update({ where: { id: jobId }, data: { status: "processing" } }); + const projectId = job.projectId; + + const originalAsset = await prisma.asset.findFirst({ where: { projectId, kind: "original" } }); + if (!originalAsset) throw new Error("No source video for this project."); + + const audioDir = path.posix.join("projects", projectId, "audio"); + await mkdir(resolveInDataDir(audioDir), { recursive: true }); + + const audioRelativePath = path.posix.join(audioDir, "speech.mp3"); + const audioPath = resolveInDataDir(audioRelativePath); + await runFfmpeg(buildExtractAudioArgs(resolveInDataDir(originalAsset.filePath), audioPath)); + + const audioStats = await statAsset(audioRelativePath); + await prisma.asset.deleteMany({ where: { projectId, kind: "audio" } }); + await prisma.asset.create({ + data: { + projectId, + kind: "audio", + filePath: audioRelativePath, + mimeType: "audio/mpeg", + sizeBytes: audioStats.size, + }, + }); + + const duration = await probeDuration(audioPath); + const silences = parseSilenceDetectOutput(await runFfmpegCapturingStderr(buildSilenceDetectArgs(audioPath))); + const chunks = planChunks(duration, silences); + await prisma.transcriptionJob.update({ where: { id: jobId }, data: { totalChunks: chunks.length } }); + + const parts = await mapWithConcurrency(chunks, CHUNK_CONCURRENCY, async (chunk: AudioChunk) => { + const chunkRelativePath = path.posix.join(audioDir, `chunk-${jobId}-${chunk.index}.mp3`); + const chunkPath = resolveInDataDir(chunkRelativePath); + chunkPaths.push(chunkPath); + + await runFfmpeg(buildAudioChunkArgs(audioPath, chunkPath, chunk.start, chunk.end - chunk.start)); + const bytes = await readFile(chunkPath); + const transcript: Transcript = await transcribeFile( + new File([new Uint8Array(bytes)], path.basename(chunkPath), { type: "audio/mpeg" }) + ); + await rm(chunkPath, { force: true }); + + await prisma.transcriptionJob.update({ where: { id: jobId }, data: { completedChunks: { increment: 1 } } }); + return { offset: chunk.start, transcript }; + }); + + const merged = mergeChunkTranscripts(parts); + const segmentsJson = JSON.stringify(merged.segments); + await prisma.transcript.upsert({ + where: { projectId }, + create: { projectId, segmentsJson }, + update: { segmentsJson }, + }); + + await prisma.transcriptionJob.update({ where: { id: jobId }, data: { status: "completed", error: null } }); + } catch (err) { + logger.error(`Transcription job ${jobId} failed:`, err); + await prisma.transcriptionJob.update({ + where: { id: jobId }, + data: { + status: "failed", + error: err instanceof Error ? err.message : "Transcription failed for an unknown reason.", + }, + }); + } finally { + await Promise.all(chunkPaths.map((p) => rm(p, { force: true }))); + } +} diff --git a/server/src/lib/limits.ts b/server/src/lib/limits.ts new file mode 100644 index 0000000..a09ae3e --- /dev/null +++ b/server/src/lib/limits.ts @@ -0,0 +1,13 @@ +/** + * Largest accepted source upload. Video podcasts routinely run to several + * GB, and the upload route streams straight to disk, so memory isn't the + * constraint -- disk space is. Overridable with MAX_UPLOAD_MB for + * self-hosters with smaller (or larger) disks. + */ +const DEFAULT_MAX_UPLOAD_MB = 10 * 1024; + +export function getMaxUploadBytes(): number { + const fromEnv = Number(process.env.MAX_UPLOAD_MB); + const mb = Number.isFinite(fromEnv) && fromEnv > 0 ? fromEnv : DEFAULT_MAX_UPLOAD_MB; + return mb * 1024 * 1024; +} diff --git a/server/src/routes/transcribe.ts b/server/src/routes/transcribe.ts index 69e582a..0e8f230 100644 --- a/server/src/routes/transcribe.ts +++ b/server/src/routes/transcribe.ts @@ -1,16 +1,16 @@ import { Hono } from "hono"; -import { readFile } from "node:fs/promises"; -import path from "node:path"; -import { logger } from "@/lib/logger"; import { prisma } from "@/lib/db/client"; -import { resolveInDataDir } from "@/lib/storage/local"; -import { transcribeFile } from "@/lib/ai/transcribe"; +import { runTranscriptionJob } from "@/lib/ai/transcription-job"; import { errorResponse } from "@/lib/http"; export const transcribeRoute = new Hono(); -const MAX_TRANSCRIBE_BYTES = 25 * 1024 * 1024; // Whisper API limit - +/** + * Starts a background transcription job and returns its id immediately (202); + * the client polls GET /:id/transcribe/:jobId for progress. The job extracts + * a small audio track and transcribes it in chunks (lib/ai/transcription-job.ts), + * so source size no longer runs into Whisper's 25MB upload cap. + */ transcribeRoute.post("/:id/transcribe", async (c) => { const id = c.req.param("id"); @@ -18,32 +18,40 @@ transcribeRoute.post("/:id/transcribe", async (c) => { if (!asset) { return errorResponse(c, "NO_VIDEO", "Upload a video before transcribing.", 400); } - if (asset.sizeBytes > MAX_TRANSCRIBE_BYTES) { - return errorResponse( - c, - "FILE_TOO_LARGE", - `Video exceeds the ${MAX_TRANSCRIBE_BYTES / (1024 * 1024)}MB transcription limit.`, - 413 - ); + + const running = await prisma.transcriptionJob.findFirst({ + where: { projectId: id, status: { in: ["queued", "processing"] } }, + }); + if (running) { + return errorResponse(c, "ALREADY_TRANSCRIBING", "This project is already being transcribed.", 409); } - try { - const bytes = await readFile(resolveInDataDir(asset.filePath)); - const file = new File([new Uint8Array(bytes)], path.basename(asset.filePath), { - type: asset.mimeType, - }); + const job = await prisma.transcriptionJob.create({ data: { projectId: id, status: "queued" } }); + + // Fire-and-forget, same as the render route: runTranscriptionJob catches + // every error itself and records it on the job row. + void runTranscriptionJob(job.id); - const transcript = await transcribeFile(file); + return c.json({ success: true, jobId: job.id }, 202); +}); - await prisma.transcript.upsert({ - where: { projectId: id }, - create: { projectId: id, segmentsJson: JSON.stringify(transcript.segments) }, - update: { segmentsJson: JSON.stringify(transcript.segments) }, - }); +transcribeRoute.get("/:id/transcribe/:jobId", async (c) => { + const id = c.req.param("id"); + const jobId = c.req.param("jobId"); - return c.json({ success: true, transcript }); - } catch (err) { - logger.error("Transcription failed:", err); - return errorResponse(c, "TRANSCRIPTION_FAILED", "The video could not be transcribed. Please try again.", 502); + const job = await prisma.transcriptionJob.findUnique({ where: { id: jobId } }); + if (!job || job.projectId !== id) { + return errorResponse(c, "NOT_FOUND", "Transcription job not found.", 404); } + + return c.json({ + success: true, + job: { + id: job.id, + status: job.status, + totalChunks: job.totalChunks, + completedChunks: job.completedChunks, + error: job.error, + }, + }); }); diff --git a/server/src/routes/upload.ts b/server/src/routes/upload.ts index 05b2d7e..dcfe041 100644 --- a/server/src/routes/upload.ts +++ b/server/src/routes/upload.ts @@ -2,11 +2,10 @@ import { Hono } from "hono"; import { prisma } from "@/lib/db/client"; import { writeAssetStream } from "@/lib/storage/local"; import { errorResponse } from "@/lib/http"; +import { getMaxUploadBytes } from "@/lib/limits"; export const uploadRoute = new Hono(); -const MAX_FILE_BYTES = 500 * 1024 * 1024; // 500MB local upload cap - /** * POST /api/projects/:id/upload?filename=video.mp4 * @@ -40,11 +39,12 @@ uploadRoute.post("/:id/upload", async (c) => { if (!Number.isFinite(contentLength) || contentLength <= 0) { return errorResponse(c, "MISSING_CONTENT_LENGTH", "A Content-Length header is required.", 400); } - if (contentLength > MAX_FILE_BYTES) { + const maxBytes = getMaxUploadBytes(); + if (contentLength > maxBytes) { return errorResponse( c, "FILE_TOO_LARGE", - `File exceeds the ${MAX_FILE_BYTES / (1024 * 1024)}MB upload limit.`, + `File exceeds the ${Math.round(maxBytes / (1024 * 1024))}MB upload limit.`, 413 ); } @@ -56,8 +56,9 @@ uploadRoute.post("/:id/upload", async (c) => { const { relativePath, sizeBytes } = await writeAssetStream(id, "original", filename, body); - // Only one "original" asset per project for v1 -- replace any previous one. - await prisma.asset.deleteMany({ where: { projectId: id, kind: "original" } }); + // Only one "original" asset per project for v1 -- replace any previous one, + // along with the audio track extracted from it. + await prisma.asset.deleteMany({ where: { projectId: id, kind: { in: ["original", "audio"] } } }); const asset = await prisma.asset.create({ data: { projectId: id, From 2b60a2c7c5dff15176b8a5d025bb8a3fb780f478 Mon Sep 17 00:00:00 2001 From: amide-init Date: Fri, 25 Sep 2026 01:06:39 +0530 Subject: [PATCH 04/35] feat: play a 720p proxy and draw the waveform from the speech track Multi-GB 1080p/4K podcast originals stutter when the browser plays them, and the timeline waveform fetched and decoded the entire video. For a 2-hour episode that is over 1GB of decoded audio at 48kHz. After upload, a 720p proxy is built in the background when the source is above 720p or over 300MB (claude.md section 16). It is served at a separate ?variant=proxy URL rather than swapped in behind /video, so a proxy that finishes mid-session can't change the bytes under an in-flight Range request. Proxy creation is best-effort: on failure the editor plays the original. Export always uses the original. The waveform now decodes the small extracted speech track (GET /audio) at 8kHz, and falls back to the video for projects transcribed before the track existed. Co-Authored-By: Claude Opus 5.5 --- .../src/components/editor/EditorHydrator.tsx | 1 + client/src/components/timeline/Timeline.tsx | 5 +- client/src/lib/timeline/useWaveform.ts | 12 +++-- client/src/routes/EditorRoute.tsx | 4 +- client/src/stores/project-store.ts | 5 ++ server/src/lib/ffmpeg/proxy-job.ts | 51 +++++++++++++++++++ server/src/routes/projects.ts | 7 ++- server/src/routes/upload.ts | 8 ++- server/src/routes/video.ts | 23 ++++++++- 9 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 server/src/lib/ffmpeg/proxy-job.ts diff --git a/client/src/components/editor/EditorHydrator.tsx b/client/src/components/editor/EditorHydrator.tsx index 51676a9..092adf7 100644 --- a/client/src/components/editor/EditorHydrator.tsx +++ b/client/src/components/editor/EditorHydrator.tsx @@ -24,6 +24,7 @@ type HydrateProject = { logoPaddingY: number; logoOpacity: number; videoUrl: string | null; + audioUrl: string | null; logoUrl: string | null; }; diff --git a/client/src/components/timeline/Timeline.tsx b/client/src/components/timeline/Timeline.tsx index cb9f65e..2dd5617 100644 --- a/client/src/components/timeline/Timeline.tsx +++ b/client/src/components/timeline/Timeline.tsx @@ -39,7 +39,10 @@ export function Timeline() { const silenceGaps = useTranscriptStore((s) => s.silenceGaps); const videoUrl = useProjectStore((s) => s.videoUrl); - const audioBuffer = useWaveform(videoUrl); + const audioUrl = useProjectStore((s) => s.audioUrl); + // Prefer the small extracted speech track; projects transcribed before it + // existed fall back to decoding the video's own audio. + const audioBuffer = useWaveform(audioUrl ?? videoUrl); const operations = useTimelineStore((s) => s.operations); const undo = useTimelineStore((s) => s.undo); diff --git a/client/src/lib/timeline/useWaveform.ts b/client/src/lib/timeline/useWaveform.ts index 6bd0137..29cbb2b 100644 --- a/client/src/lib/timeline/useWaveform.ts +++ b/client/src/lib/timeline/useWaveform.ts @@ -8,6 +8,8 @@ import { useEffect, useState } from "react"; * if the source has no audio track — callers should treat that as "no * waveform to show" rather than an error. */ +const WAVEFORM_SAMPLE_RATE = 8000; + export function useWaveform(videoUrl: string | null): AudioBuffer | null { const [buffer, setBuffer] = useState(null); @@ -20,9 +22,11 @@ export function useWaveform(videoUrl: string | null): AudioBuffer | null { // completion for no reason, doubling load on a video route that's // already juggling concurrent Range requests from the