From 28f3b43ddfad16e6e52bcd931467a3995647f6f2 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 23 Oct 2026 21:44:21 -0400 Subject: [PATCH 1/5] fix(studio-server): project history no longer copies video, audio or huge files --- .../studio-server/src/history/blobStore.ts | 12 ++ .../src/history/projectHistory.test.ts | 46 +++---- .../src/history/projectHistory.ts | 38 +++++- .../src/history/uncopied.test.ts | 118 ++++++++++++++++++ .../studio-server/src/history/uncopied.ts | 26 ++++ 5 files changed, 213 insertions(+), 27 deletions(-) create mode 100644 packages/studio-server/src/history/uncopied.test.ts create mode 100644 packages/studio-server/src/history/uncopied.ts diff --git a/packages/studio-server/src/history/blobStore.ts b/packages/studio-server/src/history/blobStore.ts index 67159aff117..c6cefb4a18a 100644 --- a/packages/studio-server/src/history/blobStore.ts +++ b/packages/studio-server/src/history/blobStore.ts @@ -9,6 +9,7 @@ import { atomicTempPath } from "@hyperframes/core/atomic-file"; export interface BlobStore { /** Copies the file in (a clone where the file system can) and returns the copy's hash. */ put(absPath: string): Promise; + reference(absPath: string): Promise; has(hash: string): boolean; read(hash: string): Promise; /** Writes the blob's bytes to `absPath` by clone-or-copy and rename, so a reader never sees half a file. */ @@ -16,6 +17,7 @@ export interface BlobStore { bytes(): number; size(hash: string): number; prune(keep: ReadonlySet): Promise; + drop(hashes: Iterable): Promise; } const BLOB_HASH = /^[0-9a-f]{64}$/; @@ -87,12 +89,22 @@ export async function openBlobStore(dir: string): Promise { total += size; return hash; }, + reference: (absPath) => hashFile(absPath), has: (hash) => sizes.has(hash), read: async (hash) => readBlob(dir, hash), writeTo: async (hash, absPath, beforeReplace) => cloneOrCopy(pathOf(hash), absPath, beforeReplace), bytes: () => total, size: (hash) => sizes.get(hash) ?? 0, + async drop(hashes) { + for (const hash of hashes) { + const size = sizes.get(hash); + if (size === undefined) continue; + await rm(pathOf(hash), { force: true }); + sizes.delete(hash); + total -= size; + } + }, async prune(keep) { for (const [hash, size] of sizes) { if (keep.has(hash)) continue; diff --git a/packages/studio-server/src/history/projectHistory.test.ts b/packages/studio-server/src/history/projectHistory.test.ts index 7fb199317dc..9ee0f0fad00 100644 --- a/packages/studio-server/src/history/projectHistory.test.ts +++ b/packages/studio-server/src/history/projectHistory.test.ts @@ -48,13 +48,13 @@ vi.mock("./blobStore", async (importOriginal) => { return { ...store, put: async (path: string) => { - if (path.endsWith(".mp4")) { + if (path.endsWith(".bin")) { await mediaCopy.held; mediaCopy.copies.push(basename(path)); if (mediaCopy.fails?.(mediaCopy.copies.length)) throw new Error("The disk is full."); } const hash = await store.put(path); - if (path.endsWith(".mp4")) await mediaCopy.stored?.(); + if (path.endsWith(".bin")) await mediaCopy.stored?.(); return hash; }, }; @@ -306,10 +306,10 @@ describe("openProjectHistory", () => { it("a ledger made while closed is still a change after the project's media was copied in the background", async () => { const { history, write, has, projectDir, historyRoot } = await project( - { "index.html": "A", "clip.mp4": Buffer.alloc(2 * 1024 ** 2, 7) }, + { "index.html": "A", "clip.bin": Buffer.alloc(2 * 1024 ** 2, 7) }, { quietMs: 30 }, ); - await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.mp4"]), { + await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.bin"]), { timeout: 10_000, }); await history.close(); @@ -331,12 +331,12 @@ describe("openProjectHistory", () => { mediaCopy.held = new Promise((resolve) => (release = resolve)); const { history, historyRoot } = await project({ "index.html": "A", - "clip.mp4": Buffer.alloc(2 * 1024 ** 2, 7), + "clip.bin": Buffer.alloc(2 * 1024 ** 2, 7), }); const adopting = join(historyRoot, history.projectId, "adopting.json"); - expect(JSON.parse(readFileSync(adopting, "utf-8"))).toEqual(["clip.mp4"]); + expect(JSON.parse(readFileSync(adopting, "utf-8"))).toEqual(["clip.bin"]); release(); - await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.mp4"]), { + await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.bin"]), { timeout: 10_000, }); expect(existsSync(adopting)).toBe(false); @@ -344,7 +344,7 @@ describe("openProjectHistory", () => { it("keeps media whose recording failed for the next open, rather than log it as added", async () => { const { history, write, historyRoot } = await project( - { "index.html": "A", "clip.mp4": Buffer.alloc(2 * 1024 ** 2, 7) }, + { "index.html": "A", "clip.bin": Buffer.alloc(2 * 1024 ** 2, 7) }, { budgetBytes: 1, quietMs: 30 }, ); // The edit's prune takes the first copy; storing it again then fails. @@ -356,18 +356,18 @@ describe("openProjectHistory", () => { await vi.waitFor(() => expect(mediaCopy.copies).toHaveLength(2), { timeout: 10_000 }); await history.flush(); expect(history.list().flatMap((entry) => entry.files.map((file) => file.path))).not.toContain( - "clip.mp4", + "clip.bin", ); const adopting = join(historyRoot, history.projectId, "adopting.json"); - expect(JSON.parse(readFileSync(adopting, "utf-8"))).toEqual(["clip.mp4"]); + expect(JSON.parse(readFileSync(adopting, "utf-8"))).toEqual(["clip.bin"]); }); it("never copies the media of a folder that replaced the project", async () => { let release = () => {}; mediaCopy.held = new Promise((resolve) => (release = resolve)); const media = { - "a.mp4": Buffer.alloc(2 * 1024 ** 2, 1), - "b.mp4": Buffer.alloc(2 * 1024 ** 2, 2), + "a.bin": Buffer.alloc(2 * 1024 ** 2, 1), + "b.bin": Buffer.alloc(2 * 1024 ** 2, 2), }; const { history, write, projectDir } = await project({ "index.html": "A", ...media }); renameSync(projectDir, `${projectDir}-moved`); @@ -377,7 +377,7 @@ describe("openProjectHistory", () => { release(); await history.close(); // Only the copy already running when the folder changed may finish. - expect(mediaCopy.copies).toEqual(["a.mp4"]); + expect(mediaCopy.copies).toEqual(["a.bin"]); }); it("undoes a clip replaced while closed, though the history closed before its copy finished", async () => { @@ -385,30 +385,30 @@ describe("openProjectHistory", () => { let release = () => {}; mediaCopy.held = new Promise((resolve) => (release = resolve)); const { history, write, projectDir, historyRoot } = await project( - { "index.html": "A", "clip.mp4": clip }, + { "index.html": "A", "clip.bin": clip }, { quietMs: 30 }, ); const closed = history.close(); release(); await closed; - write("clip.mp4", Buffer.alloc(2 * 1024 ** 2, 9)); + write("clip.bin", Buffer.alloc(2 * 1024 ** 2, 9)); const reopened = await open(projectDir, historyRoot, { quietMs: 30 }); await reopened.flush(); expect(reopened.list().map((entry) => entry.files.map((file) => file.path))).toEqual([ - ["clip.mp4"], + ["clip.bin"], ]); expect(await reopened.step("back", you)).toMatchObject({ ok: true }); - expect(readFileSync(join(projectDir, "clip.mp4")).equals(clip)).toBe(true); + expect(readFileSync(join(projectDir, "clip.bin")).equals(clip)).toBe(true); }); it("adopts media whose copy a process exit cut short, rather than log it as added", async () => { const { history, write, projectDir, historyRoot } = await project({ "index.html": "A" }); await history.close(); // What an exit mid-copy leaves: the clip on disk, named as still being copied, not in the baseline. - write("clip.mp4", Buffer.alloc(2 * 1024 ** 2, 7)); - writeFileSync(join(historyRoot, history.projectId, "adopting.json"), '["clip.mp4"]'); + write("clip.bin", Buffer.alloc(2 * 1024 ** 2, 7)); + writeFileSync(join(historyRoot, history.projectId, "adopting.json"), '["clip.bin"]'); const reopened = await open(projectDir, historyRoot, { quietMs: 30 }); - await vi.waitFor(() => expect(reopened.peek(START)).toHaveProperty(["clip.mp4"]), { + await vi.waitFor(() => expect(reopened.peek(START)).toHaveProperty(["clip.bin"]), { timeout: 10_000, }); await reopened.flush(); @@ -420,7 +420,7 @@ describe("openProjectHistory", () => { let release = () => {}; mediaCopy.held = new Promise((resolve) => (release = resolve)); const { history, write } = await project( - { "index.html": "A", "clip.mp4": clip }, + { "index.html": "A", "clip.bin": clip }, { budgetBytes: 1, quietMs: 30 }, ); // The copy is stored; an edit commits and prunes before history records the copy. @@ -429,10 +429,10 @@ describe("openProjectHistory", () => { return change(history, you, "Color", () => write("index.html", "B")); }; release(); - await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.mp4"]), { + await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.bin"]), { timeout: 10_000, }); - const start = history.peek(START)["clip.mp4"]!; + const start = history.peek(START)["clip.bin"]!; expect((await history.readBlob(start)).equals(clip)).toBe(true); }); diff --git a/packages/studio-server/src/history/projectHistory.ts b/packages/studio-server/src/history/projectHistory.ts index 1c90a6b8c81..e0e8b1fd5c8 100644 --- a/packages/studio-server/src/history/projectHistory.ts +++ b/packages/studio-server/src/history/projectHistory.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { mkdir, readdir, rm, rmdir, writeFile } from "node:fs/promises"; +import { mkdir, readdir, rm, rmdir, stat, writeFile } from "node:fs/promises"; import { type Dirent, type Stats, @@ -30,6 +30,7 @@ import { listProjectFiles, } from "../helpers/projectSignature.js"; import { openBlobStore, readBlob, type BlobStore } from "./blobStore.js"; +import { isUncopied } from "./uncopied.js"; import { pruneGoneProjectHistoriesDaily } from "./pruneHistories.js"; import { ID_PATH, @@ -460,6 +461,7 @@ class Engine { ); if (!log) return this.firstOpen(); this.log = withoutHiddenPaths(log); + await this.reclaimUncopied(); if (!this.log.keepsLedger) await this.takeInUnnamedLedger(); const cache = this.readStatCache(); const last = this.log.entries.at(-1)?.id ?? START; @@ -476,6 +478,22 @@ class Engine { await this.settleAll(); } + /** Copies an older version took of files that are no longer copied; text and small files are never touched. */ + async reclaimUncopied(): Promise { + const copied = new Set(); + const uncopied = new Set(); + const note = (path: string, hash: string | null) => { + if (hash) (isUncopied(path, this.blobs.size(hash)) ? uncopied : copied).add(hash); + }; + for (const [path, hash] of this.log.baseline) note(path, hash); + for (const entry of this.log.entries) + for (const file of entry.files) { + note(file.path, file.before); + note(file.path, file.after); + } + await this.blobs.drop([...uncopied].filter((hash) => !copied.has(hash))); + } + hearWrites(): void { const realDir = realFilePath(this.dir); this.stopHearing = onFileOverwritten((absPath, version, bytes) => { @@ -491,7 +509,8 @@ class Engine { chainBackFrom(this.overwritten.get(path), replaced).set(after, replaced), ); this.queue(async () => { - if (!this.blobs.has(replaced)) await this.storeBytes(bytes); + if (!this.blobs.has(replaced) && !isUncopied(path, bytes.length)) + await this.storeBytes(bytes); }).catch((error) => this.options.onError?.(error)); }); } @@ -589,7 +608,8 @@ class Engine { async recordAdopted(file: ListedFile, sweptAt: number, stored: string | null) { if (stored === null || this.whereFolder() !== "here") return; // A prune that ran while this copy waited for the queue may have taken its bytes. - const hash = this.blobs.has(stored) ? stored : await this.storeIfPresent(file.path); + const kept = this.blobs.has(stored) || isUncopied(file.path, file.size); + const hash = kept ? stored : await this.storeIfPresent(file.path); if (hash === null) return; const change = { path: file.path, before: null, after: hash }; const found = this.overwrittenBy(change, undefined, new Set()); @@ -679,7 +699,9 @@ class Engine { /** The file's hash once copied in; null when it was removed before the copy (the next sweep records that). */ async storeIfPresent(path: string): Promise { try { - return await this.blobs.put(join(this.dir, path)); + const abs = join(this.dir, path); + const { size } = await stat(abs); + return await (isUncopied(path, size) ? this.blobs.reference(abs) : this.blobs.put(abs)); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; throw error; @@ -1096,6 +1118,11 @@ class Engine { const changes = [...target] .filter(([path, hash]) => (this.tracked.get(path)?.hash ?? null) !== hash) .sort(([, a], [, b]) => Number(a !== null) - Number(b !== null)); + const gone = changes.find(([, hash]) => hash !== null && !this.blobs.has(hash)); + if (gone) + throw new Error( + `An earlier version of ${gone[0]} is not kept, so this cannot be brought back.`, + ); const deleted = new Set(changes.filter(([, hash]) => hash === null).map(([path]) => path)); const blocked = changes .filter(([, hash]) => hash !== null) @@ -1288,6 +1315,9 @@ class Engine { if ((await readdir(dest).catch(missingIsEmpty)).length > 0) throw new Error(`Checkout writes into an empty folder only: ${dest}`); const files = manifestAround(this.log, entryId, side)!; + for (const [path, hash] of files) + if (!this.blobs.has(hash)) + throw new Error(`The version of ${path} from then is not kept.`); try { for (const [path, hash] of files) await this.blobs.writeTo(hash, join(dest, path)); } catch (error) { diff --git a/packages/studio-server/src/history/uncopied.test.ts b/packages/studio-server/src/history/uncopied.test.ts new file mode 100644 index 00000000000..58982cd0528 --- /dev/null +++ b/packages/studio-server/src/history/uncopied.test.ts @@ -0,0 +1,118 @@ +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { openProjectHistory, type ProjectHistory } from "./projectHistory"; +import type { HistoryWho } from "./historyLog"; + +const you: HistoryWho = { kind: "person", name: "You" }; +const MiB = 1024 ** 2; +const cleanup: Array<() => unknown> = []; + +afterEach(async () => { + for (const step of cleanup.splice(0).reverse()) await step(); +}); + +function tempDir(prefix: string): string { + const dir = mkdtempSync(join(tmpdir(), prefix)); + cleanup.push(() => rmSync(dir, { recursive: true, force: true })); + return dir; +} + +async function open(projectDir: string, historyRoot: string) { + const history = await openProjectHistory({ projectDir, historyRoot, quietMs: 30 }); + cleanup.push(() => history.close()); + return history; +} + +function blobBytes(historyRoot: string, history: ProjectHistory): number { + const dir = join(historyRoot, history.projectId, "blobs"); + return readdirSync(dir).reduce( + (sum, shard) => + sum + + readdirSync(join(dir, shard)).reduce( + (inner, name) => inner + statSync(join(dir, shard, name)).size, + 0, + ), + 0, + ); +} + +async function project(files: Record) { + const projectDir = tempDir("hf-uncopied-project-"); + const historyRoot = tempDir("hf-uncopied-root-"); + const write = (path: string, content: string | Buffer) => { + mkdirSync(join(projectDir, path, ".."), { recursive: true }); + writeFileSync(join(projectDir, path), content); + }; + for (const [path, content] of Object.entries(files)) write(path, content); + const history = await open(projectDir, historyRoot); + return { projectDir, historyRoot, history, write }; +} + +describe("project history and media files", () => { + it("copies no bytes of a video that was there when history opened", async () => { + const { history, historyRoot } = await project({ + "index.html": "A", + "assets/camera.mp4": Buffer.alloc(50 * MiB, 3), + }); + await vi.waitFor(() => expect(history.peek("start")).toHaveProperty(["assets/camera.mp4"]), { + timeout: 10_000, + }); + expect(blobBytes(historyRoot, history)).toBeLessThan(1024); + }); + + it("copies no bytes of media written or replaced while history is open", async () => { + const { history, historyRoot, write } = await project({ "index.html": "A" }); + const window = await history.beginWindow(you, "Render"); + write("out.mp4", Buffer.alloc(20 * MiB, 1)); + write("voice.wav", Buffer.alloc(5 * MiB, 2)); + await window.close(); + const again = await history.beginWindow(you, "Render again"); + write("out.mp4", Buffer.alloc(20 * MiB, 9)); + const entry = await again.close(); + expect(entry?.files.map((file) => file.path)).toEqual(["out.mp4"]); + expect(blobBytes(historyRoot, history)).toBeLessThan(1024); + }); + + it("still copies a text edit and says plainly that a media version is not kept", async () => { + const { history, historyRoot, write } = await project({ + "index.html": "A", + "clip.mp4": Buffer.alloc(2 * MiB, 1), + }); + await vi.waitFor(() => expect(history.peek("start")).toHaveProperty(["clip.mp4"]), { + timeout: 10_000, + }); + const edit = await history.beginWindow(you, "Edit"); + write("index.html", "B"); + write("clip.mp4", Buffer.alloc(2 * MiB, 2)); + const entry = (await edit.close())!; + expect(blobBytes(historyRoot, history)).toBeLessThan(2048); + await expect(history.undo(entry.id, { who: you })).rejects.toThrow(/clip\.mp4 is not kept/); + }); + + it("reclaims media copies an older version left, and keeps every text copy", async () => { + const { history, historyRoot, projectDir } = await project({ + "index.html": "A", + "clip.mp4": Buffer.alloc(2 * MiB, 5), + }); + await vi.waitFor(() => expect(history.peek("start")).toHaveProperty(["clip.mp4"]), { + timeout: 10_000, + }); + await history.close(); + const media = Buffer.alloc(2 * MiB, 5); + const hash = createHash("sha256").update(media).digest("hex"); + const planted = join(historyRoot, history.projectId, "blobs", hash.slice(0, 2)); + mkdirSync(planted, { recursive: true }); + writeFileSync(join(planted, hash), media); + expect(blobBytes(historyRoot, history)).toBeGreaterThanOrEqual(2 * MiB); + + const reopened = await open(projectDir, historyRoot); + await reopened.flush(); + expect(blobBytes(historyRoot, reopened)).toBeLessThan(1024); + expect(await reopened.readBlob(reopened.peek("start")!["index.html"]!)).toEqual( + Buffer.from("A"), + ); + }); +}); diff --git a/packages/studio-server/src/history/uncopied.ts b/packages/studio-server/src/history/uncopied.ts new file mode 100644 index 00000000000..f7c44959afb --- /dev/null +++ b/packages/studio-server/src/history/uncopied.ts @@ -0,0 +1,26 @@ +import { extname } from "node:path"; + +const MEDIA_EXTENSIONS = new Set([ + ".mp4", + ".mov", + ".m4v", + ".webm", + ".mkv", + ".avi", + ".mxf", + ".mts", + ".m2ts", + ".ts", + ".mp3", + ".wav", + ".aac", + ".m4a", + ".flac", + ".ogg", + ".opus", +]); +const COPY_NOTHING_ABOVE_BYTES = 64 * 1024 ** 2; + +/** Video, audio and any file over 64 MiB are hashed in place, never copied into history: restoring one is not offered. */ +export const isUncopied = (path: string, size: number): boolean => + size > COPY_NOTHING_ABOVE_BYTES || MEDIA_EXTENSIONS.has(extname(path).toLowerCase()); From 59cfd6ed108f6eba3d384085ea643e64ec1dd310 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 23 Oct 2026 22:16:04 -0400 Subject: [PATCH 2/5] fix(studio-server): clean drops old video and audio copies in every history --- packages/cli/src/commands/clean.ts | 12 ++++ .../studio-server/src/history/blobStore.ts | 2 +- packages/studio-server/src/history/index.ts | 1 + .../src/history/projectHistory.ts | 15 +---- .../src/history/reclaimUncopied.test.ts | 61 ++++++++++++++++++ .../src/history/reclaimUncopied.ts | 62 +++++++++++++++++++ .../studio-server/src/history/uncopied.ts | 19 +++++- 7 files changed, 157 insertions(+), 15 deletions(-) create mode 100644 packages/studio-server/src/history/reclaimUncopied.test.ts create mode 100644 packages/studio-server/src/history/reclaimUncopied.ts diff --git a/packages/cli/src/commands/clean.ts b/packages/cli/src/commands/clean.ts index bf7d1b9cb87..b23decc12a0 100644 --- a/packages/cli/src/commands/clean.ts +++ b/packages/cli/src/commands/clean.ts @@ -14,6 +14,7 @@ import { cleanupProxyCache, listProjectHistories, pruneGoneProjectHistories, + reclaimUncopiedBlobs, } from "@hyperframes/studio-server"; import type { Example } from "./_examples.js"; import { c } from "../ui/colors.js"; @@ -172,6 +173,17 @@ async function sweepHistories(sweep: Sweep, historyRoot: string, tempDir: string bytes: history.bytes, }); } + const reclaimed = await reclaimUncopiedBlobs(historyRoot, { + dryRun: sweep.dryRun, + onError: (error) => recordError(sweep, error), + }); + for (const history of reclaimed) { + sweep.result.removed.push({ + what: "Video and audio copies in project history", + path: join(historyRoot, history.id, "blobs"), + bytes: history.bytes, + }); + } } function sweepExtractCache(sweep: Sweep): void { diff --git a/packages/studio-server/src/history/blobStore.ts b/packages/studio-server/src/history/blobStore.ts index c6cefb4a18a..52dc05ab4ea 100644 --- a/packages/studio-server/src/history/blobStore.ts +++ b/packages/studio-server/src/history/blobStore.ts @@ -22,7 +22,7 @@ export interface BlobStore { const BLOB_HASH = /^[0-9a-f]{64}$/; -function blobPath(dir: string, hash: string): string { +export function blobPath(dir: string, hash: string): string { // Checked where the path is joined, so no caller can read or write outside the store. if (!BLOB_HASH.test(hash)) throw new Error("That is not a history blob."); return join(dir, hash.slice(0, 2), hash); diff --git a/packages/studio-server/src/history/index.ts b/packages/studio-server/src/history/index.ts index 1c4c1ee2903..c048388de00 100644 --- a/packages/studio-server/src/history/index.ts +++ b/packages/studio-server/src/history/index.ts @@ -14,6 +14,7 @@ export { type HistoryWindow, HistoryClosedError, } from "./projectHistory.js"; +export { reclaimUncopiedBlobs, type ReclaimedHistory } from "./reclaimUncopied.js"; export { HistoryBusyError } from "./ownerLock.js"; export { HistoryIdError } from "./historyId.js"; export { historyCache } from "./historyCache.js"; diff --git a/packages/studio-server/src/history/projectHistory.ts b/packages/studio-server/src/history/projectHistory.ts index e0e8b1fd5c8..a5eef2707d5 100644 --- a/packages/studio-server/src/history/projectHistory.ts +++ b/packages/studio-server/src/history/projectHistory.ts @@ -30,7 +30,7 @@ import { listProjectFiles, } from "../helpers/projectSignature.js"; import { openBlobStore, readBlob, type BlobStore } from "./blobStore.js"; -import { isUncopied } from "./uncopied.js"; +import { isUncopied, uncopiedOnlyHashes } from "./uncopied.js"; import { pruneGoneProjectHistoriesDaily } from "./pruneHistories.js"; import { ID_PATH, @@ -480,18 +480,7 @@ class Engine { /** Copies an older version took of files that are no longer copied; text and small files are never touched. */ async reclaimUncopied(): Promise { - const copied = new Set(); - const uncopied = new Set(); - const note = (path: string, hash: string | null) => { - if (hash) (isUncopied(path, this.blobs.size(hash)) ? uncopied : copied).add(hash); - }; - for (const [path, hash] of this.log.baseline) note(path, hash); - for (const entry of this.log.entries) - for (const file of entry.files) { - note(file.path, file.before); - note(file.path, file.after); - } - await this.blobs.drop([...uncopied].filter((hash) => !copied.has(hash))); + await this.blobs.drop(uncopiedOnlyHashes(this.log, (hash) => this.blobs.size(hash))); } hearWrites(): void { diff --git a/packages/studio-server/src/history/reclaimUncopied.test.ts b/packages/studio-server/src/history/reclaimUncopied.test.ts new file mode 100644 index 00000000000..c824ae6b8a5 --- /dev/null +++ b/packages/studio-server/src/history/reclaimUncopied.test.ts @@ -0,0 +1,61 @@ +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { openProjectHistory } from "./projectHistory"; +import { reclaimUncopiedBlobs } from "./reclaimUncopied"; +import { START } from "./historyLog"; + +const MiB = 1024 ** 2; +const cleanup: Array<() => unknown> = []; +afterEach(async () => { + for (const step of cleanup.splice(0).reverse()) await step(); +}); + +function tempDir(prefix: string): string { + const dir = mkdtempSync(join(tmpdir(), prefix)); + cleanup.push(() => rmSync(dir, { recursive: true, force: true })); + return dir; +} + +const blobFile = (home: string, content: Buffer) => { + const hash = createHash("sha256").update(content).digest("hex"); + return join(home, "blobs", hash.slice(0, 2), hash); +}; + +/** A closed history as an older version left it: the media's bytes sit in the blob store next to the text. */ +async function closedHistoryWithOldMediaCopy() { + const projectDir = tempDir("hf-reclaim-project-"); + const historyRoot = tempDir("hf-reclaim-root-"); + const media = Buffer.alloc(2 * MiB, 5); + writeFileSync(join(projectDir, "index.html"), "A"); + writeFileSync(join(projectDir, "clip.mp4"), media); + const history = await openProjectHistory({ projectDir, historyRoot, quietMs: 30 }); + await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.mp4"]), { + timeout: 10_000, + }); + await history.close(); + const home = join(historyRoot, history.projectId); + const copy = blobFile(home, media); + mkdirSync(join(copy, ".."), { recursive: true }); + writeFileSync(copy, media); + return { historyRoot, home, copy, text: blobFile(home, Buffer.from("A")) }; +} + +describe("reclaimUncopiedBlobs", () => { + it("lists the bytes of old media copies in every history and removes nothing on a dry run", async () => { + const { historyRoot, copy } = await closedHistoryWithOldMediaCopy(); + const found = await reclaimUncopiedBlobs(historyRoot, { dryRun: true }); + expect(found.map((history) => history.bytes)).toEqual([2 * MiB]); + expect(existsSync(copy)).toBe(true); + }); + + it("removes the media copies of a history nobody has open, and keeps its text copies", async () => { + const { historyRoot, copy, text } = await closedHistoryWithOldMediaCopy(); + await reclaimUncopiedBlobs(historyRoot); + expect(existsSync(copy)).toBe(false); + expect(existsSync(text)).toBe(true); + expect(await reclaimUncopiedBlobs(historyRoot)).toEqual([]); + }); +}); diff --git a/packages/studio-server/src/history/reclaimUncopied.ts b/packages/studio-server/src/history/reclaimUncopied.ts new file mode 100644 index 00000000000..57379f43f65 --- /dev/null +++ b/packages/studio-server/src/history/reclaimUncopied.ts @@ -0,0 +1,62 @@ +import { readdirSync } from "node:fs"; +import { rm, stat } from "node:fs/promises"; +import { join } from "node:path"; +import { blobPath } from "./blobStore.js"; +import { isHistoryId } from "./historyId.js"; +import { readLog } from "./historyLog.js"; +import { HistoryBusyError, takeHistoryOwnership } from "./ownerLock.js"; +import { uncopiedOnlyHashes } from "./uncopied.js"; + +export interface ReclaimedHistory { + id: string; + bytes: number; +} + +async function sizeOfBlob(blobs: string, hash: string): Promise { + return (await stat(blobPath(blobs, hash)).catch(() => null))?.size ?? 0; +} + +/** Drops the media copies older versions left in every history no running process holds; the same rule as on open. */ +export async function reclaimUncopiedBlobs( + historyRoot: string, + { dryRun = false, onError }: { dryRun?: boolean; onError?: (error: unknown) => void } = {}, +): Promise { + const reclaimed: ReclaimedHistory[] = []; + const ids = (() => { + try { + return readdirSync(historyRoot).filter(isHistoryId); + } catch { + return []; + } + })(); + for (const id of ids) { + const home = join(historyRoot, id); + let release: (() => void) | undefined; + try { + release = await takeHistoryOwnership(home, 0); + const log = readLog(join(home, "log.jsonl"), () => {}); + if (!log) continue; + const blobs = join(home, "blobs"); + const sizes = new Map(); + for (const entry of log.entries) + for (const file of entry.files) + for (const hash of [file.before, file.after]) + if (hash && !sizes.has(hash)) sizes.set(hash, await sizeOfBlob(blobs, hash)); + for (const hash of log.baseline.values()) + if (!sizes.has(hash)) sizes.set(hash, await sizeOfBlob(blobs, hash)); + let bytes = 0; + for (const hash of uncopiedOnlyHashes(log, (known) => sizes.get(known) ?? 0)) { + const size = sizes.get(hash) ?? 0; + if (size === 0) continue; + bytes += size; + if (!dryRun) await rm(blobPath(blobs, hash), { force: true }); + } + if (bytes > 0) reclaimed.push({ id, bytes }); + } catch (error) { + if (!(error instanceof HistoryBusyError)) onError?.(error); + } finally { + release?.(); + } + } + return reclaimed; +} diff --git a/packages/studio-server/src/history/uncopied.ts b/packages/studio-server/src/history/uncopied.ts index f7c44959afb..0d5e2c222af 100644 --- a/packages/studio-server/src/history/uncopied.ts +++ b/packages/studio-server/src/history/uncopied.ts @@ -1,4 +1,5 @@ import { extname } from "node:path"; +import type { HistoryLog } from "./historyLog.js"; const MEDIA_EXTENSIONS = new Set([ ".mp4", @@ -21,6 +22,22 @@ const MEDIA_EXTENSIONS = new Set([ ]); const COPY_NOTHING_ABOVE_BYTES = 64 * 1024 ** 2; -/** Video, audio and any file over 64 MiB are hashed in place, never copied into history: restoring one is not offered. */ +/** Video, audio and any file over 64 MiB are hashed in place, never copied into history. */ export const isUncopied = (path: string, size: number): boolean => size > COPY_NOTHING_ABOVE_BYTES || MEDIA_EXTENSIONS.has(extname(path).toLowerCase()); + +/** Hashes whose every reference in the log is a file that is no longer copied: their old blobs can go. */ +export function uncopiedOnlyHashes(log: HistoryLog, sizeOf: (hash: string) => number): string[] { + const copied = new Set(); + const uncopied = new Set(); + const note = (path: string, hash: string | null) => { + if (hash) (isUncopied(path, sizeOf(hash)) ? uncopied : copied).add(hash); + }; + for (const [path, hash] of log.baseline) note(path, hash); + for (const entry of log.entries) + for (const file of entry.files) { + note(file.path, file.before); + note(file.path, file.after); + } + return [...uncopied].filter((hash) => !copied.has(hash)); +} From 9baed70abd2d5405fb7275bb2cc3375d8a351eff Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 23 Oct 2026 22:30:18 -0400 Subject: [PATCH 3/5] fix(studio-server): TypeScript source is not media, so its history is kept --- .../src/history/uncopied.test.ts | 29 ++++++++++++++++++- .../studio-server/src/history/uncopied.ts | 2 -- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/packages/studio-server/src/history/uncopied.test.ts b/packages/studio-server/src/history/uncopied.test.ts index 58982cd0528..a3e935487df 100644 --- a/packages/studio-server/src/history/uncopied.test.ts +++ b/packages/studio-server/src/history/uncopied.test.ts @@ -1,5 +1,13 @@ import { createHash } from "node:crypto"; -import { mkdirSync, mkdtempSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -115,4 +123,23 @@ describe("project history and media files", () => { Buffer.from("A"), ); }); + + it("keeps TypeScript source edits undoable, and reclaim leaves their copies", async () => { + const { history, historyRoot, projectDir, write } = await project({ + "index.html": "A", + "setup.ts": "export const a = 1;", + "intro.mts": "export const b = 1;", + }); + await history.flush(); + const edit = await history.beginWindow(you, "Edit"); + write("setup.ts", "export const a = 2;"); + write("intro.mts", "export const b = 2;"); + const entry = (await edit.close())!; + expect(blobBytes(historyRoot, history)).toBeGreaterThan(40); + await history.close(); + const reopened = await open(projectDir, historyRoot); + expect(blobBytes(historyRoot, reopened)).toBeGreaterThan(40); + expect((await reopened.undo(entry.id, { who: you })).ok).toBe(true); + expect(readFileSync(join(projectDir, "setup.ts"), "utf-8")).toBe("export const a = 1;"); + }); }); diff --git a/packages/studio-server/src/history/uncopied.ts b/packages/studio-server/src/history/uncopied.ts index 0d5e2c222af..34460b29ed4 100644 --- a/packages/studio-server/src/history/uncopied.ts +++ b/packages/studio-server/src/history/uncopied.ts @@ -9,9 +9,7 @@ const MEDIA_EXTENSIONS = new Set([ ".mkv", ".avi", ".mxf", - ".mts", ".m2ts", - ".ts", ".mp3", ".wav", ".aac", From 5ea5730cd8bbede5147c4f0a34754d7dc38f9e56 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 23 Oct 2026 23:00:39 -0400 Subject: [PATCH 4/5] fix(studio-server): split the media reclaim and point the slow-copy route test at a non-media file --- .../src/history/reclaimUncopied.ts | 67 +++++++++++-------- .../studio-server/src/routes/history.test.ts | 14 ++-- 2 files changed, 46 insertions(+), 35 deletions(-) diff --git a/packages/studio-server/src/history/reclaimUncopied.ts b/packages/studio-server/src/history/reclaimUncopied.ts index 57379f43f65..c65a7eefcd4 100644 --- a/packages/studio-server/src/history/reclaimUncopied.ts +++ b/packages/studio-server/src/history/reclaimUncopied.ts @@ -3,7 +3,7 @@ import { rm, stat } from "node:fs/promises"; import { join } from "node:path"; import { blobPath } from "./blobStore.js"; import { isHistoryId } from "./historyId.js"; -import { readLog } from "./historyLog.js"; +import { readLog, type HistoryLog } from "./historyLog.js"; import { HistoryBusyError, takeHistoryOwnership } from "./ownerLock.js"; import { uncopiedOnlyHashes } from "./uncopied.js"; @@ -12,8 +12,42 @@ export interface ReclaimedHistory { bytes: number; } -async function sizeOfBlob(blobs: string, hash: string): Promise { - return (await stat(blobPath(blobs, hash)).catch(() => null))?.size ?? 0; +function historyIds(historyRoot: string): string[] { + try { + return readdirSync(historyRoot).filter(isHistoryId); + } catch { + return []; + } +} + +function namedHashes(log: HistoryLog): Set { + const hashes = new Set(log.baseline.values()); + for (const entry of log.entries) + for (const file of entry.files) + for (const hash of [file.before, file.after]) hash && hashes.add(hash); + return hashes; +} + +async function blobSizes(blobs: string, hashes: Iterable): Promise> { + const sizes = new Map(); + for (const hash of hashes) + sizes.set(hash, (await stat(blobPath(blobs, hash)).catch(() => null))?.size ?? 0); + return sizes; +} + +async function reclaimOne(home: string, dryRun: boolean): Promise { + const log = readLog(join(home, "log.jsonl"), () => {}); + if (!log) return 0; + const blobs = join(home, "blobs"); + const sizes = await blobSizes(blobs, namedHashes(log)); + let bytes = 0; + for (const hash of uncopiedOnlyHashes(log, (known) => sizes.get(known) ?? 0)) { + const size = sizes.get(hash) ?? 0; + if (size === 0) continue; + bytes += size; + if (!dryRun) await rm(blobPath(blobs, hash), { force: true }); + } + return bytes; } /** Drops the media copies older versions left in every history no running process holds; the same rule as on open. */ @@ -22,35 +56,12 @@ export async function reclaimUncopiedBlobs( { dryRun = false, onError }: { dryRun?: boolean; onError?: (error: unknown) => void } = {}, ): Promise { const reclaimed: ReclaimedHistory[] = []; - const ids = (() => { - try { - return readdirSync(historyRoot).filter(isHistoryId); - } catch { - return []; - } - })(); - for (const id of ids) { + for (const id of historyIds(historyRoot)) { const home = join(historyRoot, id); let release: (() => void) | undefined; try { release = await takeHistoryOwnership(home, 0); - const log = readLog(join(home, "log.jsonl"), () => {}); - if (!log) continue; - const blobs = join(home, "blobs"); - const sizes = new Map(); - for (const entry of log.entries) - for (const file of entry.files) - for (const hash of [file.before, file.after]) - if (hash && !sizes.has(hash)) sizes.set(hash, await sizeOfBlob(blobs, hash)); - for (const hash of log.baseline.values()) - if (!sizes.has(hash)) sizes.set(hash, await sizeOfBlob(blobs, hash)); - let bytes = 0; - for (const hash of uncopiedOnlyHashes(log, (known) => sizes.get(known) ?? 0)) { - const size = sizes.get(hash) ?? 0; - if (size === 0) continue; - bytes += size; - if (!dryRun) await rm(blobPath(blobs, hash), { force: true }); - } + const bytes = await reclaimOne(home, dryRun); if (bytes > 0) reclaimed.push({ id, bytes }); } catch (error) { if (!(error instanceof HistoryBusyError)) onError?.(error); diff --git a/packages/studio-server/src/routes/history.test.ts b/packages/studio-server/src/routes/history.test.ts index 390e62f142a..f1823cb6087 100644 --- a/packages/studio-server/src/routes/history.test.ts +++ b/packages/studio-server/src/routes/history.test.ts @@ -25,7 +25,7 @@ vi.mock("../history/blobStore", async (importOriginal) => { return { ...store, put: async (path: string) => { - if (path.endsWith(".mp4")) await mediaCopy.held; + if (path.endsWith(".bin")) await mediaCopy.held; return store.put(path); }, }; @@ -410,7 +410,7 @@ describe("history routes", () => { let finishCopy = () => {}; mediaCopy.held = new Promise((resolve) => (finishCopy = resolve)); const { projectDir, file, put, post, histories } = lazyHistoryProject(); - writeFileSync(join(projectDir, "clip.mp4"), Buffer.alloc(4 * 1024 ** 2, 7)); + writeFileSync(join(projectDir, "clip.bin"), Buffer.alloc(4 * 1024 ** 2, 7)); // The copy stays held until finishCopy, so a save or claim that waited on it never returns. expect((await put("B")).status).toBe(200); @@ -420,7 +420,7 @@ describe("history routes", () => { finishCopy(); const history = (await histories.get(projectDir))!; - await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.mp4"]), { + await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.bin"]), { timeout: 10_000, }); await history.flush(); @@ -431,12 +431,12 @@ describe("history routes", () => { let finishCopy = () => {}; mediaCopy.held = new Promise((resolve) => (finishCopy = resolve)); const { projectDir, api, post } = lazyHistoryProject(); - const clip = join(projectDir, "clip.mp4"); + const clip = join(projectDir, "clip.bin"); const original = Buffer.alloc(4 * 1024 ** 2, 7); writeFileSync(clip, original); await post("/claim", { label: "Nothing", paths: [] }); - const written = await api.request("/projects/demo/files/clip.mp4", { + const written = await api.request("/projects/demo/files/clip.bin", { method: "PUT", body: Buffer.alloc(4 * 1024 ** 2, 9), headers: { "If-Match": fileContentVersion(original) }, @@ -445,8 +445,8 @@ describe("history routes", () => { finishCopy(); const claimed = await post("/claim", { label: "Replace clip", - paths: ["clip.mp4"], - overwrote: { "clip.mp4": fileContentVersion(original) }, + paths: ["clip.bin"], + overwrote: { "clip.bin": fileContentVersion(original) }, }); expect(((await claimed.json()) as { claimed: unknown }).claimed).not.toBeNull(); From 51f914c9cf6ad541cf80e8f61ff53f7f51cbaa32 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 23 Oct 2026 23:44:50 -0400 Subject: [PATCH 5/5] fix(studio-server): no unused expression when collecting a history's hashes --- packages/studio-server/src/history/reclaimUncopied.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/studio-server/src/history/reclaimUncopied.ts b/packages/studio-server/src/history/reclaimUncopied.ts index c65a7eefcd4..6179d375ad6 100644 --- a/packages/studio-server/src/history/reclaimUncopied.ts +++ b/packages/studio-server/src/history/reclaimUncopied.ts @@ -22,9 +22,12 @@ function historyIds(historyRoot: string): string[] { function namedHashes(log: HistoryLog): Set { const hashes = new Set(log.baseline.values()); - for (const entry of log.entries) - for (const file of entry.files) - for (const hash of [file.before, file.after]) hash && hashes.add(hash); + for (const entry of log.entries) { + for (const file of entry.files) { + if (file.before) hashes.add(file.before); + if (file.after) hashes.add(file.after); + } + } return hashes; }