diff --git a/packages/cli/src/commands/clean.ts b/packages/cli/src/commands/clean.ts index bf7d1b9cb87..b23decc12a0 100644 --- a/packages/cli/src/commands/clean.ts +++ b/packages/cli/src/commands/clean.ts @@ -14,6 +14,7 @@ import { cleanupProxyCache, listProjectHistories, pruneGoneProjectHistories, + reclaimUncopiedBlobs, } from "@hyperframes/studio-server"; import type { Example } from "./_examples.js"; import { c } from "../ui/colors.js"; @@ -172,6 +173,17 @@ async function sweepHistories(sweep: Sweep, historyRoot: string, tempDir: string bytes: history.bytes, }); } + const reclaimed = await reclaimUncopiedBlobs(historyRoot, { + dryRun: sweep.dryRun, + onError: (error) => recordError(sweep, error), + }); + for (const history of reclaimed) { + sweep.result.removed.push({ + what: "Video and audio copies in project history", + path: join(historyRoot, history.id, "blobs"), + bytes: history.bytes, + }); + } } function sweepExtractCache(sweep: Sweep): void { diff --git a/packages/studio-server/src/history/blobStore.ts b/packages/studio-server/src/history/blobStore.ts index 67159aff117..52dc05ab4ea 100644 --- a/packages/studio-server/src/history/blobStore.ts +++ b/packages/studio-server/src/history/blobStore.ts @@ -9,6 +9,7 @@ import { atomicTempPath } from "@hyperframes/core/atomic-file"; export interface BlobStore { /** Copies the file in (a clone where the file system can) and returns the copy's hash. */ put(absPath: string): Promise; + reference(absPath: string): Promise; has(hash: string): boolean; read(hash: string): Promise; /** Writes the blob's bytes to `absPath` by clone-or-copy and rename, so a reader never sees half a file. */ @@ -16,11 +17,12 @@ export interface BlobStore { bytes(): number; size(hash: string): number; prune(keep: ReadonlySet): Promise; + drop(hashes: Iterable): Promise; } const BLOB_HASH = /^[0-9a-f]{64}$/; -function blobPath(dir: string, hash: string): string { +export function blobPath(dir: string, hash: string): string { // Checked where the path is joined, so no caller can read or write outside the store. if (!BLOB_HASH.test(hash)) throw new Error("That is not a history blob."); return join(dir, hash.slice(0, 2), hash); @@ -87,12 +89,22 @@ export async function openBlobStore(dir: string): Promise { total += size; return hash; }, + reference: (absPath) => hashFile(absPath), has: (hash) => sizes.has(hash), read: async (hash) => readBlob(dir, hash), writeTo: async (hash, absPath, beforeReplace) => cloneOrCopy(pathOf(hash), absPath, beforeReplace), bytes: () => total, size: (hash) => sizes.get(hash) ?? 0, + async drop(hashes) { + for (const hash of hashes) { + const size = sizes.get(hash); + if (size === undefined) continue; + await rm(pathOf(hash), { force: true }); + sizes.delete(hash); + total -= size; + } + }, async prune(keep) { for (const [hash, size] of sizes) { if (keep.has(hash)) continue; diff --git a/packages/studio-server/src/history/index.ts b/packages/studio-server/src/history/index.ts index 1c4c1ee2903..c048388de00 100644 --- a/packages/studio-server/src/history/index.ts +++ b/packages/studio-server/src/history/index.ts @@ -14,6 +14,7 @@ export { type HistoryWindow, HistoryClosedError, } from "./projectHistory.js"; +export { reclaimUncopiedBlobs, type ReclaimedHistory } from "./reclaimUncopied.js"; export { HistoryBusyError } from "./ownerLock.js"; export { HistoryIdError } from "./historyId.js"; export { historyCache } from "./historyCache.js"; diff --git a/packages/studio-server/src/history/projectHistory.test.ts b/packages/studio-server/src/history/projectHistory.test.ts index 7fb199317dc..9ee0f0fad00 100644 --- a/packages/studio-server/src/history/projectHistory.test.ts +++ b/packages/studio-server/src/history/projectHistory.test.ts @@ -48,13 +48,13 @@ vi.mock("./blobStore", async (importOriginal) => { return { ...store, put: async (path: string) => { - if (path.endsWith(".mp4")) { + if (path.endsWith(".bin")) { await mediaCopy.held; mediaCopy.copies.push(basename(path)); if (mediaCopy.fails?.(mediaCopy.copies.length)) throw new Error("The disk is full."); } const hash = await store.put(path); - if (path.endsWith(".mp4")) await mediaCopy.stored?.(); + if (path.endsWith(".bin")) await mediaCopy.stored?.(); return hash; }, }; @@ -306,10 +306,10 @@ describe("openProjectHistory", () => { it("a ledger made while closed is still a change after the project's media was copied in the background", async () => { const { history, write, has, projectDir, historyRoot } = await project( - { "index.html": "A", "clip.mp4": Buffer.alloc(2 * 1024 ** 2, 7) }, + { "index.html": "A", "clip.bin": Buffer.alloc(2 * 1024 ** 2, 7) }, { quietMs: 30 }, ); - await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.mp4"]), { + await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.bin"]), { timeout: 10_000, }); await history.close(); @@ -331,12 +331,12 @@ describe("openProjectHistory", () => { mediaCopy.held = new Promise((resolve) => (release = resolve)); const { history, historyRoot } = await project({ "index.html": "A", - "clip.mp4": Buffer.alloc(2 * 1024 ** 2, 7), + "clip.bin": Buffer.alloc(2 * 1024 ** 2, 7), }); const adopting = join(historyRoot, history.projectId, "adopting.json"); - expect(JSON.parse(readFileSync(adopting, "utf-8"))).toEqual(["clip.mp4"]); + expect(JSON.parse(readFileSync(adopting, "utf-8"))).toEqual(["clip.bin"]); release(); - await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.mp4"]), { + await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.bin"]), { timeout: 10_000, }); expect(existsSync(adopting)).toBe(false); @@ -344,7 +344,7 @@ describe("openProjectHistory", () => { it("keeps media whose recording failed for the next open, rather than log it as added", async () => { const { history, write, historyRoot } = await project( - { "index.html": "A", "clip.mp4": Buffer.alloc(2 * 1024 ** 2, 7) }, + { "index.html": "A", "clip.bin": Buffer.alloc(2 * 1024 ** 2, 7) }, { budgetBytes: 1, quietMs: 30 }, ); // The edit's prune takes the first copy; storing it again then fails. @@ -356,18 +356,18 @@ describe("openProjectHistory", () => { await vi.waitFor(() => expect(mediaCopy.copies).toHaveLength(2), { timeout: 10_000 }); await history.flush(); expect(history.list().flatMap((entry) => entry.files.map((file) => file.path))).not.toContain( - "clip.mp4", + "clip.bin", ); const adopting = join(historyRoot, history.projectId, "adopting.json"); - expect(JSON.parse(readFileSync(adopting, "utf-8"))).toEqual(["clip.mp4"]); + expect(JSON.parse(readFileSync(adopting, "utf-8"))).toEqual(["clip.bin"]); }); it("never copies the media of a folder that replaced the project", async () => { let release = () => {}; mediaCopy.held = new Promise((resolve) => (release = resolve)); const media = { - "a.mp4": Buffer.alloc(2 * 1024 ** 2, 1), - "b.mp4": Buffer.alloc(2 * 1024 ** 2, 2), + "a.bin": Buffer.alloc(2 * 1024 ** 2, 1), + "b.bin": Buffer.alloc(2 * 1024 ** 2, 2), }; const { history, write, projectDir } = await project({ "index.html": "A", ...media }); renameSync(projectDir, `${projectDir}-moved`); @@ -377,7 +377,7 @@ describe("openProjectHistory", () => { release(); await history.close(); // Only the copy already running when the folder changed may finish. - expect(mediaCopy.copies).toEqual(["a.mp4"]); + expect(mediaCopy.copies).toEqual(["a.bin"]); }); it("undoes a clip replaced while closed, though the history closed before its copy finished", async () => { @@ -385,30 +385,30 @@ describe("openProjectHistory", () => { let release = () => {}; mediaCopy.held = new Promise((resolve) => (release = resolve)); const { history, write, projectDir, historyRoot } = await project( - { "index.html": "A", "clip.mp4": clip }, + { "index.html": "A", "clip.bin": clip }, { quietMs: 30 }, ); const closed = history.close(); release(); await closed; - write("clip.mp4", Buffer.alloc(2 * 1024 ** 2, 9)); + write("clip.bin", Buffer.alloc(2 * 1024 ** 2, 9)); const reopened = await open(projectDir, historyRoot, { quietMs: 30 }); await reopened.flush(); expect(reopened.list().map((entry) => entry.files.map((file) => file.path))).toEqual([ - ["clip.mp4"], + ["clip.bin"], ]); expect(await reopened.step("back", you)).toMatchObject({ ok: true }); - expect(readFileSync(join(projectDir, "clip.mp4")).equals(clip)).toBe(true); + expect(readFileSync(join(projectDir, "clip.bin")).equals(clip)).toBe(true); }); it("adopts media whose copy a process exit cut short, rather than log it as added", async () => { const { history, write, projectDir, historyRoot } = await project({ "index.html": "A" }); await history.close(); // What an exit mid-copy leaves: the clip on disk, named as still being copied, not in the baseline. - write("clip.mp4", Buffer.alloc(2 * 1024 ** 2, 7)); - writeFileSync(join(historyRoot, history.projectId, "adopting.json"), '["clip.mp4"]'); + write("clip.bin", Buffer.alloc(2 * 1024 ** 2, 7)); + writeFileSync(join(historyRoot, history.projectId, "adopting.json"), '["clip.bin"]'); const reopened = await open(projectDir, historyRoot, { quietMs: 30 }); - await vi.waitFor(() => expect(reopened.peek(START)).toHaveProperty(["clip.mp4"]), { + await vi.waitFor(() => expect(reopened.peek(START)).toHaveProperty(["clip.bin"]), { timeout: 10_000, }); await reopened.flush(); @@ -420,7 +420,7 @@ describe("openProjectHistory", () => { let release = () => {}; mediaCopy.held = new Promise((resolve) => (release = resolve)); const { history, write } = await project( - { "index.html": "A", "clip.mp4": clip }, + { "index.html": "A", "clip.bin": clip }, { budgetBytes: 1, quietMs: 30 }, ); // The copy is stored; an edit commits and prunes before history records the copy. @@ -429,10 +429,10 @@ describe("openProjectHistory", () => { return change(history, you, "Color", () => write("index.html", "B")); }; release(); - await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.mp4"]), { + await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.bin"]), { timeout: 10_000, }); - const start = history.peek(START)["clip.mp4"]!; + const start = history.peek(START)["clip.bin"]!; expect((await history.readBlob(start)).equals(clip)).toBe(true); }); diff --git a/packages/studio-server/src/history/projectHistory.ts b/packages/studio-server/src/history/projectHistory.ts index 1c90a6b8c81..a5eef2707d5 100644 --- a/packages/studio-server/src/history/projectHistory.ts +++ b/packages/studio-server/src/history/projectHistory.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { mkdir, readdir, rm, rmdir, writeFile } from "node:fs/promises"; +import { mkdir, readdir, rm, rmdir, stat, writeFile } from "node:fs/promises"; import { type Dirent, type Stats, @@ -30,6 +30,7 @@ import { listProjectFiles, } from "../helpers/projectSignature.js"; import { openBlobStore, readBlob, type BlobStore } from "./blobStore.js"; +import { isUncopied, uncopiedOnlyHashes } from "./uncopied.js"; import { pruneGoneProjectHistoriesDaily } from "./pruneHistories.js"; import { ID_PATH, @@ -460,6 +461,7 @@ class Engine { ); if (!log) return this.firstOpen(); this.log = withoutHiddenPaths(log); + await this.reclaimUncopied(); if (!this.log.keepsLedger) await this.takeInUnnamedLedger(); const cache = this.readStatCache(); const last = this.log.entries.at(-1)?.id ?? START; @@ -476,6 +478,11 @@ class Engine { await this.settleAll(); } + /** Copies an older version took of files that are no longer copied; text and small files are never touched. */ + async reclaimUncopied(): Promise { + await this.blobs.drop(uncopiedOnlyHashes(this.log, (hash) => this.blobs.size(hash))); + } + hearWrites(): void { const realDir = realFilePath(this.dir); this.stopHearing = onFileOverwritten((absPath, version, bytes) => { @@ -491,7 +498,8 @@ class Engine { chainBackFrom(this.overwritten.get(path), replaced).set(after, replaced), ); this.queue(async () => { - if (!this.blobs.has(replaced)) await this.storeBytes(bytes); + if (!this.blobs.has(replaced) && !isUncopied(path, bytes.length)) + await this.storeBytes(bytes); }).catch((error) => this.options.onError?.(error)); }); } @@ -589,7 +597,8 @@ class Engine { async recordAdopted(file: ListedFile, sweptAt: number, stored: string | null) { if (stored === null || this.whereFolder() !== "here") return; // A prune that ran while this copy waited for the queue may have taken its bytes. - const hash = this.blobs.has(stored) ? stored : await this.storeIfPresent(file.path); + const kept = this.blobs.has(stored) || isUncopied(file.path, file.size); + const hash = kept ? stored : await this.storeIfPresent(file.path); if (hash === null) return; const change = { path: file.path, before: null, after: hash }; const found = this.overwrittenBy(change, undefined, new Set()); @@ -679,7 +688,9 @@ class Engine { /** The file's hash once copied in; null when it was removed before the copy (the next sweep records that). */ async storeIfPresent(path: string): Promise { try { - return await this.blobs.put(join(this.dir, path)); + const abs = join(this.dir, path); + const { size } = await stat(abs); + return await (isUncopied(path, size) ? this.blobs.reference(abs) : this.blobs.put(abs)); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; throw error; @@ -1096,6 +1107,11 @@ class Engine { const changes = [...target] .filter(([path, hash]) => (this.tracked.get(path)?.hash ?? null) !== hash) .sort(([, a], [, b]) => Number(a !== null) - Number(b !== null)); + const gone = changes.find(([, hash]) => hash !== null && !this.blobs.has(hash)); + if (gone) + throw new Error( + `An earlier version of ${gone[0]} is not kept, so this cannot be brought back.`, + ); const deleted = new Set(changes.filter(([, hash]) => hash === null).map(([path]) => path)); const blocked = changes .filter(([, hash]) => hash !== null) @@ -1288,6 +1304,9 @@ class Engine { if ((await readdir(dest).catch(missingIsEmpty)).length > 0) throw new Error(`Checkout writes into an empty folder only: ${dest}`); const files = manifestAround(this.log, entryId, side)!; + for (const [path, hash] of files) + if (!this.blobs.has(hash)) + throw new Error(`The version of ${path} from then is not kept.`); try { for (const [path, hash] of files) await this.blobs.writeTo(hash, join(dest, path)); } catch (error) { diff --git a/packages/studio-server/src/history/reclaimUncopied.test.ts b/packages/studio-server/src/history/reclaimUncopied.test.ts new file mode 100644 index 00000000000..c824ae6b8a5 --- /dev/null +++ b/packages/studio-server/src/history/reclaimUncopied.test.ts @@ -0,0 +1,61 @@ +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { openProjectHistory } from "./projectHistory"; +import { reclaimUncopiedBlobs } from "./reclaimUncopied"; +import { START } from "./historyLog"; + +const MiB = 1024 ** 2; +const cleanup: Array<() => unknown> = []; +afterEach(async () => { + for (const step of cleanup.splice(0).reverse()) await step(); +}); + +function tempDir(prefix: string): string { + const dir = mkdtempSync(join(tmpdir(), prefix)); + cleanup.push(() => rmSync(dir, { recursive: true, force: true })); + return dir; +} + +const blobFile = (home: string, content: Buffer) => { + const hash = createHash("sha256").update(content).digest("hex"); + return join(home, "blobs", hash.slice(0, 2), hash); +}; + +/** A closed history as an older version left it: the media's bytes sit in the blob store next to the text. */ +async function closedHistoryWithOldMediaCopy() { + const projectDir = tempDir("hf-reclaim-project-"); + const historyRoot = tempDir("hf-reclaim-root-"); + const media = Buffer.alloc(2 * MiB, 5); + writeFileSync(join(projectDir, "index.html"), "A"); + writeFileSync(join(projectDir, "clip.mp4"), media); + const history = await openProjectHistory({ projectDir, historyRoot, quietMs: 30 }); + await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.mp4"]), { + timeout: 10_000, + }); + await history.close(); + const home = join(historyRoot, history.projectId); + const copy = blobFile(home, media); + mkdirSync(join(copy, ".."), { recursive: true }); + writeFileSync(copy, media); + return { historyRoot, home, copy, text: blobFile(home, Buffer.from("A")) }; +} + +describe("reclaimUncopiedBlobs", () => { + it("lists the bytes of old media copies in every history and removes nothing on a dry run", async () => { + const { historyRoot, copy } = await closedHistoryWithOldMediaCopy(); + const found = await reclaimUncopiedBlobs(historyRoot, { dryRun: true }); + expect(found.map((history) => history.bytes)).toEqual([2 * MiB]); + expect(existsSync(copy)).toBe(true); + }); + + it("removes the media copies of a history nobody has open, and keeps its text copies", async () => { + const { historyRoot, copy, text } = await closedHistoryWithOldMediaCopy(); + await reclaimUncopiedBlobs(historyRoot); + expect(existsSync(copy)).toBe(false); + expect(existsSync(text)).toBe(true); + expect(await reclaimUncopiedBlobs(historyRoot)).toEqual([]); + }); +}); diff --git a/packages/studio-server/src/history/reclaimUncopied.ts b/packages/studio-server/src/history/reclaimUncopied.ts new file mode 100644 index 00000000000..6179d375ad6 --- /dev/null +++ b/packages/studio-server/src/history/reclaimUncopied.ts @@ -0,0 +1,76 @@ +import { readdirSync } from "node:fs"; +import { rm, stat } from "node:fs/promises"; +import { join } from "node:path"; +import { blobPath } from "./blobStore.js"; +import { isHistoryId } from "./historyId.js"; +import { readLog, type HistoryLog } from "./historyLog.js"; +import { HistoryBusyError, takeHistoryOwnership } from "./ownerLock.js"; +import { uncopiedOnlyHashes } from "./uncopied.js"; + +export interface ReclaimedHistory { + id: string; + bytes: number; +} + +function historyIds(historyRoot: string): string[] { + try { + return readdirSync(historyRoot).filter(isHistoryId); + } catch { + return []; + } +} + +function namedHashes(log: HistoryLog): Set { + const hashes = new Set(log.baseline.values()); + for (const entry of log.entries) { + for (const file of entry.files) { + if (file.before) hashes.add(file.before); + if (file.after) hashes.add(file.after); + } + } + return hashes; +} + +async function blobSizes(blobs: string, hashes: Iterable): Promise> { + const sizes = new Map(); + for (const hash of hashes) + sizes.set(hash, (await stat(blobPath(blobs, hash)).catch(() => null))?.size ?? 0); + return sizes; +} + +async function reclaimOne(home: string, dryRun: boolean): Promise { + const log = readLog(join(home, "log.jsonl"), () => {}); + if (!log) return 0; + const blobs = join(home, "blobs"); + const sizes = await blobSizes(blobs, namedHashes(log)); + let bytes = 0; + for (const hash of uncopiedOnlyHashes(log, (known) => sizes.get(known) ?? 0)) { + const size = sizes.get(hash) ?? 0; + if (size === 0) continue; + bytes += size; + if (!dryRun) await rm(blobPath(blobs, hash), { force: true }); + } + return bytes; +} + +/** Drops the media copies older versions left in every history no running process holds; the same rule as on open. */ +export async function reclaimUncopiedBlobs( + historyRoot: string, + { dryRun = false, onError }: { dryRun?: boolean; onError?: (error: unknown) => void } = {}, +): Promise { + const reclaimed: ReclaimedHistory[] = []; + for (const id of historyIds(historyRoot)) { + const home = join(historyRoot, id); + let release: (() => void) | undefined; + try { + release = await takeHistoryOwnership(home, 0); + const bytes = await reclaimOne(home, dryRun); + if (bytes > 0) reclaimed.push({ id, bytes }); + } catch (error) { + if (!(error instanceof HistoryBusyError)) onError?.(error); + } finally { + release?.(); + } + } + return reclaimed; +} diff --git a/packages/studio-server/src/history/uncopied.test.ts b/packages/studio-server/src/history/uncopied.test.ts new file mode 100644 index 00000000000..a3e935487df --- /dev/null +++ b/packages/studio-server/src/history/uncopied.test.ts @@ -0,0 +1,145 @@ +import { createHash } from "node:crypto"; +import { + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { openProjectHistory, type ProjectHistory } from "./projectHistory"; +import type { HistoryWho } from "./historyLog"; + +const you: HistoryWho = { kind: "person", name: "You" }; +const MiB = 1024 ** 2; +const cleanup: Array<() => unknown> = []; + +afterEach(async () => { + for (const step of cleanup.splice(0).reverse()) await step(); +}); + +function tempDir(prefix: string): string { + const dir = mkdtempSync(join(tmpdir(), prefix)); + cleanup.push(() => rmSync(dir, { recursive: true, force: true })); + return dir; +} + +async function open(projectDir: string, historyRoot: string) { + const history = await openProjectHistory({ projectDir, historyRoot, quietMs: 30 }); + cleanup.push(() => history.close()); + return history; +} + +function blobBytes(historyRoot: string, history: ProjectHistory): number { + const dir = join(historyRoot, history.projectId, "blobs"); + return readdirSync(dir).reduce( + (sum, shard) => + sum + + readdirSync(join(dir, shard)).reduce( + (inner, name) => inner + statSync(join(dir, shard, name)).size, + 0, + ), + 0, + ); +} + +async function project(files: Record) { + const projectDir = tempDir("hf-uncopied-project-"); + const historyRoot = tempDir("hf-uncopied-root-"); + const write = (path: string, content: string | Buffer) => { + mkdirSync(join(projectDir, path, ".."), { recursive: true }); + writeFileSync(join(projectDir, path), content); + }; + for (const [path, content] of Object.entries(files)) write(path, content); + const history = await open(projectDir, historyRoot); + return { projectDir, historyRoot, history, write }; +} + +describe("project history and media files", () => { + it("copies no bytes of a video that was there when history opened", async () => { + const { history, historyRoot } = await project({ + "index.html": "A", + "assets/camera.mp4": Buffer.alloc(50 * MiB, 3), + }); + await vi.waitFor(() => expect(history.peek("start")).toHaveProperty(["assets/camera.mp4"]), { + timeout: 10_000, + }); + expect(blobBytes(historyRoot, history)).toBeLessThan(1024); + }); + + it("copies no bytes of media written or replaced while history is open", async () => { + const { history, historyRoot, write } = await project({ "index.html": "A" }); + const window = await history.beginWindow(you, "Render"); + write("out.mp4", Buffer.alloc(20 * MiB, 1)); + write("voice.wav", Buffer.alloc(5 * MiB, 2)); + await window.close(); + const again = await history.beginWindow(you, "Render again"); + write("out.mp4", Buffer.alloc(20 * MiB, 9)); + const entry = await again.close(); + expect(entry?.files.map((file) => file.path)).toEqual(["out.mp4"]); + expect(blobBytes(historyRoot, history)).toBeLessThan(1024); + }); + + it("still copies a text edit and says plainly that a media version is not kept", async () => { + const { history, historyRoot, write } = await project({ + "index.html": "A", + "clip.mp4": Buffer.alloc(2 * MiB, 1), + }); + await vi.waitFor(() => expect(history.peek("start")).toHaveProperty(["clip.mp4"]), { + timeout: 10_000, + }); + const edit = await history.beginWindow(you, "Edit"); + write("index.html", "B"); + write("clip.mp4", Buffer.alloc(2 * MiB, 2)); + const entry = (await edit.close())!; + expect(blobBytes(historyRoot, history)).toBeLessThan(2048); + await expect(history.undo(entry.id, { who: you })).rejects.toThrow(/clip\.mp4 is not kept/); + }); + + it("reclaims media copies an older version left, and keeps every text copy", async () => { + const { history, historyRoot, projectDir } = await project({ + "index.html": "A", + "clip.mp4": Buffer.alloc(2 * MiB, 5), + }); + await vi.waitFor(() => expect(history.peek("start")).toHaveProperty(["clip.mp4"]), { + timeout: 10_000, + }); + await history.close(); + const media = Buffer.alloc(2 * MiB, 5); + const hash = createHash("sha256").update(media).digest("hex"); + const planted = join(historyRoot, history.projectId, "blobs", hash.slice(0, 2)); + mkdirSync(planted, { recursive: true }); + writeFileSync(join(planted, hash), media); + expect(blobBytes(historyRoot, history)).toBeGreaterThanOrEqual(2 * MiB); + + const reopened = await open(projectDir, historyRoot); + await reopened.flush(); + expect(blobBytes(historyRoot, reopened)).toBeLessThan(1024); + expect(await reopened.readBlob(reopened.peek("start")!["index.html"]!)).toEqual( + Buffer.from("A"), + ); + }); + + it("keeps TypeScript source edits undoable, and reclaim leaves their copies", async () => { + const { history, historyRoot, projectDir, write } = await project({ + "index.html": "A", + "setup.ts": "export const a = 1;", + "intro.mts": "export const b = 1;", + }); + await history.flush(); + const edit = await history.beginWindow(you, "Edit"); + write("setup.ts", "export const a = 2;"); + write("intro.mts", "export const b = 2;"); + const entry = (await edit.close())!; + expect(blobBytes(historyRoot, history)).toBeGreaterThan(40); + await history.close(); + const reopened = await open(projectDir, historyRoot); + expect(blobBytes(historyRoot, reopened)).toBeGreaterThan(40); + expect((await reopened.undo(entry.id, { who: you })).ok).toBe(true); + expect(readFileSync(join(projectDir, "setup.ts"), "utf-8")).toBe("export const a = 1;"); + }); +}); diff --git a/packages/studio-server/src/history/uncopied.ts b/packages/studio-server/src/history/uncopied.ts new file mode 100644 index 00000000000..34460b29ed4 --- /dev/null +++ b/packages/studio-server/src/history/uncopied.ts @@ -0,0 +1,41 @@ +import { extname } from "node:path"; +import type { HistoryLog } from "./historyLog.js"; + +const MEDIA_EXTENSIONS = new Set([ + ".mp4", + ".mov", + ".m4v", + ".webm", + ".mkv", + ".avi", + ".mxf", + ".m2ts", + ".mp3", + ".wav", + ".aac", + ".m4a", + ".flac", + ".ogg", + ".opus", +]); +const COPY_NOTHING_ABOVE_BYTES = 64 * 1024 ** 2; + +/** Video, audio and any file over 64 MiB are hashed in place, never copied into history. */ +export const isUncopied = (path: string, size: number): boolean => + size > COPY_NOTHING_ABOVE_BYTES || MEDIA_EXTENSIONS.has(extname(path).toLowerCase()); + +/** Hashes whose every reference in the log is a file that is no longer copied: their old blobs can go. */ +export function uncopiedOnlyHashes(log: HistoryLog, sizeOf: (hash: string) => number): string[] { + const copied = new Set(); + const uncopied = new Set(); + const note = (path: string, hash: string | null) => { + if (hash) (isUncopied(path, sizeOf(hash)) ? uncopied : copied).add(hash); + }; + for (const [path, hash] of log.baseline) note(path, hash); + for (const entry of log.entries) + for (const file of entry.files) { + note(file.path, file.before); + note(file.path, file.after); + } + return [...uncopied].filter((hash) => !copied.has(hash)); +} diff --git a/packages/studio-server/src/routes/history.test.ts b/packages/studio-server/src/routes/history.test.ts index 390e62f142a..f1823cb6087 100644 --- a/packages/studio-server/src/routes/history.test.ts +++ b/packages/studio-server/src/routes/history.test.ts @@ -25,7 +25,7 @@ vi.mock("../history/blobStore", async (importOriginal) => { return { ...store, put: async (path: string) => { - if (path.endsWith(".mp4")) await mediaCopy.held; + if (path.endsWith(".bin")) await mediaCopy.held; return store.put(path); }, }; @@ -410,7 +410,7 @@ describe("history routes", () => { let finishCopy = () => {}; mediaCopy.held = new Promise((resolve) => (finishCopy = resolve)); const { projectDir, file, put, post, histories } = lazyHistoryProject(); - writeFileSync(join(projectDir, "clip.mp4"), Buffer.alloc(4 * 1024 ** 2, 7)); + writeFileSync(join(projectDir, "clip.bin"), Buffer.alloc(4 * 1024 ** 2, 7)); // The copy stays held until finishCopy, so a save or claim that waited on it never returns. expect((await put("B")).status).toBe(200); @@ -420,7 +420,7 @@ describe("history routes", () => { finishCopy(); const history = (await histories.get(projectDir))!; - await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.mp4"]), { + await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.bin"]), { timeout: 10_000, }); await history.flush(); @@ -431,12 +431,12 @@ describe("history routes", () => { let finishCopy = () => {}; mediaCopy.held = new Promise((resolve) => (finishCopy = resolve)); const { projectDir, api, post } = lazyHistoryProject(); - const clip = join(projectDir, "clip.mp4"); + const clip = join(projectDir, "clip.bin"); const original = Buffer.alloc(4 * 1024 ** 2, 7); writeFileSync(clip, original); await post("/claim", { label: "Nothing", paths: [] }); - const written = await api.request("/projects/demo/files/clip.mp4", { + const written = await api.request("/projects/demo/files/clip.bin", { method: "PUT", body: Buffer.alloc(4 * 1024 ** 2, 9), headers: { "If-Match": fileContentVersion(original) }, @@ -445,8 +445,8 @@ describe("history routes", () => { finishCopy(); const claimed = await post("/claim", { label: "Replace clip", - paths: ["clip.mp4"], - overwrote: { "clip.mp4": fileContentVersion(original) }, + paths: ["clip.bin"], + overwrote: { "clip.bin": fileContentVersion(original) }, }); expect(((await claimed.json()) as { claimed: unknown }).claimed).not.toBeNull();