fix(studio-server): project history no longer copies video, audio or huge files

packages/cli/src/commands/clean.ts

@@ -14,6 +14,7 @@ import {
   cleanupProxyCache,
   listProjectHistories,
   pruneGoneProjectHistories,
+  reclaimUncopiedBlobs,
 } from "@hyperframes/studio-server";
 import type { Example } from "./_examples.js";
 import { c } from "../ui/colors.js";
@@ -172,6 +173,17 @@ async function sweepHistories(sweep: Sweep, historyRoot: string, tempDir: string
       bytes: history.bytes,
     });
   }
+  const reclaimed = await reclaimUncopiedBlobs(historyRoot, {
+    dryRun: sweep.dryRun,
+    onError: (error) => recordError(sweep, error),
+  });
+  for (const history of reclaimed) {
+    sweep.result.removed.push({
+      what: "Video and audio copies in project history",
+      path: join(historyRoot, history.id, "blobs"),
+      bytes: history.bytes,
+    });
+  }
 }
 
 function sweepExtractCache(sweep: Sweep): void {

packages/studio-server/src/history/blobStore.ts

@@ -9,18 +9,20 @@ import { atomicTempPath } from "@hyperframes/core/atomic-file";
 export interface BlobStore {
   /** Copies the file in (a clone where the file system can) and returns the copy's hash. */
   put(absPath: string): Promise<string>;
+  reference(absPath: string): Promise<string>;
   has(hash: string): boolean;
   read(hash: string): Promise<Buffer>;
   /** Writes the blob's bytes to `absPath` by clone-or-copy and rename, so a reader never sees half a file. */
   writeTo(hash: string, absPath: string, beforeReplace?: () => void): Promise<void>;
   bytes(): number;
   size(hash: string): number;
   prune(keep: ReadonlySet<string>): Promise<void>;
+  drop(hashes: Iterable<string>): Promise<void>;
 }
 
 const BLOB_HASH = /^[0-9a-f]{64}$/;
 
-function blobPath(dir: string, hash: string): string {
+export function blobPath(dir: string, hash: string): string {
   // Checked where the path is joined, so no caller can read or write outside the store.
   if (!BLOB_HASH.test(hash)) throw new Error("That is not a history blob.");
   return join(dir, hash.slice(0, 2), hash);
@@ -87,12 +89,22 @@ export async function openBlobStore(dir: string): Promise<BlobStore> {
       total += size;
       return hash;
     },
+    reference: (absPath) => hashFile(absPath),
     has: (hash) => sizes.has(hash),
     read: async (hash) => readBlob(dir, hash),
     writeTo: async (hash, absPath, beforeReplace) =>
       cloneOrCopy(pathOf(hash), absPath, beforeReplace),
     bytes: () => total,
     size: (hash) => sizes.get(hash) ?? 0,
+    async drop(hashes) {
+      for (const hash of hashes) {
+        const size = sizes.get(hash);
+        if (size === undefined) continue;
+        await rm(pathOf(hash), { force: true });
+        sizes.delete(hash);
+        total -= size;
+      }
+    },
     async prune(keep) {
       for (const [hash, size] of sizes) {
         if (keep.has(hash)) continue;

packages/studio-server/src/history/index.ts

@@ -14,6 +14,7 @@ export {
   type HistoryWindow,
   HistoryClosedError,
 } from "./projectHistory.js";
+export { reclaimUncopiedBlobs, type ReclaimedHistory } from "./reclaimUncopied.js";
 export { HistoryBusyError } from "./ownerLock.js";
 export { HistoryIdError } from "./historyId.js";
 export { historyCache } from "./historyCache.js";

packages/studio-server/src/history/projectHistory.test.ts

@@ -48,13 +48,13 @@ vi.mock("./blobStore", async (importOriginal) => {
       return {
         ...store,
         put: async (path: string) => {
-          if (path.endsWith(".mp4")) {
+          if (path.endsWith(".bin")) {
             await mediaCopy.held;
             mediaCopy.copies.push(basename(path));
             if (mediaCopy.fails?.(mediaCopy.copies.length)) throw new Error("The disk is full.");
           }
           const hash = await store.put(path);
-          if (path.endsWith(".mp4")) await mediaCopy.stored?.();
+          if (path.endsWith(".bin")) await mediaCopy.stored?.();
           return hash;
         },
       };
@@ -306,10 +306,10 @@ describe("openProjectHistory", () => {
 
   it("a ledger made while closed is still a change after the project's media was copied in the background", async () => {
     const { history, write, has, projectDir, historyRoot } = await project(
-      { "index.html": "A", "clip.mp4": Buffer.alloc(2 * 1024 ** 2, 7) },
+      { "index.html": "A", "clip.bin": Buffer.alloc(2 * 1024 ** 2, 7) },
       { quietMs: 30 },
     );
-    await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.mp4"]), {
+    await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.bin"]), {
       timeout: 10_000,
     });
     await history.close();
@@ -331,20 +331,20 @@ describe("openProjectHistory", () => {
     mediaCopy.held = new Promise<void>((resolve) => (release = resolve));
     const { history, historyRoot } = await project({
       "index.html": "A",
-      "clip.mp4": Buffer.alloc(2 * 1024 ** 2, 7),
+      "clip.bin": Buffer.alloc(2 * 1024 ** 2, 7),
     });
     const adopting = join(historyRoot, history.projectId, "adopting.json");
-    expect(JSON.parse(readFileSync(adopting, "utf-8"))).toEqual(["clip.mp4"]);
+    expect(JSON.parse(readFileSync(adopting, "utf-8"))).toEqual(["clip.bin"]);
     release();
-    await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.mp4"]), {
+    await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.bin"]), {
       timeout: 10_000,
     });
     expect(existsSync(adopting)).toBe(false);
   });
 
   it("keeps media whose recording failed for the next open, rather than log it as added", async () => {
     const { history, write, historyRoot } = await project(
-      { "index.html": "A", "clip.mp4": Buffer.alloc(2 * 1024 ** 2, 7) },
+      { "index.html": "A", "clip.bin": Buffer.alloc(2 * 1024 ** 2, 7) },
       { budgetBytes: 1, quietMs: 30 },
     );
     // The edit's prune takes the first copy; storing it again then fails.
@@ -356,18 +356,18 @@ describe("openProjectHistory", () => {
     await vi.waitFor(() => expect(mediaCopy.copies).toHaveLength(2), { timeout: 10_000 });
     await history.flush();
     expect(history.list().flatMap((entry) => entry.files.map((file) => file.path))).not.toContain(
-      "clip.mp4",
+      "clip.bin",
     );
     const adopting = join(historyRoot, history.projectId, "adopting.json");
-    expect(JSON.parse(readFileSync(adopting, "utf-8"))).toEqual(["clip.mp4"]);
+    expect(JSON.parse(readFileSync(adopting, "utf-8"))).toEqual(["clip.bin"]);
   });
 
   it("never copies the media of a folder that replaced the project", async () => {
     let release = () => {};
     mediaCopy.held = new Promise<void>((resolve) => (release = resolve));
     const media = {
-      "a.mp4": Buffer.alloc(2 * 1024 ** 2, 1),
-      "b.mp4": Buffer.alloc(2 * 1024 ** 2, 2),
+      "a.bin": Buffer.alloc(2 * 1024 ** 2, 1),
+      "b.bin": Buffer.alloc(2 * 1024 ** 2, 2),
     };
     const { history, write, projectDir } = await project({ "index.html": "A", ...media });
     renameSync(projectDir, `${projectDir}-moved`);
@@ -377,38 +377,38 @@ describe("openProjectHistory", () => {
     release();
     await history.close();
     // Only the copy already running when the folder changed may finish.
-    expect(mediaCopy.copies).toEqual(["a.mp4"]);
+    expect(mediaCopy.copies).toEqual(["a.bin"]);
   });
 
   it("undoes a clip replaced while closed, though the history closed before its copy finished", async () => {
     const clip = Buffer.alloc(2 * 1024 ** 2, 7);
     let release = () => {};
     mediaCopy.held = new Promise<void>((resolve) => (release = resolve));
     const { history, write, projectDir, historyRoot } = await project(
-      { "index.html": "A", "clip.mp4": clip },
+      { "index.html": "A", "clip.bin": clip },
       { quietMs: 30 },
     );
     const closed = history.close();
     release();
     await closed;
-    write("clip.mp4", Buffer.alloc(2 * 1024 ** 2, 9));
+    write("clip.bin", Buffer.alloc(2 * 1024 ** 2, 9));
     const reopened = await open(projectDir, historyRoot, { quietMs: 30 });
     await reopened.flush();
     expect(reopened.list().map((entry) => entry.files.map((file) => file.path))).toEqual([
-      ["clip.mp4"],
+      ["clip.bin"],
     ]);
     expect(await reopened.step("back", you)).toMatchObject({ ok: true });
-    expect(readFileSync(join(projectDir, "clip.mp4")).equals(clip)).toBe(true);
+    expect(readFileSync(join(projectDir, "clip.bin")).equals(clip)).toBe(true);
   });
 
   it("adopts media whose copy a process exit cut short, rather than log it as added", async () => {
     const { history, write, projectDir, historyRoot } = await project({ "index.html": "A" });
     await history.close();
     // What an exit mid-copy leaves: the clip on disk, named as still being copied, not in the baseline.
-    write("clip.mp4", Buffer.alloc(2 * 1024 ** 2, 7));
-    writeFileSync(join(historyRoot, history.projectId, "adopting.json"), '["clip.mp4"]');
+    write("clip.bin", Buffer.alloc(2 * 1024 ** 2, 7));
+    writeFileSync(join(historyRoot, history.projectId, "adopting.json"), '["clip.bin"]');
     const reopened = await open(projectDir, historyRoot, { quietMs: 30 });
-    await vi.waitFor(() => expect(reopened.peek(START)).toHaveProperty(["clip.mp4"]), {
+    await vi.waitFor(() => expect(reopened.peek(START)).toHaveProperty(["clip.bin"]), {
       timeout: 10_000,
     });
     await reopened.flush();
@@ -420,7 +420,7 @@ describe("openProjectHistory", () => {
     let release = () => {};
     mediaCopy.held = new Promise<void>((resolve) => (release = resolve));
     const { history, write } = await project(
-      { "index.html": "A", "clip.mp4": clip },
+      { "index.html": "A", "clip.bin": clip },
       { budgetBytes: 1, quietMs: 30 },
     );
     // The copy is stored; an edit commits and prunes before history records the copy.
@@ -429,10 +429,10 @@ describe("openProjectHistory", () => {
       return change(history, you, "Color", () => write("index.html", "B"));
     };
     release();
-    await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.mp4"]), {
+    await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.bin"]), {
       timeout: 10_000,
     });
-    const start = history.peek(START)["clip.mp4"]!;
+    const start = history.peek(START)["clip.bin"]!;
     expect((await history.readBlob(start)).equals(clip)).toBe(true);
   });
 

packages/studio-server/src/history/projectHistory.ts

@@ -1,5 +1,5 @@
 import { randomUUID } from "node:crypto";
-import { mkdir, readdir, rm, rmdir, writeFile } from "node:fs/promises";
+import { mkdir, readdir, rm, rmdir, stat, writeFile } from "node:fs/promises";
 import {
   type Dirent,
   type Stats,
@@ -30,6 +30,7 @@ import {
   listProjectFiles,
 } from "../helpers/projectSignature.js";
 import { openBlobStore, readBlob, type BlobStore } from "./blobStore.js";
+import { isUncopied, uncopiedOnlyHashes } from "./uncopied.js";
 import { pruneGoneProjectHistoriesDaily } from "./pruneHistories.js";
 import {
   ID_PATH,
@@ -460,6 +461,7 @@ class Engine {
     );
     if (!log) return this.firstOpen();
     this.log = withoutHiddenPaths(log);
+    await this.reclaimUncopied();
     if (!this.log.keepsLedger) await this.takeInUnnamedLedger();
     const cache = this.readStatCache();
     const last = this.log.entries.at(-1)?.id ?? START;
@@ -476,6 +478,11 @@ class Engine {
     await this.settleAll();
   }
 
+  /** Copies an older version took of files that are no longer copied; text and small files are never touched. */
+  async reclaimUncopied(): Promise<void> {
+    await this.blobs.drop(uncopiedOnlyHashes(this.log, (hash) => this.blobs.size(hash)));
+  }
+
   hearWrites(): void {
     const realDir = realFilePath(this.dir);
     this.stopHearing = onFileOverwritten((absPath, version, bytes) => {
@@ -491,7 +498,8 @@ class Engine {
         chainBackFrom(this.overwritten.get(path), replaced).set(after, replaced),
       );
       this.queue(async () => {
-        if (!this.blobs.has(replaced)) await this.storeBytes(bytes);
+        if (!this.blobs.has(replaced) && !isUncopied(path, bytes.length))
+          await this.storeBytes(bytes);
       }).catch((error) => this.options.onError?.(error));
     });
   }
@@ -589,7 +597,8 @@ class Engine {
   async recordAdopted(file: ListedFile, sweptAt: number, stored: string | null) {
     if (stored === null || this.whereFolder() !== "here") return;
     // A prune that ran while this copy waited for the queue may have taken its bytes.
-    const hash = this.blobs.has(stored) ? stored : await this.storeIfPresent(file.path);
+    const kept = this.blobs.has(stored) || isUncopied(file.path, file.size);
+    const hash = kept ? stored : await this.storeIfPresent(file.path);
     if (hash === null) return;
     const change = { path: file.path, before: null, after: hash };
     const found = this.overwrittenBy(change, undefined, new Set());
@@ -679,7 +688,9 @@ class Engine {
   /** The file's hash once copied in; null when it was removed before the copy (the next sweep records that). */
   async storeIfPresent(path: string): Promise<string | null> {
     try {
-      return await this.blobs.put(join(this.dir, path));
+      const abs = join(this.dir, path);
+      const { size } = await stat(abs);
+      return await (isUncopied(path, size) ? this.blobs.reference(abs) : this.blobs.put(abs));
     } catch (error) {
       if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
       throw error;
@@ -1096,6 +1107,11 @@ class Engine {
     const changes = [...target]
       .filter(([path, hash]) => (this.tracked.get(path)?.hash ?? null) !== hash)
       .sort(([, a], [, b]) => Number(a !== null) - Number(b !== null));
+    const gone = changes.find(([, hash]) => hash !== null && !this.blobs.has(hash));
+    if (gone)
+      throw new Error(
+        `An earlier version of ${gone[0]} is not kept, so this cannot be brought back.`,
+      );
     const deleted = new Set(changes.filter(([, hash]) => hash === null).map(([path]) => path));
     const blocked = changes
       .filter(([, hash]) => hash !== null)
@@ -1288,6 +1304,9 @@ class Engine {
           if ((await readdir(dest).catch(missingIsEmpty)).length > 0)
             throw new Error(`Checkout writes into an empty folder only: ${dest}`);
           const files = manifestAround(this.log, entryId, side)!;
+          for (const [path, hash] of files)
+            if (!this.blobs.has(hash))
+              throw new Error(`The version of ${path} from then is not kept.`);
           try {
             for (const [path, hash] of files) await this.blobs.writeTo(hash, join(dest, path));
           } catch (error) {

packages/studio-server/src/history/reclaimUncopied.test.ts

@@ -0,0 +1,61 @@
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { openProjectHistory } from "./projectHistory";
+import { reclaimUncopiedBlobs } from "./reclaimUncopied";
+import { START } from "./historyLog";
+
+const MiB = 1024 ** 2;
+const cleanup: Array<() => unknown> = [];
+afterEach(async () => {
+  for (const step of cleanup.splice(0).reverse()) await step();
+});
+
+function tempDir(prefix: string): string {
+  const dir = mkdtempSync(join(tmpdir(), prefix));
+  cleanup.push(() => rmSync(dir, { recursive: true, force: true }));
+  return dir;
+}
+
+const blobFile = (home: string, content: Buffer) => {
+  const hash = createHash("sha256").update(content).digest("hex");
+  return join(home, "blobs", hash.slice(0, 2), hash);
+};
+
+/** A closed history as an older version left it: the media's bytes sit in the blob store next to the text. */
+async function closedHistoryWithOldMediaCopy() {
+  const projectDir = tempDir("hf-reclaim-project-");
+  const historyRoot = tempDir("hf-reclaim-root-");
+  const media = Buffer.alloc(2 * MiB, 5);
+  writeFileSync(join(projectDir, "index.html"), "A");
+  writeFileSync(join(projectDir, "clip.mp4"), media);
+  const history = await openProjectHistory({ projectDir, historyRoot, quietMs: 30 });
+  await vi.waitFor(() => expect(history.peek(START)).toHaveProperty(["clip.mp4"]), {
+    timeout: 10_000,
+  });
+  await history.close();
+  const home = join(historyRoot, history.projectId);
+  const copy = blobFile(home, media);
+  mkdirSync(join(copy, ".."), { recursive: true });
+  writeFileSync(copy, media);
+  return { historyRoot, home, copy, text: blobFile(home, Buffer.from("A")) };
+}
+
+describe("reclaimUncopiedBlobs", () => {
+  it("lists the bytes of old media copies in every history and removes nothing on a dry run", async () => {
+    const { historyRoot, copy } = await closedHistoryWithOldMediaCopy();
+    const found = await reclaimUncopiedBlobs(historyRoot, { dryRun: true });
+    expect(found.map((history) => history.bytes)).toEqual([2 * MiB]);
+    expect(existsSync(copy)).toBe(true);
+  });
+
+  it("removes the media copies of a history nobody has open, and keeps its text copies", async () => {
+    const { historyRoot, copy, text } = await closedHistoryWithOldMediaCopy();
+    await reclaimUncopiedBlobs(historyRoot);
+    expect(existsSync(copy)).toBe(false);
+    expect(existsSync(text)).toBe(true);
+    expect(await reclaimUncopiedBlobs(historyRoot)).toEqual([]);
+  });
+});