diff --git a/README.md b/README.md index 8a6c1b1..fddec50 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ Run allowlisted commands in a named tmux session, capture output + exit code. 11. **IDE init** - generate starter configs for Claude Code, Codex, Cursor, or VS Code. 12. **ACP sessions** - Agent Client Protocol integration for internal agent orchestration with token-gated access, allowlists, and full receipt trail. 13. **Background consolidation** - optional `light / REM / deep` pass over recent queue items, with append-only sidecars and no effect on the foreground room loop by default. +14. **model-tidy** - plan/apply tool for moving idle local LLM model directories off a full GPU-box drive onto another mount, leaving a symlink behind. Read-only `plan` by default; `apply` needs an explicit flag plus a verified, cross-filesystem target. See [docs/model-tidy.md](docs/model-tidy.md). No dependencies. Node.js ≥ 18 only. diff --git a/bin/model-tidy.mjs b/bin/model-tidy.mjs new file mode 100755 index 0000000..68cef64 --- /dev/null +++ b/bin/model-tidy.mjs @@ -0,0 +1,255 @@ +#!/usr/bin/env node +// SPDX-License-Identifier: AGPL-3.0-only + +/** + * model-tidy CLI. + * + * model-tidy plan [--home ] [--keep-file ] [--min-idle-days N] + * [--max-gb N] [--json] [--log-dir ] + * [--report-to-room] [--room ] [--config ] + * Read-only. Never mutates anything. If a previous `apply` was + * interrupted (crash, kill, power loss), plan DETECTS and reports it + * per unit ("interrupted move found: ...") and refuses to select that + * unit — it does not attempt to fix it. Run `recover` for that. + * + * model-tidy apply --apply --target [same options as plan] + * Runs recovery once at the very start (only reached because --apply + * was given), then copies/verifies/swaps every selected unit. + * + * model-tidy recover [--home ] + * Explicit, mutating recovery pass for interrupted `apply` swaps. Acts + * ONLY on units with a journal record whose on-disk state matches the + * journaled manifest; anything else (including a directory merely + * *named* like a leftover, with no journal) is reported and left + * alone. Safe to run at any time, including on a healthy tree (a + * no-op). + * + * model-tidy --dry-run-remote [--remote-home ] + * ssh's to and runs `plan` there, read-only. Never copies, + * deletes, or symlinks anything. If the remote has no `node`, this + * refuses and exits non-zero rather than installing one. + * + * See docs/model-tidy.md for full documentation. + */ + +import { execFileSync, spawnSync } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { homedir } from 'node:os'; +import { planRun, applyRun, recoverInterruptedMoves, writeRunLog } from '../src/model-tidy.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = join(__dirname, '..'); +const DEFAULT_LOG_DIR = join(homedir(), '.cache', 'ide-agent-kit', 'model-tidy-logs'); +const DEFAULT_KEEP_FILE = join(homedir(), '.config', 'ide-agent-kit', 'model-tidy.keep'); + +function parseArgs(argv) { + const out = { _: [] }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg.startsWith('--')) { + const eq = arg.indexOf('='); + if (eq !== -1) { + out[arg.slice(2, eq)] = arg.slice(eq + 1); + continue; + } + const key = arg.slice(2); + const next = argv[i + 1]; + if (next === undefined || next.startsWith('--')) { + out[key] = true; + } else { + out[key] = next; + i++; + } + } else { + out._.push(arg); + } + } + return out; +} + +function printPlan(plan, opts) { + if (opts.json) { + console.log(JSON.stringify(plan, null, 2)); + return; + } + console.log(plan.summaryLine); + console.log(''); + console.log(`Would move (${plan.selected.length}):`); + const printedGroups = new Set(); + for (const r of plan.selected) { + const gib = (r.sizeBytes / 2 ** 30).toFixed(2); + console.log(` [move] ${r.path} (${gib} GiB, ${r.kind}) — ${r.reason}`); + printedGroups.add(r.groupId); + } + console.log(''); + console.log(`Skipped (${plan.skipped.length}):`); + for (const r of plan.skipped) { + const gib = (r.sizeBytes / 2 ** 30).toFixed(2); + console.log(` [skip] ${r.path} (${gib} GiB, ${r.kind}) — ${r.reason}`); + } + if (plan.interrupted && plan.interrupted.length > 0) { + console.log(''); + console.log(`Interrupted-move findings (${plan.interrupted.length}) — plan never mutates these, run 'model-tidy recover':`); + for (const f of plan.interrupted) { + console.log(` [${f.status}] ${f.path || f.journalFile} — ${f.note}`); + } + } +} + +function reportToRoom(summaryLine, args) { + const room = args.room; + if (!room) { + console.error('--report-to-room requires --room '); + return false; + } + let apiKey = process.env.IAK_API_KEY; + if (!apiKey && args.config && existsSync(args.config)) { + try { + const cfg = JSON.parse(readFileSync(args.config, 'utf8')); + apiKey = cfg?.poller?.api_key; + } catch (e) { + console.error(`could not read --config ${args.config}: ${e.message}`); + } + } + if (!apiKey) { + console.error('--report-to-room: no API key (set IAK_API_KEY or pass --config pointing at a poller config with poller.api_key)'); + return false; + } + const payload = JSON.stringify({ room, body: summaryLine }); + try { + execFileSync('curl', ['-sS', '-X', 'POST', 'https://groupmind.one/api/v1/messages', + '-H', `X-API-Key: ${apiKey}`, '-H', 'Content-Type: application/json', '-d', payload], + { timeout: 15000 }); + return true; + } catch (e) { + console.error(`--report-to-room: post failed: ${e.message}`); + return false; + } +} + +function runDryRunRemote(host, args) { + console.log(`[dry-run-remote] checking node on ${host}...`); + const check = spawnSync('ssh', ['-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8', host, 'command -v node'], { encoding: 'utf8' }); + if (check.status !== 0 || !check.stdout.trim()) { + console.error(`[dry-run-remote] ${host}: node not found on remote. Refusing to install anything.`); + console.error('[dry-run-remote] Install Node.js >= 18 on the remote host, then re-run --dry-run-remote.'); + return 1; + } + const nodeVersionOut = check.stdout.trim(); + console.log(`[dry-run-remote] ${host}: found node at ${nodeVersionOut}`); + + const remoteDir = `/tmp/.model-tidy-dryrun-${process.pid}-${Date.now()}`; + console.log(`[dry-run-remote] copying tool to ${host}:${remoteDir} (temp, read-only run, self-cleaned)...`); + const mkdir = spawnSync('ssh', ['-o', 'BatchMode=yes', host, `mkdir -p '${remoteDir}/bin' '${remoteDir}/src'`], { encoding: 'utf8' }); + if (mkdir.status !== 0) { + console.error(`[dry-run-remote] mkdir on remote failed: ${mkdir.stderr}`); + return 1; + } + const scpBin = spawnSync('scp', ['-q', join(REPO_ROOT, 'bin', 'model-tidy.mjs'), `${host}:${remoteDir}/bin/model-tidy.mjs`], { encoding: 'utf8' }); + const scpSrc = spawnSync('scp', ['-q', join(REPO_ROOT, 'src', 'model-tidy.mjs'), `${host}:${remoteDir}/src/model-tidy.mjs`], { encoding: 'utf8' }); + if (scpBin.status !== 0 || scpSrc.status !== 0) { + console.error(`[dry-run-remote] scp failed: ${scpBin.stderr || ''} ${scpSrc.stderr || ''}`); + spawnSync('ssh', [host, `rm -rf '${remoteDir}'`]); + return 1; + } + + const remoteHome = args['remote-home'] || '$HOME'; + const remoteCmd = `node '${remoteDir}/bin/model-tidy.mjs' plan --home "${remoteHome}"`; + console.log(`[dry-run-remote] running (read-only): ${remoteCmd}`); + const run = spawnSync('ssh', ['-o', 'BatchMode=yes', host, remoteCmd], { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }); + console.log(run.stdout || ''); + if (run.stderr) console.error(run.stderr); + + spawnSync('ssh', [host, `rm -rf '${remoteDir}'`]); + console.log(`[dry-run-remote] cleaned up ${host}:${remoteDir}`); + return run.status === 0 ? 0 : (run.status ?? 1); +} + +function main() { + const argv = process.argv.slice(2); + const args = parseArgs(argv); + const mode = args._[0] || (args.apply ? 'apply' : 'plan'); + + if (args['dry-run-remote']) { + process.exit(runDryRunRemote(args['dry-run-remote'], args)); + return; + } + + const home = args.home || homedir(); + const keepFile = args['keep-file'] || (existsSync(DEFAULT_KEEP_FILE) ? DEFAULT_KEEP_FILE : undefined); + const minIdleDays = args['min-idle-days'] !== undefined ? Number(args['min-idle-days']) : 14; + if (!Number.isFinite(minIdleDays) || minIdleDays < 0) { + console.error(`--min-idle-days must be a finite, non-negative number (got ${JSON.stringify(args['min-idle-days'])}) — refusing, since a NaN here would silently disable the freshness guard`); + process.exit(2); + } + const maxGb = args['max-gb'] !== undefined ? Number(args['max-gb']) : Infinity; + if (!Number.isFinite(maxGb) && args['max-gb'] !== undefined) { + console.error(`--max-gb must be a finite number (got ${JSON.stringify(args['max-gb'])})`); + process.exit(2); + } + const logDir = args['log-dir'] || DEFAULT_LOG_DIR; + + if (mode === 'recover') { + const recovered = recoverInterruptedMoves(home); + writeRunLog(logDir, { mode: 'recover', home, recovered, argv }); + if (args.json) { + console.log(JSON.stringify(recovered, null, 2)); + } else if (recovered.length === 0) { + console.log('recover: nothing to do (no journaled interrupted moves, no stray leftovers found).'); + } else { + console.log(`recover: ${recovered.length} finding(s):`); + for (const r of recovered) { + console.log(` [${r.action}] ${r.path || r.journalFile} — ${r.note}`); + } + } + process.exit(recovered.some(r => r.action === 'error') ? 1 : 0); + return; + } + + const plan = planRun({ home, keepFile, minIdleDays, maxGb }); + + const logFile = writeRunLog(logDir, { + mode, + ...plan, + argv + }); + + if (mode === 'plan') { + printPlan(plan, args); + console.log(`\n(log: ${logFile})`); + if (args['report-to-room']) reportToRoom(plan.summaryLine, args); + process.exit(0); + } + + if (mode === 'apply') { + if (!args.apply) { + console.error('apply mode requires the explicit --apply flag'); + process.exit(2); + } + if (!args.target) { + console.error('apply mode requires --target '); + process.exit(2); + } + printPlan(plan, args); + const result = applyRun({ plan, target: args.target, home }); + writeRunLog(logDir, { mode: 'apply', target: args.target, ...result }); + if (!result.ok) { + console.error('\napply FAILED for one or more units — sources for failed units were left untouched:'); + for (const err of result.errors) { + console.error(` [error] group ${err.groupId} (${err.step}): ${err.error}`); + } + } + console.log(`\nmoved ${result.moved.length} unit(s), ${result.errors.length} error(s). (log dir: ${logDir})`); + if (args['report-to-room']) { + reportToRoom(`model-tidy apply: moved ${result.moved.length} unit(s), ${result.errors.length} error(s)`, args); + } + process.exit(result.ok ? 0 : 1); + } + + console.error(`unknown mode: ${mode} (expected "plan", "apply", or "recover")`); + process.exit(2); +} + +main(); diff --git a/config/model-tidy.keep.example b/config/model-tidy.keep.example new file mode 100644 index 0000000..425fc44 --- /dev/null +++ b/config/model-tidy.keep.example @@ -0,0 +1,16 @@ +# model-tidy KEEP list — example file, NOT active by default. +# +# One path or glob per line. Anything matching is never moved by +# model-tidy, in either plan or apply mode. '#' starts a comment; blank +# lines are ignored. A leading '~' is expanded to the home directory the +# tool is run with (--home, or $HOME by default). +# +# To activate: copy this file to ~/.config/ide-agent-kit/model-tidy.keep +# (the default --keep-file path) or pass --keep-file explicitly, then edit +# it to match what you actually want kept. +# +# Seeded from the owner's notes (do not delete without petrus's word): +~/models/glm53-flash-q6/UD-Q6_K_XL +~/models/GLM-5.3-Flash-exl3-2.05bpw +~/DeepSeek-v4.1-Flash-EXL3-2x-DGX-Sparks +~/models/ltx-2.5 diff --git a/docs/model-tidy.md b/docs/model-tidy.md new file mode 100644 index 0000000..996fcc9 --- /dev/null +++ b/docs/model-tidy.md @@ -0,0 +1,565 @@ +# model-tidy + +Moves idle local LLM model directories off a full internal drive onto a +target mount (external SSD / NAS share), leaving a symlink behind so every +path that already points at the model — a docker bind mount, a script, a +shell alias — keeps working without edits. + +Built for the asus1/asus2 GPU boxes (single NVMe, running near-full), +serving models with vLLM in docker containers bind-mounting +`~/.cache/huggingface`. + +## Why Node, not a shell/Python script + +The repo is Node-only ("No dependencies. Node.js >= 18 only.", per the main +README) and every other host-side tool here (`src/session-keepalive.mjs`, +`src/room-automation.mjs`, etc.) follows that pattern: logic in +`src/.mjs`, a thin CLI wrapper in `bin/`, tests under `test/` using +`node:test` with `mkdtempSync`/`afterEach` cleanup. `model-tidy` follows the +same shape (`src/model-tidy.mjs` + `bin/model-tidy.mjs` + +`test/model-tidy.test.mjs`) rather than introducing Python or a new shell +tool family. There is no existing systemd pattern in this repo (it ships +macOS launchd `.plist` files for its own daemons), but the target hosts are +Ubuntu, so `systemd/model-tidy.{service,timer}` are new but self-contained +units, not wired into the macOS installer. + +One deliberate deviation from the spec's suggested implementation: the copy +step is a small hand-written recursive Node copy +(`copyUnitPureNode` in `src/model-tidy.mjs`), not a shelled-out `rsync`. +Reasons: +- Keeps the zero-dependency, Node-only posture consistent with the rest of + the repo — no assumption that `rsync` is installed or that its flags + behave identically across the dev machine, CI, and asus1/asus2. +- `rsync -H` only preserves a hardlink relationship between files that are + named in the *same* `rsync` invocation. The pure-Node copy tracks a + `dev:ino -> target path` map explicitly across every member of a + hardlink unit and calls `fs.linkSync` for repeats — the same safety + property, verified directly by a test rather than relying on `rsync` + flag behavior. +- It makes the apply tests deterministic on any machine (no dependency on + `rsync` being present, or on its exact hardlink/`--checksum` semantics). + +Byte-for-byte verification (size + SHA-256 of every regular file, plus +symlink-target and file-count comparison) happens after the copy and before +any deletion, which is the same safety bar the spec's `rsync +--checksum-after` approach targets. + +## Modes + +### `plan` (default) — read-only, never touches disk + +``` +node bin/model-tidy.mjs plan [options] +``` + +Prints, for every discovered candidate: +- `[move]` — would be moved, with its size and unit reason +- `[skip]` — would NOT be moved, with the specific reason (see Selection + rules below) + +...then a one-line summary with real GiB sizes (bytes / 2^30) and a +"free before -> ~free after" estimate, and writes a JSON record to the log +dir (`--log-dir`, default `~/.cache/ide-agent-kit/model-tidy-logs/`). + +**`plan` makes zero filesystem writes, full stop** — including when it +finds a unit left mid-swap by an interrupted `apply`. It reports that as +an "interrupted move found" skip (see "Crash safety" below) and moves on; +resolving it is `recover`'s job, not `plan`'s. + +### `recover` — explicit, mutating, journal-validated + +``` +node bin/model-tidy.mjs recover [--home ] [--json] +``` + +Heals units left mid-swap by an interrupted `apply` (a crash, a kill, a +power loss). Acts **only** on units that have a matching journal record +whose manifest still matches what's on disk; a directory merely *named* +like a leftover, with no journal record, is reported and left alone. Safe +to run any time, including against a fully healthy tree (a no-op). See +"Crash safety" below for exactly what it checks and does. + +### `apply` — only with `--apply --target ` + +``` +node bin/model-tidy.mjs apply --apply --target /mnt/model-archive [options] +``` + +Refuses immediately, before touching anything, unless `--target`: +- exists and is a directory +- is writable +- is on a **different filesystem device** than `--home` (checked via + `stat().dev`, not by path string — a bind mount of the same device would + still be refused) + +`plan` is **read-only** — it never mutates anything, including when it +finds a unit left mid-swap by a previous interrupted `apply`. It only +*detects* that state (via the journal — see "Crash safety" below) and +*reports* it per unit as skipped, with reason `"interrupted move found: +..."`; it never touches the filesystem to fix it. Only two things ever run +recovery: the explicit `model-tidy recover` subcommand, and `apply` itself +(once, at its own start — reached only because `--apply` was actually +given). + +For each selected unit (a single directory, or a whole hardlink set moved +together), `apply`: +0. Refuses the whole run if a unit's journal record shows it's already + mid-swap in a way this run's own recovery pass (see below) couldn't + validate and resolve — that means an ambiguous leftover, and this run + will not guess which state is authoritative. +1. Copies every file into `--target`, preserving the relative-to-home path, + symlinks, and hardlink relationships. +2. Verifies every file: same relative paths, same symlink targets, same + byte size AND SHA-256 for every regular file. +3. Only if verification passed for every file in the unit, swaps the + source for a symlink via the journaled, crash-safe sequence below. + +### Crash safety + +A plain "delete, then create the symlink" has a window where a **hard** +crash — SIGKILL, an OOM kill, a power loss; anything a JS `try/catch` +cannot intercept, unlike a thrown exception — leaves neither the original +nor a symlink at the source path. Every path that referenced the model +breaks, and nothing in the same process ever gets a chance to roll back. + +**Before touching a unit at all**, `apply` writes a journal record — JSON, +one file per unit (named by a hash of the source path), under +`/.cache/ide-agent-kit/model-tidy-journal/`, written with an +explicit `fsync` so it survives a crash immediately after the write +returns. The record holds the source/staged/temp-link/target paths, the +already-computed per-file size+SHA-256 manifest, and the step reached. It +is rewritten (and re-fsynced) after every subsequent step: + +| Step written | What happens next | +| --- | --- | +| `pending` | build `.tidy-link -> target-copy`; verify it resolves and matches the manifest. **The source itself is not touched by anything up to and including this step** — a crash here leaves the source exactly as it was. | +| `linked` | `renameSync(source, source + '.tidy-moving')` — atomic. | +| `staged` | `renameSync(source + '.tidy-link', source)` — atomic, run immediately after the previous rename. | +| `swapped` | re-verify the symlink at `source` resolves to the target copy, then remove `source + '.tidy-moving'` and the journal record. | + +**The safety claim, exactly:** the original data is preserved and +recoverable at every step; the original path is unavailable for the +instant between the two renames, and until recovery runs if a crash lands +there. It is not continuously available — that instant is real and the +tests below cover it, not just the steps either side of it. + +Interrupted moves are **not** recovered automatically by `plan` or by the +passage of time. They are recovered by `model-tidy recover` (or by the +next `apply`, which runs the same recovery at its own start). Recovery is +**journal-driven, not naming-driven**: it reads every journal record, +checks that whichever of {source, staged copy} currently exists on disk +still matches that record's manifest, and only then acts: + +| Journaled state found | Recovery action | +| --- | --- | +| staged exists, source missing, temp link exists | complete the pending rename (the link was already verified before it was ever created), then run the guarded delete decision below before removing anything | +| staged exists, source missing, no temp link | rename staged back to source, remove the journal record — no verified pending swap existed to trust instead | +| staged exists, source is a symlink | the swap itself already completed; run the guarded delete decision below before removing the staged copy | +| temp link exists, source is a real directory, no staged | crashed before source was ever touched; remove the stray link and the journal record | +| source is a symlink, no staged | the move had already fully completed; just remove the stale journal record | +| source is a real directory, nothing else exists | never touched at all; remove the stale journal record | +| manifest mismatch against whatever currently exists (outside the guarded delete decision) | **left alone**, reported, regardless of step | + +### The guarded delete decision (`finalizeSwapOrRestore`) + +This recovery path has had two different real data-loss bugs found by +review, both from collapsing "is the live path OK?" and "is the backup OK?" +into a single pass/fail: + +- **Round 2:** the code verified the staged copy's manifest and that *a* + realpath existed for the pending symlink, but never re-verified the + **target's actual content**, nor that the symlink's realpath was + **exactly** the recorded target path. A target that changed after the + link was created (corruption, a second process, disk issues) still made + the old code delete the only good copy — "a link resolves to something" + was being treated as proof it resolves to something *correct*. +- **Round 3:** fixing that by requiring the staged copy to *also* match + its manifest before deleting created a NEW bug: if the live path (target + + symlink) was completely correct but the staged *backup* happened to be + partially damaged, the single pass/fail collapsed straight to "restore", + which **unlinked the good, working symlink and replaced it with the + known-damaged backup** — trading a verified-good live path for + known-bad data. + +There is now exactly one function allowed to delete OR rename away a +staged original — `finalizeSwapOrRestore`, used by **both** `apply`'s own +final step (same run) and `recoverInterruptedMoves` (a later run) — and it +is an explicit 4-row decision table, not a single boolean, over three +independently-checked facts: + +- **targetOk** — every file under the journal's target path matches the + manifest by size + SHA-256, no extra or missing paths (re-checked now, + never trusted from when the link was created). +- **linkOk** — the live path (already placed at the source, or still + pending, unconsumed, at the temp-link path) is a symlink whose realpath + resolves to exactly the recorded target path. +- **stagedState** — `absent` (nothing to protect), `matching` (exists and + matches the manifest exactly), or `damaged` (exists but doesn't match). + +| # | targetOk && linkOk | stagedState | Action | +| --- | --- | --- | --- | +| 1 | true | absent or matching | **Complete.** Finish the pending rename if one was still outstanding, delete the staged copy if it matched, clear the journal. | +| 2 | true | damaged | **Preserve + quarantine.** The live path is correct and complete — it is left completely untouched (finishing the pending rename if needed). The damaged backup is never deleted or silently restored over the good live path: it's renamed to `.tidy-quarantine-`, and that path is recorded in the journal (`step: 'completed-partial-staging-quarantined'`). Nothing is deleted in this row. | +| 3 | false | matching | **Restore.** Remove a bad/half symlink wherever it currently sits (source or the pending temp-link), rename the staged original back to the source path, re-verify it matches the manifest, journal `step: 'failed'`. | +| 4 | false | absent or damaged | **Touch nothing.** Nothing verified good exists anywhere for this unit. DELETE NOTHING, RENAME NOTHING — not even a still-pending, uncompleted rename — leave every path exactly as found. Journal `step: 'failed-both-copies-damaged'` with the per-check reasons, and report every path involved so a human can recover by hand. | + +Row 1 only ever completes a pending rename as part of deciding the WHOLE +unit is fine — it's never done speculatively "just in case" before the +decision, which was exactly how the round-3 bug reached its bad state via +a supposedly-harmless prep step. Row 4 exists specifically so that, when +there is genuinely nothing good anywhere, this function refuses to guess +— it does not pick a side. + +A `*.tidy-moving`, `*.tidy-link`, or `*.tidy-quarantine-*` path with **no +matching journal record at all** is reported but never touched, no matter +how it's named — recovery does not treat a name as ownership. A failure +for one unit does not roll back units that already succeeded earlier in +the same `apply` run, but the process still exits non-zero and prints +exactly which unit failed, at which step, and why. + +### Never recursive-delete a path assumed to be a symlink (`unlinkOwnedSymlink`) + +A fourth review round found that several places removed a path they +*believed* was a symlink using `rmSync(path, {recursive: true})`. If that +belief were ever wrong — a real directory happens to sit at that path, or +the symlink itself points somewhere other than expected — a recursive +remove can destroy an entire directory tree that was never model-tidy's to +touch. + +`unlinkOwnedSymlink(path, expectedTargetPath)` is now the only way +anything in this file removes a path it believes is one of its own +symlinks — including `swapToSymlink`'s own pre-swap cleanup when the +freshly-created temp link fails its own verification (round 6 closed that +fourth site the same way). It never recurses: + +1. `lstatSync(path)` — if the path doesn't exist, or isn't a symlink at + all, **refuse**. Touch nothing. +2. its realpath must resolve to exactly `expectedTargetPath`'s own + realpath — a symlink pointing somewhere else is **refused**, not + unlinked. +3. only then: `unlinkSync(path)` — never `rmSync`, never recursive. A + plain unlink can only ever remove the one symlink entry itself. + +Every caller treats a refusal as "left alone, reported, journal marked +`failed`" — never a reason to fall back to something more aggressive. + +### Journal durability + +A naive `open(file, 'w')` + write + `fsync` has its own crash window: a +kill landing between the truncating open and the write leaves an EMPTY or +PARTIAL record at the path readers expect — making a real, in-flight unit +look like there's nothing to recover, or worse, giving a reader a document +that parses as JSON but describes something incoherent. + +Every journal write follows write-temp-fsync-rename: the full record is +written to a throwaway `.tmp--` in the same directory, +that temp file is `fsync`ed and closed, then `renameSync`d atomically over +the real path (same-directory renames are atomic — a reader sees either +the old complete record or the new complete one, never a partial one), +then the journal directory itself is `fsync`ed. + +**No step is allowed to fail silently.** On the supported target (Linux), +ANY error from the temp-file fsync, the rename, or the directory fsync +aborts the write — and the caller (`swapToSymlink`) does not catch that, +so it aborts the whole unit before whatever mutation the write was meant +to guard, and reports the error. On a non-Linux dev platform, the same is +true with exactly one exception: `ENOTSUP` or `EINVAL` specifically from +the *directory* fsync (common on filesystems/platforms that don't support +fsync-ing a directory fd at all) is tolerated — the write still succeeds, +but a `durability degraded: ` note is attached to the result rather +than silently claiming durability that wasn't achieved. `EIO` and +everything else still abort there too. The fsync/rename functions and the +platform are all test-injectable (`--journal-fsync`-style options, not +exposed on the CLI — internal to `applyRun`/`recoverInterruptedMoves`), +so this rule is tested without needing two different operating systems. + +On the read side, a journal record that is **missing, empty, truncated, +fails to parse, or doesn't match the expected schema** is never treated as +"nothing to see here" — it's reported as `journal-unreadable` +(`plan`/`detectInterruptedMoves`) or `left-alone` +(`recover`/`recoverInterruptedMoves`), and the unit it might describe is +never acted on, never deleted, by any of `plan`, `apply`, or `recover`. A +stray `.json.tmp-*` file — the leftover of a write that itself got +interrupted — is recognized by name and reported the same way; it is never +parsed as a record. + +**Recovering ownership from the filename.** An unreadable record's +*content* can't say which candidate it was protecting — but its *name* +still can: `journalFilePath` always names a record +`.json`, and that 64-hex-char key survives even +when the content doesn't. `plan` builds a `journalKey(path) -> candidate` +map from every currently discovered candidate and matches every unreadable +record's filename against it: +- a match refuses exactly that candidate, with reason `interrupted move + with unreadable journal record: ` — no other candidate is + affected; +- **no match at all** (an unparseable filename, or a key that doesn't + correspond to any currently discovered candidate) means the unreadable + record *might* describe something plan can't even see right now — so + `plan` refuses to select **anything** this run, reporting exactly why + (`globalJournalBlockReason`, and `REFUSING TO SELECT ANYTHING THIS RUN` + in the summary line) rather than guess. `apply` enforces the same rule + even more strictly, and — after a review round found the ordering itself + was a bug — checks it **first**: before its own automatic recovery pass, + before validating `--target`, before copying, before anything. If any + unreadable or stray journal file exists, `apply` refuses the *entire* + run — `unresolved journal state: ; run recover, or resolve by + hand` — without running recovery at all, so it never mutates so much as + one otherwise-valid interrupted unit while any journal file's content + can't be trusted. **`apply` refuses globally on any unreadable journal; + `recover` heals what it can and reports the rest** — the explicit + `recover` subcommand still recovers every valid journaled unit it finds + in the same run, reporting (never touching) any unreadable one + alongside them. + +## Selection rules (in order, each with an explicit reason string) + +0. **Interrupted move found** — highest priority, checked before anything + else. If the journal (see "Crash safety" above) shows this unit is + mid-swap from a previous interrupted `apply`, it is skipped with + `interrupted move found:
` and never selected, regardless of + any other rule. `plan` only reports this; run `recover` to resolve it. +0.5. **Hardlink completeness** — also checked before KEEP/process/docker/ + etc. If any file's `st_nlink` exceeds the number of links this scan + actually found under every discovered candidate root, the candidate is + skipped with `hardlinked N times, only M links found under scanned + roots; refusing incomplete set`. A hardlink partner can sit entirely + outside every discovered root (e.g. a manual backup copy directly under + `$HOME`, above `~/models`) — no amount of scanning wider closes this in + general, only comparing against `st_nlink` does. +1. **KEEP list** — anything matching `--keep-file` (path or glob, `~` + expanded, `#` comments) is always skipped. See + `config/model-tidy.keep.example`. +2. **In use by a process, or unverifiable** — skipped if: + - any process has one of the model's files open under `/proc/*/fd`, or + - a recognized serving process's command line references the path + (`vllm`, `llama-server`, `llama.cpp`, `sglang`, `exllama`, `tabby`, + `ollama`, `mlx`, `text-generation`). + **Fail-closed:** this check has to succeed for *every* pid on the box to + count as verified. Off Linux (no `/proc`), if `/proc` itself can't be + listed, or if even one pid's `fd` directory, an individual fd's + `readlink`/`realpath`, or `cmdline` can't be read, the candidate is + skipped with `in-use status unverified: ` rather than + treated as idle for lack of evidence. The one exception, at every one of + those read points: `ENOENT` specifically (the fd or process vanished + between being listed and being read) is a normal race, not a failure, + and is ignored — everything else (`EACCES`, `EPERM`, anything else) is + treated as "could not verify" and fails that pid's check closed. In + practice, on a typical non-root Linux host with other users' or root's + processes running, this makes the tool quite conservative unless it + runs with enough privilege to read every pid's `/proc` entry — that is + intentional: an unreadable process is exactly the case where we cannot + prove a model is idle. +3. **Bind-mounted into (or containing) the bind-mount source of a running + docker container, or unverifiable** — reads `docker inspect` of every + running container's `Mounts`; a candidate is in use if it is at or under + a mount source, OR if a mount source is at or under the candidate (a + container mounting a subdirectory of a larger candidate still makes + that whole candidate unsafe to move). **Fail-closed:** if `docker` is + not installed, not running, or its `ps`/`inspect` output can't be read, + EVERY candidate on the box is skipped with `in-use status unverified: + docker not readable: ` — not just `~/.cache/huggingface`. A + container can bind-mount anything; only being able to enumerate every + running container's mounts makes any candidate provably idle. +4. **Too recent** — skipped if the newest mtime of any real file in the + directory is within `--min-idle-days` (default 14). The CLI rejects a + non-finite or negative `--min-idle-days` up front (e.g. a typo like + `--min-idle-days fourteen`) rather than let a stray `NaN` silently + disable this check. +5. **Already tidied** — skipped if the candidate is already a symlink. +6. **Hardlink sets move as one unit** — candidates are grouped by shared + `dev:ino`, scanning every discovered model root regardless of its own + KEEP/in-use/recent status (so a hardlink to a KEEP-listed copy, or to a + directory that's in use, is still detected). A group is selected only if + *every* member independently passed rules 1-5; otherwise every member is + skipped with `hardlinked to , which is not moving ('s own + reason)` (moving one copy of a hardlinked pair to another filesystem + breaks the hardlink and doubles disk use — this is the whole point of + the rule). + +Remaining candidates are sorted by size (desc) and capped by `--max-gb` +(whole units only — a unit is either fully included in this run's budget or +fully deferred to the next run). + +## Candidate discovery + +- `~/.cache/huggingface/hub/models--*` — the whole `models--X` directory is + one unit (its `snapshots//*` symlinks point into `blobs/`; moving a + snapshot without its blobs breaks it). +- `~/models/*` — each top-level entry is one unit. +- Ad-hoc project directories: a heuristic. A top-level entry under `--home` + is a candidate if it (or an immediate subdirectory, e.g. `.../model`) + contains a model marker: `config.json`, `tokenizer*.json`, or a file + ending in `.safetensors`, `.gguf`, `.bin`, `.pt`, `.pth`, `.exl2`, + `.exl3`, `.awq`, or `.gptq`. This is NOT exhaustive — see "Not verified". + +## Reporting + +- `plan` and `apply` both print a one-line summary suitable for pasting + into the room, and write a JSON record per run to `--log-dir` + (default `~/.cache/ide-agent-kit/model-tidy-logs/.jsonl`). +- `--report-to-room --room [--config ]` posts that summary + line via the same `groupmind.one` POST the rest of this repo uses + (`src/room-automation.mjs`'s `postMessage`). **Off by default.** Needs an + API key: `IAK_API_KEY` env var, or `poller.api_key` from `--config`'s + JSON. + +## Scheduling — systemd user timer (Linux, nightly, plan-only by default) + +Units are in `systemd/`. They are **not installed by this PR** — the owner +installs them explicitly on each box: + +``` +mkdir -p ~/.config/systemd/user && cp systemd/model-tidy.service systemd/model-tidy.timer ~/.config/systemd/user/ && systemctl --user daemon-reload && systemctl --user enable --now model-tidy.timer +``` + +The shipped `model-tidy.service` runs `plan` only — it never moves anything +until the `ExecStart` line is edited to add `apply --apply --target ...`, +which the unit's own comments call out explicitly. Do this only after +reviewing several plan runs' logs. + +Check it fired: `systemctl --user list-timers model-tidy.timer` and +`journalctl --user -u model-tidy.service`. + +## `--dry-run-remote ` + +``` +node bin/model-tidy.mjs --dry-run-remote asus1 [--remote-home /home/petrus] +``` + +SSHes to ``, checks for `node` there, and if found, `scp`s +`bin/model-tidy.mjs` + `src/model-tidy.mjs` into a throwaway +`/tmp/.model-tidy-dryrun-*` directory, runs `plan` there (read-only, no +`--apply`), prints the output, then `ssh`es back to `rm -rf` **only that +throwaway directory it just created** (never model data). If `node` is not +found on the remote, it refuses and tells you to install Node >= 18 first — +it never attempts to install anything itself. + +## Safety invariants + +- **Nothing removes a path it believes is a symlink without verifying it + first, and never recursively.** `unlinkOwnedSymlink` is the only code + path allowed to do this: it refuses (touches nothing) if the path isn't + actually a symlink, or resolves to somewhere other than the journal's + recorded target — a plain `unlinkSync`, never `rmSync`, only after both + checks pass. +- **Every journal write is crash-safe: write-temp-fsync-rename, then fsync + the directory — and any failure in that sequence aborts the write rather + than silently claiming durability.** On Linux, no error is tolerated + anywhere in that sequence. Off Linux, only `ENOTSUP`/`EINVAL` from the + directory fsync specifically is tolerated, and only with an explicit + `durability degraded: ` note in the result — never silently. +- A journal record that's missing, empty, truncated, unparseable, or the + wrong shape is treated as `journal-unreadable` by every reader (`plan`, + `apply`, `recover`) — that unit is reported and left alone, never acted + on. Its filename (not its content) is still used to identify which + candidate it protects: a match refuses that one candidate; no match at + all (unparseable name, or a key with no current candidate) makes `plan` + refuse to select anything and makes `apply` refuse to touch anything, + rather than risk missing what an unreadable record was protecting. +- `plan` makes zero filesystem writes, ever — including for interrupted + moves, which it only detects and reports. Only `recover` and `apply` + (once, at its own start) mutate anything. +- `apply` requires both `--apply` AND `--target ` — neither alone is + enough. +- `--target` must be an absolute, existing, writable directory on a + different filesystem device than `--home`, or apply refuses before + touching anything. +- A source directory's original copy is only ever removed after (a) its + copy has been verified byte-for-byte (size + SHA-256) at the target, and + (b) the replacement symlink at the source has itself been created and + verified to resolve to that copy. **Exact claim: the original data is + preserved and recoverable at every step; the original path is + unavailable for the instant between the two renames, and until recovery + runs if a crash lands there. It is not continuously available.** +- Recovery is journal-driven, not naming-driven: it only acts on a unit + whose on-disk state (source and/or staged copy, whichever exist) matches + the manifest recorded in that unit's own journal file. A directory named + like a leftover with no matching journal record — or a journaled unit + whose content has since changed — is reported and left alone, never + guessed at. +- **A staged original is only ever deleted or renamed away by + `finalizeSwapOrRestore`, the one function both `apply`'s own final step + and `recoverInterruptedMoves` call for that decision** — never on the + strength of "a symlink resolves to something." It follows an explicit + 4-row decision table (see "The guarded delete decision" above) over + three independently re-checked facts — target content, symlink + correctness, staged-copy state (absent/matching/damaged) — so that "the + live path is fine" and "the backup is fine" are never collapsed into one + answer: a live path that's correct and complete is always preserved + exactly as-is, even when its backup turns out to be damaged (the backup + is quarantined, not deleted, and never used to overwrite a working + symlink); and when nothing anywhere is verified good, nothing is deleted + or renamed at all. +- A hardlink set moves as a unit or not at all — never partially — and the + hardlink scan covers every discovered model root, not just the ones that + already passed every other rule. It also refuses a candidate whose + `st_nlink` exceeds the number of links actually found under the scanned + roots — a hardlink partner can sit entirely outside every discovered + candidate, where no amount of scanning wider would find it. +- Process- and docker-in-use checks are fail-closed: anything that could + not be fully verified is treated as in-use, never as idle. +- KEEP-listed paths are never touched by any mode. +- `--report-to-room` and any nightly `apply` are both opt-in and off by + default. + +## Not verified + +- **Never run against a real machine except `--dry-run-remote` in plan + mode.** No `apply` run has ever executed outside the test fixtures in + this PR. +- `asus1` / `asus2` have no `node` binary installed (checked via SSH during + this PR's review, read-only: `ssh asus1 command -v node` / + `ssh asus2 command -v node`, both empty). `--dry-run-remote` against both + hosts therefore stops at that check and prints an install instruction — + it does not, and did not, run the actual plan logic on either box. Real + candidate discovery, HF-cache sizing, docker-mount detection, and + process-scanning on asus1/asus2 are unverified until Node is installed + there (the owner's call, not done as part of this change). +- The ad-hoc project-directory heuristic (marker files at the top level or + one level deep) is written to match the two examples in the brief + (`~/DeepSeek-v4.1-Flash-EXL3-2x-DGX-Sparks/`, + `~/GLM-5.3-Flash-EXL3-2x-DGX-Sparks/model`) but has not been run against + the real directory layout on either box. +- The `/proc/*/fd` and cmdline process-in-use check only runs meaningfully + on Linux; it degrades to `checked:false` (not "confirmed idle") off + Linux, but that degraded path itself is only exercised by this dev + machine being macOS, not by a real Linux box lacking `/proc` access. +- `docker inspect`'s exact `Mounts[].Source` string format was assumed from + documented docker behavior, not confirmed against the actual + vLLM/docker-compose setup on asus1/asus2. +- No real hardlinked pair between `~/models/*` and the HF cache has been + inspected on either box — the fixture in `test/model-tidy.test.mjs` + constructs a synthetic one via `fs.linkSync`. +- **Practical consequence of the fail-closed process check, not verified + on a real box:** on a typical multi-process Linux host, at least some + pids (root's, other users', kernel-adjacent processes) will be + unreadable to a non-root `model-tidy` process. As written, that makes + EVERY run report every non-KEEP/non-tidied candidate as unverified unless + `model-tidy` runs with enough privilege to read every pid's `/proc` + entry, or unless asus1/asus2 turn out to be single-user boxes where + `model-tidy`'s own user can read every relevant pid. This has not been + checked against the real process list on either box, so it's unknown + whether the tool would select anything at all there today. +- One finding from the automated Codex review is known and NOT addressed in + this pass (out of scope, tracked here instead of silently dropped): + there is no `lsof`-based fallback for the process-in-use check, only + `/proc`. (The other Codex finding from that review — `sha256File` + reading whole files via `readFileSync` rather than streaming — was fixed + in round 6: it now hashes in bounded 8 MiB chunks via `readSync`, used + uniformly by manifest computation, manifest verification, and copy + verification. Tested against a file larger than the chunk size for + correctness, and with `Buffer.allocUnsafe`/`Buffer.alloc` instrumented to + assert no single allocation exceeds the chunk size — but never against + an actual multi-GiB model file on real hardware, only synthetic + multi-chunk files on this dev machine.) +- The journal-based crash recovery (`recoverInterruptedMoves`, the + `recover` subcommand) has only ever been exercised against synthetic + fixtures with tiny files and a hard-killed child process on this dev + machine (macOS). It has never been exercised against a real interrupted + `apply` on asus1/asus2, against a multi-GiB real model, or against a + genuinely full disk (an `ENOSPC` mid-copy, mid-rename, or mid-journal- + write is untested). The journal directory + (`/.cache/ide-agent-kit/model-tidy-journal/`) itself is assumed to + be on the same filesystem as `--home` and writable; that assumption is + untested on either box. diff --git a/package.json b/package.json index de944bc..1110509 100644 --- a/package.json +++ b/package.json @@ -7,11 +7,13 @@ "ide-agent-kit": "./bin/cli.mjs", "ide-agent-kit-mcp": "./bin/iak-mcp.mjs", "iak-pending": "./bin/iak-pending.mjs", - "iak-scan-history": "./scripts/scan-history-for-secrets.mjs" + "iak-scan-history": "./scripts/scan-history-for-secrets.mjs", + "ide-agent-kit-model-tidy": "./bin/model-tidy.mjs" }, "scripts": { "test": "node --test test/*.test.mjs packages/user-intent-kit/test/*.test.js", "scan:history": "node scripts/scan-history-for-secrets.mjs", + "model-tidy": "node bin/model-tidy.mjs", "start": "node bin/cli.mjs serve", "mcp": "node bin/iak-mcp.mjs", "relay": "node scripts/local-relay.mjs", diff --git a/src/model-tidy.mjs b/src/model-tidy.mjs new file mode 100644 index 0000000..2f8fab5 --- /dev/null +++ b/src/model-tidy.mjs @@ -0,0 +1,1953 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +/** + * model-tidy — move idle local LLM model directories off a full internal + * drive onto a target mount (external SSD / NAS), leaving a symlink behind + * so every path that already points at the model keeps working. + * + * Two modes: + * plan (default) — read-only. Lists what WOULD move, and why every other + * candidate is skipped. Never touches disk. + * apply — only runs with --apply AND --target . Copies + * (verified byte-for-byte), then replaces the source + * directory with a symlink to the copy via a crash-safe + * rename/symlink/verify/cleanup sequence (swapToSymlink). + * A failure at any step leaves the unit EITHER with its + * original untouched (copy/verify failures, or a swap + * failure that rolled back) OR with a working symlink + * to a verified copy (a swap that completed) — never + * neither, and never a bare deletion with no symlink. + * + * Zero external dependencies (Node >= 18 only), matching the rest of + * ide-agent-kit. The copy step is a small hand-written recursive copy + * (not a shelled-out `rsync`) specifically so that hardlink relationships + * *within a moved unit* are recreated at the destination via fs.linkSync — + * a plain recursive copy (or `cp -a` without `-H` semantics) would silently + * double disk usage for a hardlinked model. See copyUnitPureNode(). + * + * Selection rules run in this order, each producing an explicit reason. + * Rules 2 and 3 are FAIL-CLOSED: if either check cannot be fully completed + * for a candidate, it is skipped as unverified rather than treated as idle + * on absence of evidence. + * 1. KEEP list match -> always skipped + * 2. open by a process, or referenced by a -> skipped, "in use", or + * known serving process's command line "in-use status unverified" + * if /proc can't be fully read for every pid if unverifiable + * 3. under (or containing) the bind-mount source -> skipped, "in use", or + * of a RUNNING docker container; if docker is "in-use status unverified" + * unreadable, EVERY candidate on the box is if docker is unreadable + * unverified, not just the HF cache + * 4. newest mtime within --min-idle-days -> skipped, "too recent" + * 5. already a symlink (previously tidied) -> skipped, "already tidied" + * 6. hardlink sets move as one unit: every member -> skipped, "hardlinked to + * must independently clear rules 1-5 or the , which is not moving" + * whole set is skipped + * + * Remaining candidates are sorted by size (desc) and capped by --max-gb. + */ + +import { spawnSync } from 'node:child_process'; +import { + readdirSync, lstatSync, existsSync, readFileSync, readlinkSync, + realpathSync, symlinkSync, rmSync, mkdirSync, copyFileSync, linkSync, + statSync, appendFileSync, constants as FS_CONSTANTS, accessSync, renameSync, + openSync, writeSync, fsyncSync, closeSync, unlinkSync, readSync +} from 'node:fs'; +import { join, relative, sep, isAbsolute, basename } from 'node:path'; +import { createHash, randomBytes } from 'node:crypto'; +import { homedir } from 'node:os'; + +export const SERVING_PROCESS_NAMES = [ + 'vllm', 'llama-server', 'llama.cpp', 'sglang', 'exllama', + 'tabby', 'ollama', 'mlx', 'text-generation' +]; + +const MODEL_MARKER_FILES = new Set(['config.json', 'tokenizer.json', 'tokenizer_config.json']); +const MODEL_MARKER_EXT = ['.safetensors', '.gguf', '.bin', '.pt', '.pth', '.exl2', '.exl3', '.awq', '.gptq']; +const AD_HOC_EXCLUDE = new Set([ + 'Desktop', 'Downloads', 'Documents', 'Pictures', 'Movies', 'Music', 'Public', + 'node_modules', 'go', 'snap', 'models', '.cache', '.local', '.config', + '.ssh', '.npm', '.cargo', '.rustup', '.docker', '.git' +]); + +// --------------------------------------------------------------------------- +// Filesystem walking primitives +// --------------------------------------------------------------------------- + +/** Recursively list regular files under root. Symlinks are NOT followed or + * counted here (their target is counted when the target itself is walked, + * e.g. HF cache blobs/ vs snapshots/ symlinks). Returns [] on any read error. */ +export function walkFiles(root) { + const results = []; + function recurse(dir) { + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const full = join(dir, entry.name); + let st; + try { + st = lstatSync(full); + } catch { + continue; + } + if (st.isSymbolicLink()) continue; + if (st.isDirectory()) { + recurse(full); + continue; + } + if (st.isFile()) { + results.push({ path: full, size: st.size, mtimeMs: st.mtimeMs, dev: st.dev, ino: st.ino, nlink: st.nlink }); + } + } + } + recurse(root); + return results; +} + +/** Sum of real (non-symlink) file bytes under a directory. */ +export function dirSizeBytes(path, filesCache) { + const files = filesCache || walkFiles(path); + return files.reduce((sum, f) => sum + f.size, 0); +} + +/** Newest mtime (ms since epoch) of any real file under a directory. Falls + * back to the directory's own mtime if it has no real files (e.g. an + * all-symlink snapshots/ dir). */ +export function newestMtimeMs(path, filesCache) { + const files = filesCache || walkFiles(path); + if (files.length === 0) { + try { + return lstatSync(path).mtimeMs; + } catch { + return 0; + } + } + return files.reduce((max, f) => Math.max(max, f.mtimeMs), 0); +} + +export function isSymlink(path) { + try { + return lstatSync(path).isSymbolicLink(); + } catch { + return false; + } +} + +function hasModelMarkers(dir) { + let entries; + try { + entries = readdirSync(dir); + } catch { + return false; + } + for (const name of entries) { + if (MODEL_MARKER_FILES.has(name)) return true; + if (MODEL_MARKER_EXT.some(ext => name.endsWith(ext))) return true; + } + return false; +} + +// --------------------------------------------------------------------------- +// Candidate discovery +// --------------------------------------------------------------------------- + +/** + * Discover model-directory candidates under a home directory: + * - ~/.cache/huggingface/hub/models--* (whole dir is the unit; moving a + * snapshot without its blobs breaks it) + * - ~/models/* (each top-level entry is a unit) + * - ad-hoc project dirs: a top-level home entry (or its immediate `model` + * / marker-bearing subdir) that itself contains model marker files + * (config.json, tokenizer*.json, or a *.safetensors/*.gguf/... file). + * This is a heuristic, not exhaustive — see docs/model-tidy.md. + */ +export function discoverCandidates(home) { + const candidates = []; + const seen = new Set(); + + function add(path, kind) { + if (seen.has(path)) return; + seen.add(path); + candidates.push({ id: path, path, kind }); + } + + const hfHub = join(home, '.cache', 'huggingface', 'hub'); + if (existsSync(hfHub)) { + for (const entry of safeReaddir(hfHub)) { + if (isJournalSuffixed(entry)) continue; + if (entry.startsWith('models--')) { + add(join(hfHub, entry), 'hf-cache'); + } + } + } + + const modelsDir = join(home, 'models'); + if (existsSync(modelsDir)) { + for (const entry of safeReaddir(modelsDir)) { + if (isJournalSuffixed(entry)) continue; + add(join(modelsDir, entry), 'models-dir'); + } + } + + for (const entry of safeReaddir(home)) { + if (entry.startsWith('.')) continue; + if (isJournalSuffixed(entry)) continue; + if (AD_HOC_EXCLUDE.has(entry)) continue; + const full = join(home, entry); + let st; + try { + st = lstatSync(full); + } catch { + continue; + } + if (!st.isDirectory()) continue; + if (hasModelMarkers(full)) { + add(full, 'ad-hoc'); + continue; + } + // one level deep: e.g. ~/GLM-5.3-Flash-EXL3-2x-DGX-Sparks/model + for (const child of safeReaddir(full)) { + if (isJournalSuffixed(child)) continue; + const childFull = join(full, child); + let cst; + try { + cst = lstatSync(childFull); + } catch { + continue; + } + if (cst.isDirectory() && hasModelMarkers(childFull)) { + add(childFull, 'ad-hoc'); + } + } + } + + return candidates; +} + +function safeReaddir(dir) { + try { + return readdirSync(dir); + } catch { + return []; + } +} + +// --------------------------------------------------------------------------- +// KEEP list +// --------------------------------------------------------------------------- + +/** Load a KEEP list file: one path or glob per line, '#' comments, blank + * lines ignored, leading '~' expanded to home. Missing file -> []. */ +export function loadKeepList(keepFilePath, home = homedir()) { + if (!keepFilePath || !existsSync(keepFilePath)) return []; + const lines = readFileSync(keepFilePath, 'utf8').split('\n'); + const out = []; + for (const raw of lines) { + const line = raw.trim(); + if (!line || line.startsWith('#')) continue; + out.push(line.startsWith('~') ? join(home, line.slice(1).replace(/^\//, '')) : line); + } + return out; +} + +function globToRegExp(glob) { + const escaped = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*').replace(/\?/g, '.'); + return new RegExp(`^${escaped}$`); +} + +/** True if `path` is on the KEEP list — exact match, ancestor match (a + * KEEP-listed directory contains this candidate), or glob match. */ +export function matchesKeepList(path, keepEntries) { + for (const entry of keepEntries) { + if (entry.includes('*') || entry.includes('?')) { + if (globToRegExp(entry).test(path)) return true; + continue; + } + if (path === entry || path.startsWith(entry + sep) || entry.startsWith(path + sep)) return true; + } + return false; +} + +// --------------------------------------------------------------------------- +// Process-in-use detection (Linux /proc) +// --------------------------------------------------------------------------- + +/** + * Detect processes using a path: (a) ANY process with the path open under + * /proc/*\/fd, (b) a recognized serving process (vllm, ollama, ...) whose + * command line references the path. + * + * Returns `{checked:false}` whenever the check could NOT be completed for + * every process on the box — off Linux (no /proc), if /proc itself can't be + * listed, or if even a single pid's fd directory or cmdline is unreadable + * (a permission failure, not the process having simply exited mid-scan, + * which is expected and not a failure). A permission failure on ONE pid + * means we cannot rule out THAT pid holding any of our candidates open, so + * it taints the whole result rather than being silently skipped — callers + * MUST treat `checked:false` as "could not verify", never as "confirmed + * idle". + */ +export function findProcessUsers(path, opts = {}) { + const procRoot = opts.procRoot || '/proc'; + const doRealpath = opts.realpathSync || realpathSync; // test-only injection point for the exact fd-resolution boundary + if (!existsSync(procRoot)) { + return { checked: false, users: [], note: `${procRoot} not available (not Linux); process-in-use check skipped` }; + } + let pids; + try { + pids = readdirSync(procRoot).filter(n => /^[0-9]+$/.test(n)); + } catch (e) { + return { checked: false, users: [], note: `cannot list ${procRoot}: ${e.message}` }; + } + const users = []; + const unreadable = []; // [{pid, code}] — every pid whose check could not be completed + for (const pid of pids) { + let pidErrorCode = null; // first non-ENOENT error code seen for this pid, if any + + try { + const fdDir = join(procRoot, pid, 'fd'); + let fds; + try { + fds = readdirSync(fdDir); + } catch (e) { + // ENOENT here means the process exited between the pid listing and + // this read — a benign race, not a verification failure. Anything + // else (EACCES/EPERM, or an unexpected error) means we genuinely + // could not check this pid's open files, so this pid's check + // fails closed. + if (e.code !== 'ENOENT') pidErrorCode = e.code || 'UNKNOWN'; + fds = []; + } + for (const fd of fds) { + try { + const target = doRealpath(join(fdDir, fd)); + if (target === path || target.startsWith(path + sep)) { + users.push({ pid, via: 'fd', target }); + break; + } + } catch (e) { + // ENOENT: this one fd vanished mid-scan (closed, or the whole + // process exited) — a benign race, not a verification failure. + // EACCES/EPERM/anything else: we could not resolve this fd, so + // we cannot rule it out — fail this pid's check closed. + if (e.code !== 'ENOENT') pidErrorCode = pidErrorCode || e.code || 'UNKNOWN'; + } + } + } catch (e) { + pidErrorCode = pidErrorCode || e.code || 'UNKNOWN'; + } + + try { + const cmdline = readFileSync(join(procRoot, pid, 'cmdline'), 'utf8').replace(/\0/g, ' ').trim(); + if (cmdline && cmdline.includes(path)) { + const lower = cmdline.toLowerCase(); + const servingProcess = SERVING_PROCESS_NAMES.some(n => lower.includes(n)); + users.push({ pid, via: 'cmdline', cmdline: cmdline.slice(0, 200), servingProcess }); + } + } catch (e) { + if (e.code !== 'ENOENT') pidErrorCode = pidErrorCode || e.code || 'UNKNOWN'; + } + + if (pidErrorCode) unreadable.push({ pid, code: pidErrorCode }); + } + + if (unreadable.length > 0) { + const first = unreadable[0]; + const more = unreadable.length > 1 ? ` (+${unreadable.length - 1} more pid(s) also unreadable)` : ''; + return { + checked: false, + users, + note: `${first.pid} ${first.code}${more} — cannot rule out ${unreadable.length === 1 ? 'that process' : 'those processes'} using this path` + }; + } + return { checked: true, users }; +} + +// --------------------------------------------------------------------------- +// Docker bind-mount detection +// --------------------------------------------------------------------------- + +/** + * Is `path` under, or does it CONTAIN, the bind-mount source of a RUNNING + * docker container? Both directions matter: `path` under the mount source + * means the whole candidate is served; the mount source under `path` means + * moving `path` would carry an actively-mounted subdirectory away with it. + * `{available:false}` means docker itself could not be queried — callers + * must fail closed (see planRun: an unreadable docker means every + * candidate on the box is treated as unverified, not just this one). + */ +export function findDockerBindUsers(path, opts = {}) { + const dockerBin = opts.dockerBin || 'docker'; + const ps = spawnSync(dockerBin, ['ps', '-q'], { encoding: 'utf8', timeout: 10000 }); + if (ps.error || ps.status !== 0) { + return { available: false, inUse: false, reason: `docker ps failed: ${ps.error?.message || ps.stderr?.trim() || `exit ${ps.status}`}` }; + } + const ids = ps.stdout.split('\n').map(s => s.trim()).filter(Boolean); + if (ids.length === 0) return { available: true, inUse: false, containers: [] }; + const inspect = spawnSync(dockerBin, ['inspect', ...ids], { encoding: 'utf8', timeout: 15000, maxBuffer: 64 * 1024 * 1024 }); + if (inspect.error || inspect.status !== 0) { + return { available: false, inUse: false, reason: `docker inspect failed: ${inspect.error?.message || inspect.stderr?.trim() || `exit ${inspect.status}`}` }; + } + let parsed; + try { + parsed = JSON.parse(inspect.stdout); + } catch (e) { + return { available: false, inUse: false, reason: `docker inspect returned unparseable JSON: ${e.message}` }; + } + const hits = []; + for (const c of parsed) { + for (const m of c.Mounts || []) { + if (m.Source && (path === m.Source || path.startsWith(m.Source + sep) || m.Source.startsWith(path + sep))) { + hits.push({ container: (c.Name || '').replace(/^\//, '') || (c.Id || '').slice(0, 12), source: m.Source, destination: m.Destination }); + } + } + } + return { available: true, inUse: hits.length > 0, containers: hits }; +} + +// --------------------------------------------------------------------------- +// Hardlink grouping +// --------------------------------------------------------------------------- + +/** + * Group candidates that share an inode (device+inode) across DIFFERENT + * candidate roots into a single move unit. Returns a union-find lookup plus + * a dedup'd byte size per unit (each shared inode counted once). + */ +export function computeHardlinkGroups(candidates) { + const filesByCandidate = new Map(); + const inodeMap = new Map(); // "dev:ino" -> Set(candidateId) + const inodeObservedCount = new Map(); // "dev:ino" -> how many file entries we actually found + const inodeNlink = new Map(); // "dev:ino" -> st_nlink reported by the filesystem + + for (const c of candidates) { + const files = walkFiles(c.path); + filesByCandidate.set(c.id, files); + for (const f of files) { + const key = `${f.dev}:${f.ino}`; + if (!inodeMap.has(key)) inodeMap.set(key, new Set()); + inodeMap.get(key).add(c.id); + inodeObservedCount.set(key, (inodeObservedCount.get(key) || 0) + 1); + inodeNlink.set(key, f.nlink); + } + } + + // Scanning wider cannot fully close this gap: st_nlink tells us how many + // directory entries point at this inode SYSTEM-WIDE, including ones + // completely outside any discovered candidate root (e.g. a manual backup + // hardlink sitting directly under $HOME, above ~/models). If the number + // of entries we actually found while walking the discovered candidates + // is less than st_nlink, there is at least one more link we cannot see + // and therefore cannot move safely — moving what we CAN see would still + // break the invisible link and double disk usage. Every candidate that + // owns such a file is flagged incomplete here; planRun turns this into a + // skip reason with top priority, before any other rule. + const incompleteReasonByCandidate = new Map(); + for (const c of candidates) { + if (incompleteReasonByCandidate.has(c.id)) continue; + for (const f of filesByCandidate.get(c.id)) { + const key = `${f.dev}:${f.ino}`; + const observed = inodeObservedCount.get(key); + const nlink = inodeNlink.get(key); + if (nlink > observed) { + incompleteReasonByCandidate.set( + c.id, + `hardlinked ${nlink} times, only ${observed} links found under scanned roots; refusing incomplete set` + ); + break; + } + } + } + + const parent = new Map(candidates.map(c => [c.id, c.id])); + function find(x) { + while (parent.get(x) !== x) { + parent.set(x, parent.get(parent.get(x))); + x = parent.get(x); + } + return x; + } + function union(a, b) { + const ra = find(a); + const rb = find(b); + if (ra !== rb) parent.set(ra, rb); + } + for (const ids of inodeMap.values()) { + if (ids.size > 1) { + const arr = [...ids]; + for (let i = 1; i < arr.length; i++) union(arr[0], arr[i]); + } + } + + const groupMembers = new Map(); // groupId -> [candidateId] + for (const c of candidates) { + const g = find(c.id); + if (!groupMembers.has(g)) groupMembers.set(g, []); + groupMembers.get(g).push(c.id); + } + + const groupSizeBytes = new Map(); + for (const [g, ids] of groupMembers) { + const seenInode = new Set(); + let total = 0; + for (const id of ids) { + for (const f of filesByCandidate.get(id)) { + const key = `${f.dev}:${f.ino}`; + if (seenInode.has(key)) continue; + seenInode.add(key); + total += f.size; + } + } + groupSizeBytes.set(g, total); + } + + return { groupOf: id => find(id), groupMembers, groupSizeBytes, filesByCandidate, incompleteReasonByCandidate }; +} + +// --------------------------------------------------------------------------- +// Disk free space +// --------------------------------------------------------------------------- + +/** Free bytes on the filesystem containing `path`, via `df -Pk`. Returns + * null if `df` is unavailable or unparseable (caller should omit the + * before/after line rather than print a wrong number). */ +export function diskFreeBytes(path) { + const res = spawnSync('df', ['-Pk', path], { encoding: 'utf8', timeout: 10000 }); + if (res.error || res.status !== 0) return null; + const lines = res.stdout.trim().split('\n'); + if (lines.length < 2) return null; + const cols = lines[lines.length - 1].trim().split(/\s+/); + const availKb = Number(cols[3]); + if (!Number.isFinite(availKb)) return null; + return availKb * 1024; +} + +// --------------------------------------------------------------------------- +// Plan +// --------------------------------------------------------------------------- + +export function planRun(options = {}) { + const home = options.home || homedir(); + const minIdleDays = options.minIdleDays ?? 14; + const maxGb = options.maxGb ?? Infinity; + const keepEntries = options.keepEntries || loadKeepList(options.keepFile, home); + const listProcessUsers = options.listProcessUsers || findProcessUsers; + const listDockerBindUsers = options.listDockerBindUsers || findDockerBindUsers; + const getDiskFreeBytes = options.diskFreeBytes || diskFreeBytes; + const nowMs = options.now ? options.now.getTime() : Date.now(); + const journalDir = options.journalDir || defaultJournalDir(home); + + // plan is READ-ONLY: it only DETECTS an interrupted apply from a previous + // crash (via the journal) and reports it per unit below — it never + // mutates anything. Only the explicit `recover` subcommand, or `apply` + // itself, actually heals interrupted state. + const detectFn = options.detect || detectInterruptedMoves; + const interrupted = detectFn(home, { journalDir }); + const interruptedByPath = new Map(interrupted.filter(f => f.path).map(f => [f.path, f])); + + const rawCandidates = options.candidates || discoverCandidates(home); + + // An unreadable/stray journal record (missing, empty, truncated, + // unparseable, or schema-invalid content) has `path: null` above — its + // CONTENT can't tell us which candidate it was protecting. But its + // FILENAME still encodes journalKey(sourcePath) (journalFilePath always + // names a record that way), so recover ownership from the name: match + // every unreadable/stray record's key against every currently + // discovered candidate. A match refuses that specific candidate. Any + // unreadable record whose filename doesn't match a current candidate at + // all (an unparseable name, or a key with no current candidate) means + // we cannot rule out that it was protecting something — fail closed + // GLOBALLY rather than guess which unit, if any, needs protecting. + const candidateKeyToPath = new Map(); + for (const c of rawCandidates) candidateKeyToPath.set(journalKey(c.path), c.path); + + const unresolvedJournalFiles = []; + for (const finding of interrupted) { + if (finding.path || !finding.journalFile) continue; // already attributable, or not a journal-file-shaped finding at all + const key = extractJournalKeyFromFilename(finding.journalFile); + const matchedPath = key ? candidateKeyToPath.get(key) : undefined; + if (matchedPath) { + interruptedByPath.set(matchedPath, { + path: matchedPath, + journalFile: finding.journalFile, + status: 'interrupted', + note: `interrupted move with unreadable journal record: ${finding.journalFile}`, + exactReason: `interrupted move with unreadable journal record: ${finding.journalFile}` + }); + } else { + unresolvedJournalFiles.push(finding.journalFile); + } + } + const globalJournalBlockReason = unresolvedJournalFiles.length > 0 + ? `unresolved journal state: ${unresolvedJournalFiles.join(', ')}; run recover, or resolve by hand` + : null; + + const { groupOf, groupMembers, groupSizeBytes, filesByCandidate, incompleteReasonByCandidate } = computeHardlinkGroups(rawCandidates); + + const perCandidate = new Map(); + for (const c of rawCandidates) { + let skipReason = null; + + // Rule 0, highest priority: a hardlink whose partner is invisible to + // this scan (outside every discovered candidate root) is a filesystem + // fact, not a policy choice — check it before KEEP/process/docker/etc. + if (incompleteReasonByCandidate.has(c.id)) { + skipReason = incompleteReasonByCandidate.get(c.id); + } else if (matchesKeepList(c.path, keepEntries)) { + skipReason = 'on KEEP list'; + } else if (isSymlink(c.path)) { + skipReason = 'already a symlink (tidied)'; + } else { + // Rules 2-3 (process / docker in-use) are fail-closed: if either + // check could not be fully completed, we cannot prove the candidate + // is idle, so it is skipped as unverified rather than allowed + // through on absence of evidence. An unreadable docker taints EVERY + // candidate on the box, not just the HF cache — a container could + // bind-mount anything. + const procResult = listProcessUsers(c.path); + if (procResult.checked === false) { + skipReason = `in-use status unverified: ${procResult.note || 'process check could not be completed'}`; + } else { + const fdHit = (procResult.users || []).find(u => u.via === 'fd'); + const cmdHit = (procResult.users || []).find(u => u.via === 'cmdline' && u.servingProcess); + const hit = fdHit || cmdHit; + if (hit) { + skipReason = hit.via === 'fd' + ? `in use: pid ${hit.pid} has a file open under this path` + : `in use: pid ${hit.pid} serving process references this path (${hit.cmdline})`; + } else { + const dockerResult = listDockerBindUsers(c.path); + if (dockerResult.available === false) { + skipReason = `in-use status unverified: docker not readable (${dockerResult.reason})`; + } else if (dockerResult.inUse) { + const first = dockerResult.containers[0]; + skipReason = `in use: bind-mounted into running container ${first.container} (${first.source} -> ${first.destination})`; + } + if (!skipReason) { + const files = filesByCandidate.get(c.id); + const newest = newestMtimeMs(c.path, files); + const ageDays = (nowMs - newest) / 86400000; + if (ageDays < minIdleDays) { + skipReason = `modified ${ageDays.toFixed(1)}d ago, newer than --min-idle-days ${minIdleDays}`; + } + } + } + } + } + + perCandidate.set(c.id, { + id: c.id, + path: c.path, + kind: c.kind, + sizeBytes: dirSizeBytes(c.path, filesByCandidate.get(c.id)), + skipReason + }); + } + + const results = []; + for (const [gid, memberIds] of groupMembers) { + const members = memberIds.map(id => perCandidate.get(id)); + const failing = members.find(m => m.skipReason); + const isHardlinkSet = memberIds.length > 1; + if (failing) { + for (const m of members) { + results.push({ + ...m, + groupId: gid, + groupSizeBytes: groupSizeBytes.get(gid), + selected: false, + reason: isHardlinkSet && m.id !== failing.id + ? `hardlinked to ${failing.path}, which is not moving (${failing.skipReason})` + : m.skipReason + }); + } + } else { + for (const m of members) { + results.push({ + ...m, + groupId: gid, + groupSizeBytes: groupSizeBytes.get(gid), + selected: true, + reason: isHardlinkSet ? 'idle (hardlink set, all members idle)' : 'idle' + }); + } + } + } + + // Cap by --max-gb: whole groups only, largest first. + const maxBytes = maxGb === Infinity ? Infinity : maxGb * 2 ** 30; + const idleGroupIds = [...new Set(results.filter(r => r.selected).map(r => r.groupId))] + .sort((a, b) => groupSizeBytes.get(b) - groupSizeBytes.get(a)); + let runningTotal = 0; + const cappedGroupIds = new Set(); + for (const gid of idleGroupIds) { + const size = groupSizeBytes.get(gid); + if (runningTotal + size <= maxBytes) { + runningTotal += size; + } else { + cappedGroupIds.add(gid); + } + } + for (const r of results) { + if (r.selected && cappedGroupIds.has(r.groupId)) { + r.selected = false; + r.reason = `deferred: --max-gb ${maxGb} cap reached for this run`; + } + } + + // An interrupted move takes absolute priority over every other rule: it + // is a filesystem fact discovered from the journal, not a policy choice, + // and this unit must never be selected until 'recover' resolves it. + // Override any existing result for this path, or synthesize one if the + // unit's real directory is currently missing (mid-swap) so discovery + // never saw it at all. + for (const finding of interruptedByPath.values()) { + const existingIdx = results.findIndex(r => r.path === finding.path); + // A record recovered by filename-matching (GAP 1) already carries the + // exact required reason text; a normal readable record still gets the + // generic wrapper. + const reason = finding.exactReason || `interrupted move found: ${finding.note}`; + if (existingIdx >= 0) { + results[existingIdx].selected = false; + results[existingIdx].reason = reason; + } else { + results.push({ + id: finding.path, + path: finding.path, + kind: 'interrupted', + sizeBytes: 0, + groupId: finding.path, + groupSizeBytes: 0, + selected: false, + reason + }); + } + } + + // GAP 1, global fail-closed: an unreadable/stray journal file whose + // filename could not be matched to any current candidate means we + // cannot rule out that it was protecting something — plan selects + // NOTHING this run rather than guess. Candidates that already have a + // more specific reason (KEEP, an unrelated in-use check, a matched + // interrupted record, ...) keep that reason; only would-be-selected + // candidates are overridden here. + if (globalJournalBlockReason) { + for (const r of results) { + if (r.selected) { + r.selected = false; + r.reason = globalJournalBlockReason; + } + } + } + + results.sort((a, b) => b.groupSizeBytes - a.groupSizeBytes || a.path.localeCompare(b.path)); + + const selected = results.filter(r => r.selected); + const skipped = results.filter(r => !r.selected); + const selectedGroupIds = new Set(selected.map(r => r.groupId)); + const totalSelectedBytes = [...selectedGroupIds].reduce((sum, gid) => sum + groupSizeBytes.get(gid), 0); + const unverifiedCount = skipped.filter(r => r.reason.includes('unverified')).length; + + const freeBeforeBytes = getDiskFreeBytes(home); + const freeAfterEstimateBytes = freeBeforeBytes == null ? null : freeBeforeBytes + totalSelectedBytes; + + const freeLine = freeBeforeBytes == null + ? 'free before/after: unknown (df unavailable)' + : `free before/after: ${(freeBeforeBytes / 2 ** 30).toFixed(1)} GiB -> ~${(freeAfterEstimateBytes / 2 ** 30).toFixed(1)} GiB`; + + const interruptedLine = interrupted.length > 0 + ? ` ${interrupted.length} interrupted-move finding(s) detected (never mutated by plan — run 'recover' to resolve).` + : ''; + const globalBlockLine = globalJournalBlockReason ? ` REFUSING TO SELECT ANYTHING THIS RUN: ${globalJournalBlockReason}` : ''; + const summaryLine = `model-tidy plan: ${selected.length} dir(s) in ${selectedGroupIds.size} unit(s), ` + + `${(totalSelectedBytes / 2 ** 30).toFixed(1)} GiB movable to target, ${skipped.length} skipped ` + + `(${unverifiedCount} unverified). ${freeLine}${interruptedLine}${globalBlockLine}`; + + return { + home, + minIdleDays, + maxGb, + interrupted, + globalJournalBlockReason, + selected, + skipped, + totalSelectedBytes, + unverifiedCount, + freeBeforeBytes, + freeAfterEstimateBytes, + freeLine, + summaryLine, + generatedAt: new Date(nowMs).toISOString() + }; +} + +// --------------------------------------------------------------------------- +// Apply +// --------------------------------------------------------------------------- + +export const HASH_CHUNK_BYTES = 8 * 1024 * 1024; // 8 MiB — bounded memory regardless of file size + +/** + * SHA-256 of a file's content with BOUNDED memory: reads in fixed-size + * chunks via readSync into one reused buffer, never the whole file at + * once via readFileSync. Model shards are routinely multi-GiB + * (.safetensors/.gguf); loading one whole into a Buffer to hash it could + * exhaust memory or exceed Node's Buffer size limit on some builds. Every + * hashing call site in this file (manifest computation, manifest + * verification, and rsync-style copy verification) goes through this one + * function, so fixing it here fixes all of them. + */ +export function sha256File(path) { + const hash = createHash('sha256'); + const buffer = Buffer.allocUnsafe(HASH_CHUNK_BYTES); + const fd = openSync(path, 'r'); + try { + let bytesRead; + do { + bytesRead = readSync(fd, buffer, 0, HASH_CHUNK_BYTES, null); + if (bytesRead > 0) { + hash.update(bytesRead === HASH_CHUNK_BYTES ? buffer : buffer.subarray(0, bytesRead)); + } + } while (bytesRead > 0); + } finally { + closeSync(fd); + } + return hash.digest('hex'); +} + +/** + * Pure-Node recursive copy of one or more source directories into + * `targetRoot`, preserving relative-to-`home` layout, preserving symlinks + * verbatim, and recreating hardlinks *within this call* (a shared + * `inodeToTargetPath` map across all `sourceAbsPaths`) so a hardlinked unit + * stays a hardlinked unit at the destination instead of doubling in size. + */ +export function copyUnitPureNode(sourceAbsPaths, home, targetRoot) { + const inodeToTargetPath = new Map(); + + function copyDir(srcDir, dstDir) { + mkdirSync(dstDir, { recursive: true }); + for (const entry of readdirSync(srcDir, { withFileTypes: true })) { + const s = join(srcDir, entry.name); + const d = join(dstDir, entry.name); + const lst = lstatSync(s); + if (lst.isSymbolicLink()) { + symlinkSync(readlinkSync(s), d); + } else if (lst.isDirectory()) { + copyDir(s, d); + } else if (lst.isFile()) { + const key = `${lst.dev}:${lst.ino}`; + if (inodeToTargetPath.has(key)) { + linkSync(inodeToTargetPath.get(key), d); + } else { + copyFileSync(s, d); + inodeToTargetPath.set(key, d); + } + } + } + } + + for (const src of sourceAbsPaths) { + const rel = relative(home, src); + const dst = join(targetRoot, rel); + copyDir(src, dst); + } +} + +/** Byte-for-byte verification: same set of relative paths, same symlink + * targets, same file sizes AND sha256 for every regular file. */ +export function verifyUnit(sourceAbsPaths, home, targetRoot) { + const mismatches = []; + for (const src of sourceAbsPaths) { + const rel = relative(home, src); + const dst = join(targetRoot, rel); + const srcFiles = listAllEntries(src); + for (const s of srcFiles) { + const relToUnit = relative(src, s); + const d = join(dst, relToUnit); + const lst = lstatSync(s); + if (!existsSync(d)) { + mismatches.push({ path: s, reason: 'missing at target', target: d }); + continue; + } + const dlst = lstatSync(d); + if (lst.isSymbolicLink()) { + if (!dlst.isSymbolicLink() || readlinkSync(s) !== readlinkSync(d)) { + mismatches.push({ path: s, reason: 'symlink target mismatch', target: d }); + } + } else if (lst.isFile()) { + if (!dlst.isFile()) { + mismatches.push({ path: s, reason: 'not a regular file at target', target: d }); + continue; + } + if (lst.size !== dlst.size) { + mismatches.push({ path: s, reason: `size mismatch (${lst.size} vs ${dlst.size})`, target: d }); + continue; + } + const srcHash = sha256File(s); + const dstHash = sha256File(d); + if (srcHash !== dstHash) { + mismatches.push({ path: s, reason: 'checksum mismatch', target: d }); + } + } + } + } + return { ok: mismatches.length === 0, mismatches }; +} + +function listAllEntries(root) { + const out = []; + function recurse(dir) { + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const full = join(dir, entry.name); + out.push(full); + let st; + try { + st = lstatSync(full); + } catch { + continue; + } + if (st.isDirectory() && !st.isSymbolicLink()) recurse(full); + } + } + recurse(root); + return out; +} + +/** Validate --target: must be an absolute path, must exist, be a directory, + * be writable, and be on a different filesystem device than `home`. */ +export function validateTarget(target, home) { + if (!target) return { ok: false, error: '--target is required with --apply' }; + if (!isAbsolute(target)) return { ok: false, error: `--target ${target} must be an absolute path (a relative target makes the resulting symlink text ambiguous)` }; + if (!existsSync(target)) return { ok: false, error: `--target ${target} does not exist` }; + const st = statSync(target); + if (!st.isDirectory()) return { ok: false, error: `--target ${target} is not a directory` }; + try { + accessSync(target, FS_CONSTANTS.W_OK); + } catch { + return { ok: false, error: `--target ${target} is not writable` }; + } + const homeSt = statSync(home); + if (st.dev === homeSt.dev) { + return { ok: false, error: `--target ${target} is on the same filesystem (dev ${st.dev}) as ${home}; refusing (must be a different mount)` }; + } + return { ok: true }; +} + +const STAGING_SUFFIX = '.tidy-moving'; +const LINK_SUFFIX = '.tidy-link'; +const JOURNAL_SUBDIR = 'model-tidy-journal'; + +function isJournalSuffixed(name) { + return name.endsWith(STAGING_SUFFIX) || name.endsWith(LINK_SUFFIX) || name.includes(QUARANTINE_SUFFIX_PREFIX); +} + +function defaultJournalDir(home) { + return join(home, '.cache', 'ide-agent-kit', JOURNAL_SUBDIR); +} + +function journalKey(sourcePath) { + return createHash('sha256').update(sourcePath).digest('hex'); +} + +function journalFilePath(journalDir, sourcePath) { + return join(journalDir, `${journalKey(sourcePath)}.json`); +} + +const JOURNAL_KEY_PATTERN = /^([0-9a-f]{64})\.json(?:\.tmp-.*)?$/; + +/** + * Recover ownership from a journal file's NAME alone — used when its + * CONTENT can't be trusted (missing, empty, truncated, unparseable, or + * schema-invalid). `journalFilePath` always names a record + * `.json` (or, for a stray in-flight write, + * `...json.tmp--`), so the 64-hex-char key survives even + * when the record's content doesn't. Returns null if the filename doesn't + * match that shape at all (a genuinely unrecognizable file). + */ +function extractJournalKeyFromFilename(file) { + const match = basename(file).match(JOURNAL_KEY_PATTERN); + return match ? match[1] : null; +} + +/** Minimal structural check for a parsed journal record. Anything that + * doesn't match — including a record that parsed as valid JSON but isn't + * actually one of ours (wrong shape) — is treated as unreadable, exactly + * like a truncated or corrupt file. */ +function isValidJournalRecordShape(record) { + return !!record + && typeof record === 'object' + && typeof record.sourcePath === 'string' && record.sourcePath.length > 0 + && typeof record.stagedPath === 'string' && record.stagedPath.length > 0 + && typeof record.linkPath === 'string' && record.linkPath.length > 0 + && typeof record.targetPath === 'string' && record.targetPath.length > 0 + && Array.isArray(record.manifest) + && typeof record.step === 'string' && record.step.length > 0; +} + +/** + * Write (or overwrite) the journal record for one unit, durably. Called + * BEFORE the first filesystem mutation for a unit, and again after every + * subsequent step, so the journal always reflects the furthest point + * actually reached — including across a hard crash. + * + * A naive `open(file, 'w')` + write + fsync has its own crash window: a + * kill between the truncating open and the write leaves an EMPTY or + * PARTIAL record at the well-known path readers expect — which would + * make a real, in-flight unit look like there's simply nothing to + * recover. Instead: write the full content to a throwaway temp file in + * the same directory, fsync THAT file, close it, atomically rename it + * over the real path (a same-directory rename is atomic — readers either + * see the old complete record or the new complete record, never a + * partial one), then fsync the directory itself so the rename survives a + * crash immediately after. On platforms where a directory can't be + * opened for fsync, that failure is swallowed (the rename itself is still + * safe there) — noted here in case it needs to change for a supported + * platform where journal loss would matter. + */ +/** + * Write one journal record durably — or refuse to claim it was written + * durably at all. `opts.fsyncSync`/`opts.renameSync` are test-only + * injection points (default to the real `fsyncSync`/`renameSync`); + * `opts.platform` defaults to the real `process.platform` but is + * injectable so tests can exercise the Linux-vs-other-platform rule + * without actually running on both. + * + * On the supported target (Linux): ANY error from the temp-file fsync, + * the rename into place, or the journal-directory fsync throws — the + * caller aborts the unit before any source mutation that write was meant + * to guard, and no claim of durability is ever made silently. + * + * On a non-Linux dev platform: the same is true EXCEPT for the directory + * fsync specifically — `ENOTSUP` or `EINVAL` there (common on filesystems + * or platforms that don't support fsync-ing a directory fd at all) is + * tolerated, and the write still succeeds, but the returned + * `durabilityNote` says so explicitly rather than silently pretending + * durability was achieved. `EIO` and everything else still abort there + * too — only that specific "this platform doesn't support the operation" + * shape of failure is tolerated, never an I/O error. + */ +function writeJournalRecordSync(journalDir, record, opts = {}) { + const doFsync = opts.fsyncSync || fsyncSync; + const doRename = opts.renameSync || renameSync; + const platform = opts.platform || process.platform; + + mkdirSync(journalDir, { recursive: true }); + const file = journalFilePath(journalDir, record.sourcePath); + const tmpFile = `${file}.tmp-${process.pid}-${randomBytes(6).toString('hex')}`; + const data = JSON.stringify({ ...record, updatedAt: new Date().toISOString() }, null, 2); + + const fd = openSync(tmpFile, 'w'); + try { + writeSync(fd, data); + try { + doFsync(fd); + } catch (e) { + throw new Error(`journal write for ${file} aborted: could not fsync the temp file (${e.code || e.message})`); + } + } finally { + closeSync(fd); + } + + try { + doRename(tmpFile, file); + } catch (e) { + throw new Error(`journal write for ${file} aborted: could not rename the temp file into place (${e.code || e.message})`); + } + + let durabilityNote = null; + try { + const dirFd = openSync(journalDir, 'r'); + try { + doFsync(dirFd); + } finally { + closeSync(dirFd); + } + } catch (e) { + const code = e.code || 'UNKNOWN'; + const tolerable = platform !== 'linux' && (code === 'ENOTSUP' || code === 'EINVAL'); + if (!tolerable) { + const platformNote = platform === 'linux' ? ' (no durability claim can be made on the supported Linux target)' : ''; + throw new Error(`journal write for ${file} aborted: could not fsync the journal directory (${code})${platformNote}`); + } + durabilityNote = `durability degraded: ${code}`; + } + + return { file, durabilityNote }; +} + +function removeJournalRecord(journalDir, sourcePath) { + const file = journalFilePath(journalDir, sourcePath); + try { + rmSync(file); + } catch { + // already gone — fine + } +} + +/** + * All journal records currently on disk. A record that is missing, + * empty, truncated, fails to parse, or doesn't match the expected schema + * is still returned — as `{record: null, corrupt: true, reason}` — so + * callers report it rather than silently skip it or, worse, act on + * whatever partial data it happens to contain. A leftover `.tmp-*` file + * from an interrupted journal WRITE (crashed between creating the temp + * file and the rename) is recognized by name and reported the same way; + * it is never parsed as a record. + */ +function listJournalRecords(journalDir) { + if (!existsSync(journalDir)) return []; + const out = []; + for (const name of safeReaddir(journalDir)) { + const file = join(journalDir, name); + if (name.includes('.tmp-')) { + out.push({ file, record: null, corrupt: true, reason: 'stray incomplete journal write (a .tmp file left over from an interrupted write); never parsed as a record' }); + continue; + } + if (!name.endsWith('.json')) continue; + + let raw; + try { + raw = readFileSync(file, 'utf8'); + } catch (e) { + out.push({ file, record: null, corrupt: true, reason: `journal file unreadable (${e.code || e.message})` }); + continue; + } + if (raw.length === 0) { + out.push({ file, record: null, corrupt: true, reason: 'journal file is empty (zero bytes)' }); + continue; + } + let parsed; + try { + parsed = JSON.parse(raw); + } catch (e) { + out.push({ file, record: null, corrupt: true, reason: `journal file is not valid JSON, likely truncated (${e.message})` }); + continue; + } + if (!isValidJournalRecordShape(parsed)) { + out.push({ file, record: null, corrupt: true, reason: 'journal file does not match the expected record schema' }); + continue; + } + out.push({ file, record: parsed }); + } + return out; +} + +/** size + sha256 (files) / link target (symlinks) for every entry under + * `rootDir`, relative to it. Directories are implied by their children's + * relPaths and not recorded individually. */ +function computeManifest(rootDir) { + const manifest = []; + for (const full of listAllEntries(rootDir)) { + let lst; + try { + lst = lstatSync(full); + } catch { + continue; + } + const relPath = relative(rootDir, full); + if (lst.isSymbolicLink()) { + manifest.push({ relPath, type: 'symlink', linkTarget: readlinkSync(full) }); + } else if (lst.isFile()) { + manifest.push({ relPath, type: 'file', size: lst.size, sha256: sha256File(full) }); + } + } + return manifest; +} + +/** True only if every manifest entry exists under `rootDir` with matching + * size+sha256 (files) or link target (symlinks). Used by recovery to + * refuse to act on a journal record whose paths don't actually match what + * it claims — see recoverInterruptedMoves(). */ +function verifyManifest(rootDir, manifest) { + for (const entry of manifest) { + const full = join(rootDir, entry.relPath); + let lst; + try { + lst = lstatSync(full); + } catch { + return false; + } + if (entry.type === 'symlink') { + if (!lst.isSymbolicLink() || readlinkSync(full) !== entry.linkTarget) return false; + } else { + if (!lst.isFile() || lst.size !== entry.size || sha256File(full) !== entry.sha256) return false; + } + } + return true; +} + +/** Stricter than verifyManifest: every manifest entry must be present and + * match AND there must be no extra file/symlink under `rootDir` beyond + * what the manifest lists. Used specifically for the delete-the-staged- + * original decision, where "the manifest's entries happen to be present" + * is not enough — corrupted-but-additional content must also fail this. */ +function manifestExactMatch(rootDir, manifest) { + if (!existsSync(rootDir)) return false; + if (!verifyManifest(rootDir, manifest)) return false; + const manifestRelPaths = new Set(manifest.map(m => m.relPath)); + let actualCount = 0; + for (const full of listAllEntries(rootDir)) { + let lst; + try { + lst = lstatSync(full); + } catch { + continue; + } + if (!lst.isFile() && !lst.isSymbolicLink()) continue; // directories aren't in the manifest + actualCount++; + if (!manifestRelPaths.has(relative(rootDir, full))) return false; // extra, untracked entry + } + return actualCount === manifestRelPaths.size; +} + +const QUARANTINE_SUFFIX_PREFIX = '.tidy-quarantine-'; + +/** + * The ONLY place in this file allowed to delete OR rename away a staged + * original. Called both from swapToSymlink's own final step (same run) + * and from recovery (a later run) — one guarded function, one invariant, + * everywhere a staged original could be discarded. + * + * Checks three independent facts, then follows an explicit 4-row decision + * table — no single pass/fail collapse, because "the live path is fine" + * and "the backup is fine" are NOT the same question, and conflating them + * caused two different real data-loss bugs (see git history): + * targetOk — every file under the journal's targetPath matches the + * manifest by size + SHA-256, no extra or missing paths. + * The link having been verified once, when it was + * CREATED, is not evidence about the target's content + * NOW — it can change afterwards (corruption, a second + * process, disk issues). + * linkOk — sourcePath is a symlink whose realpath resolves to + * exactly the journal's recorded targetPath. + * stagedState — 'absent' (no staged copy exists — fine, nothing to + * protect), 'matching' (exists and matches the manifest + * exactly), or 'damaged' (exists but does not match). + * + * | # | targetOk && linkOk | stagedState | Action | + * |---|---------------------|---------------------|--------| + * | 1 | true | absent or matching | complete: delete the staged copy if present, clear the journal. | + * | 2 | true | damaged | the live path is correct and complete — PRESERVE it untouched. Do NOT delete the damaged staging; QUARANTINE it to `.tidy-quarantine-` and record that path in the journal (`step: 'completed-partial-staging-quarantined'`). Nothing is deleted. | + * | 3 | false | matching | restore: remove a bad/half symlink at sourcePath if present, rename the staged copy back to sourcePath, verify it matches the manifest, journal `step: 'failed'`. | + * | 4 | false | absent or damaged | nothing verified good anywhere — DELETE NOTHING, RENAME NOTHING, leave every path exactly as found, journal `step: 'failed-both-copies-damaged'` with the per-check reasons. | + * + * Row 1 and row 3 never delete/restore a staged copy that's merely + * "absent" — there's nothing there to act on. Row 2 exists specifically + * so a live, correct, already-in-use path is never sacrificed just + * because its backup has a problem. Row 4 exists specifically so that, + * when there is genuinely nothing good anywhere, this function refuses to + * guess — it touches nothing rather than pick a side. + */ + +/** + * The ONLY way anything in this file removes a path it believes is a + * symlink it created. `rmSync(..., {recursive:true})` on an assumed + * symlink is dangerous: if that assumption is wrong — the path is + * actually a real directory someone else put there, or a symlink + * pointing somewhere unexpected — a recursive remove can delete an + * entire directory tree that was never ours to touch. This helper + * verifies before it ever unlinks anything, and never recurses: + * + * 1. lstatSync(path) — if it doesn't exist, or isn't a symlink at all + * (a real file or directory sitting where we expected a symlink), + * REFUSE. Touch nothing. + * 2. its realpath must resolve to exactly `expectedTargetPath`'s own + * realpath — a symlink pointing somewhere else (tampered, or a + * coincidentally-named path from something unrelated) is REFUSED, + * not unlinked. + * 3. only then: `unlinkSync(path)` — never `rmSync`, never recursive. + * A single unlink can only ever remove the one symlink entry itself, + * never anything reachable through it. + * + * Callers treat a refusal as "left alone, reported" — never a reason to + * fall back to a more aggressive removal. + */ +function unlinkOwnedSymlink(path, expectedTargetPath) { + let lst; + try { + lst = lstatSync(path); + } catch (e) { + return { ok: false, reason: `expected an owned symlink at ${path}, but it does not exist (${e.code || e.message})` }; + } + if (!lst.isSymbolicLink()) { + const kind = lst.isDirectory() ? 'a directory' : lst.isFile() ? 'a regular file' : 'a non-symlink entry'; + return { ok: false, reason: `expected an owned symlink at ${path}, found ${kind} instead — refusing to touch it` }; + } + let real; + try { + real = realpathSync(path); + } catch (e) { + return { ok: false, reason: `symlink at ${path} could not be resolved (${e.code || e.message}) — refusing to touch it` }; + } + let expectedReal; + try { + expectedReal = realpathSync(expectedTargetPath); + } catch (e) { + return { ok: false, reason: `recorded target ${expectedTargetPath} could not be resolved (${e.code || e.message}) — refusing to touch ${path}` }; + } + if (real !== expectedReal) { + return { ok: false, reason: `symlink at ${path} resolves to ${real}, not the recorded target ${expectedTargetPath} — refusing to touch it` }; + } + try { + unlinkSync(path); + } catch (e) { + return { ok: false, reason: `unlink of verified symlink ${path} failed (${e.code || e.message})` }; + } + return { ok: true }; +} + +function finalizeSwapOrRestore(journalDir, record, opts = {}) { + const { sourcePath, stagedPath, linkPath, targetPath, manifest } = record; + + const targetOk = manifestExactMatch(targetPath, manifest); + + let sourceLstat = null; + try { + sourceLstat = lstatSync(sourcePath); + } catch { + sourceLstat = null; + } + const srcExists = !!sourceLstat; + const srcIsSymlink = !!(sourceLstat && sourceLstat.isSymbolicLink()); + const linkExists = existsSync(linkPath); + + // Is there a verifiably-correct symlink representing this unit's live + // path — already placed at sourcePath, or still pending (unconsumed) at + // linkPath? Either counts as "the link is fine"; WHICH one it is + // determines whether completing the pending rename is even on the + // table below (it only ever happens as part of executing row 1 or row + // 2 — never speculatively, and never for row 3 or row 4). + let linkOk = false; + let linkLocation = null; // 'source' | 'pending' | null + if (srcIsSymlink) { + try { + if (realpathSync(sourcePath) === realpathSync(targetPath)) { + linkOk = true; + linkLocation = 'source'; + } + } catch { + linkOk = false; + } + } else if (!srcExists && linkExists) { + try { + if (lstatSync(linkPath).isSymbolicLink() && realpathSync(linkPath) === realpathSync(targetPath)) { + linkOk = true; + linkLocation = 'pending'; + } + } catch { + linkOk = false; + } + } + + const stagedState = !existsSync(stagedPath) + ? 'absent' + : (manifestExactMatch(stagedPath, manifest) ? 'matching' : 'damaged'); + + // Row 1: live path correct, and there's either nothing staged to worry + // about or it matches too — complete normally. Only HERE does a + // pending rename actually get completed. + if (targetOk && linkOk && stagedState !== 'damaged') { + if (linkLocation === 'pending') renameSync(linkPath, sourcePath); + if (stagedState === 'matching') rmSync(stagedPath, { recursive: true }); + removeJournalRecord(journalDir, sourcePath); + return { ok: true, row: 1 }; + } + + // Row 2: live path correct and complete, but the backup is damaged. + // Preserve the live path exactly as-is (completing the pending rename + // if needed, since the target+link half of this unit is fully + // verified); never delete the damaged backup silently — quarantine it. + if (targetOk && linkOk && stagedState === 'damaged') { + if (linkLocation === 'pending') renameSync(linkPath, sourcePath); + const quarantinePath = `${sourcePath}${QUARANTINE_SUFFIX_PREFIX}${journalKey(sourcePath)}`; + const reason = 'the staged original was damaged (partial or corrupted content) but the live target and symlink are complete and correct; the live path was left untouched and the damaged staged copy was quarantined rather than deleted or silently trusted'; + renameSync(stagedPath, quarantinePath); + const { durabilityNote } = writeJournalRecordSync(journalDir, { ...record, step: 'completed-partial-staging-quarantined', quarantinePath }, opts); + return { ok: true, row: 2, quarantined: true, quarantinePath, reason, durabilityNotes: durabilityNote ? [durabilityNote] : [] }; + } + + const reasons = []; + if (!targetOk) reasons.push('target content does not match the journaled manifest exactly (missing, extra, or corrupted files)'); + if (!linkOk) reasons.push('source is not a symlink whose realpath resolves exactly to the recorded target path'); + + // Row 3: live path has a problem, but the staged original is intact — + // restore from it. Clears away a bad symlink wherever it currently + // sits (already at sourcePath, or still pending at linkPath) — but ONLY + // via unlinkOwnedSymlink's verify-then-unlink, never a recursive + // remove on an assumed symlink. If that verification refuses (the path + // isn't actually a symlink, or resolves somewhere unexpected), row 3 + // itself refuses too: touch nothing further, report, journal 'failed'. + if (stagedState === 'matching') { + const reason = reasons.join('; '); + + let obstaclePath = null; + if (srcIsSymlink) obstaclePath = sourcePath; + else if (!srcExists && linkExists) obstaclePath = linkPath; + // else: sourcePath holds something that's neither a verifiable + // symlink nor absent-with-a-pending-link — nothing to clear first. + + if (obstaclePath) { + const unlinkResult = unlinkOwnedSymlink(obstaclePath, targetPath); + if (!unlinkResult.ok) { + const combinedReason = `${reason}; additionally, could not safely clear the way for restore: ${unlinkResult.reason}`; + try { + writeJournalRecordSync(journalDir, { ...record, step: 'failed', failureReason: combinedReason }, opts); + } catch { + // best effort + } + return { ok: false, row: 3, reason: combinedReason, restored: false, refusedUnlink: true }; + } + } + + let restored = false; + if (!existsSync(sourcePath)) { + try { + renameSync(stagedPath, sourcePath); + restored = manifestExactMatch(sourcePath, manifest); + } catch { + restored = false; + } + } + // else: sourcePath still holds something unexpected (removal above + // didn't apply) — leave both paths for manual inspection rather than + // guess which is authoritative. + try { + writeJournalRecordSync(journalDir, { ...record, step: 'failed', failureReason: reason }, opts); + } catch { + // best effort — the reason is still returned to the caller either way + } + return { ok: false, row: 3, reason, restored }; + } + + // Row 4: nothing verified good anywhere (live path has a problem AND + // the backup is absent or also damaged). DELETE NOTHING, RENAME + // NOTHING — sourcePath, stagedPath, and linkPath are all left exactly + // as found, including a still-pending, uncompleted rename. Report + // loudly with every path so a human can recover by hand. + reasons.push(stagedState === 'absent' + ? 'no staged original exists to fall back on' + : 'the staged original itself no longer matches the journaled manifest exactly'); + const reason = reasons.join('; '); + try { + writeJournalRecordSync(journalDir, { ...record, step: 'failed-both-copies-damaged', failureReason: reason }, opts); + } catch { + // best effort + } + return { + ok: false, + row: 4, + reason, + restored: false, + untouched: true, + paths: { sourcePath, stagedPath, linkPath, targetPath } + }; +} + +/** + * Swap one source directory for a symlink to its already-verified copy. + * + * A plain "delete then symlink" (or even "rename then symlink") has a + * window where a hard crash (SIGKILL, power loss, OOM kill) — which a + * try/catch cannot intercept, unlike a thrown JS exception — leaves NEITHER + * the original NOR a symlink at `src`. To close that, the symlink is built + * and verified at a side path BEFORE `src` is touched at all, and every + * step (including this one) is journaled to `opts.journalDir` — fsynced — + * BEFORE the step's filesystem mutation happens, so recovery always knows + * the furthest point actually reached: + * + * 'pending' written -> symlink(dst, src+LINK_SUFFIX); verify it resolves + * [src untouched so far — a crash here leaves src exactly as it was] + * 'linked' written -> rename(src, src+STAGING_SUFFIX) [atomic] + * [a crash here leaves the original at STAGING_SUFFIX, recoverable] + * 'staged' written -> rename(src+LINK_SUFFIX, src) [atomic] + * [THE GAP: for this instant, neither a real directory nor a symlink + * exists at src — see the module docstring. A crash here needs + * recovery to finish this exact rename from the journal.] + * 'swapped' written -> verify src resolves to dst again, remove staging + * [a crash here leaves a working symlink at src already — the only + * thing left undone is freeing the disk space, which recovery does] + * journal record removed once cleanup succeeds. + * + * This function itself does NOT roll back on a thrown (catchable) failure + * — recovery is deliberately a separate, explicit, journal-validated step + * (recoverInterruptedMoves, run via the `recover` subcommand or at the + * start of `apply`), not an automatic side effect of error handling, so a + * hard kill and a thrown exception are recovered the exact same way. + */ +function swapToSymlink(src, dst, opts = {}) { + const doSymlink = opts.symlinkSync || symlinkSync; + const staging = src + STAGING_SUFFIX; + const link = src + LINK_SUFFIX; + const journalDir = opts.journalDir; + const journalOpts = { fsyncSync: opts.journalFsyncSync, renameSync: opts.journalRenameSync, platform: opts.platform }; + const durabilityNotes = []; + function journalWrite(rec) { + // GAP 2 (round 6): a failure here — temp-file fsync, rename into + // place, or (on Linux) directory fsync — THROWS, which this function + // does not catch, so it propagates to the caller and aborts the whole + // unit before whatever mutation this write was meant to guard. Never + // silently swallowed. + const { durabilityNote } = writeJournalRecordSync(journalDir, rec, journalOpts); + if (durabilityNote) durabilityNotes.push(durabilityNote); + } + + if (existsSync(staging) || existsSync(link)) { + throw new Error(`refusing to touch ${src}: a leftover ${existsSync(staging) ? staging : link} already exists from a previous run — run the 'recover' subcommand first`); + } + + const manifest = computeManifest(src); + const base = { sourcePath: src, stagedPath: staging, linkPath: link, targetPath: dst, manifest }; + + journalWrite({ ...base, step: 'pending' }); // BEFORE any mutation — a failure here aborts with the source completely untouched + if (opts.beforeLink) opts.beforeLink(); // test-only hook: simulate a crash here + + doSymlink(dst, link); + let preSwapVerified = false; + try { + preSwapVerified = lstatSync(link).isSymbolicLink() && realpathSync(link) === realpathSync(dst); + } catch { + preSwapVerified = false; + } + if (!preSwapVerified) { + // Clear the bad/half artifact at `link` — but only via the same + // verify-then-unlink guard every other removal in this file uses, in + // case `doSymlink` didn't actually create a symlink at all (e.g. a + // real directory ended up there instead): never rmSync/recursive on + // an assumption. + const unlinkResult = unlinkOwnedSymlink(link, dst); + if (unlinkResult.ok) { + removeJournalRecord(journalDir, src); + throw new Error(`pre-swap symlink verification failed for ${link}`); + } + const reason = `pre-swap symlink verification failed for ${link}, and it could not be safely removed: ${unlinkResult.reason}`; + try { + journalWrite({ ...base, step: 'failed', failureReason: reason }); + } catch { + // best effort — the reason is still thrown below either way + } + throw new Error(reason); + } + journalWrite({ ...base, step: 'linked' }); // still before the first source mutation (the rename below) + if (opts.afterLink) opts.afterLink(); // test-only hook: simulate a crash here + + renameSync(src, staging); + journalWrite({ ...base, step: 'staged' }); + if (opts.afterStage) opts.afterStage(); // test-only hook: simulate a crash IN THE GAP between the two renames + + renameSync(link, src); + journalWrite({ ...base, step: 'swapped' }); + if (opts.afterSwap) opts.afterSwap(); // test-only hook: simulate a crash here + + // The ONLY place a staged original is deleted, in this run or later via + // recovery — see finalizeSwapOrRestore's docstring for the three checks. + // "The link resolves to something" is deliberately not one of them. + const result = finalizeSwapOrRestore(journalDir, base, journalOpts); + if (!result.ok) { + const outcome = result.row === 4 + ? 'left every path exactly as found (nothing verified good anywhere)' + : `restored the staged original to ${src}`; + throw new Error(`post-swap finalize refused to delete the staged original and ${outcome}: ${result.reason}`); + } + return { ...result, durabilityNotes: [...durabilityNotes, ...(result.durabilityNotes || [])] }; + // row 1: plain success. row 2: success, but carries {quarantined:true, quarantinePath, reason}. +} + +/** Every location discoverCandidates() looks at, reused so detection and + * recovery scan for stray *.tidy-moving/*.tidy-link siblings in exactly + * the same places. */ +function candidateParentDirs(home) { + const dirs = new Set(); + const hfHub = join(home, '.cache', 'huggingface', 'hub'); + if (existsSync(hfHub)) dirs.add(hfHub); + const modelsDir = join(home, 'models'); + if (existsSync(modelsDir)) dirs.add(modelsDir); + dirs.add(home); + for (const entry of safeReaddir(home)) { + if (entry.startsWith('.')) continue; + const full = join(home, entry); + try { + if (lstatSync(full).isDirectory()) dirs.add(full); + } catch { + // vanished between readdir and lstat — ignore + } + } + return [...dirs]; +} + +function strayJournalSuffixedPaths(home, excludePaths) { + const strays = []; + for (const dir of candidateParentDirs(home)) { + for (const name of safeReaddir(dir)) { + if (!isJournalSuffixed(name)) continue; + let src; + if (name.includes(QUARANTINE_SUFFIX_PREFIX)) { + // A quarantine dir represents itself (not some other "base" name + // with a suffix stripped) — it's excluded by its own full path, + // which is what a journal record's `quarantinePath` field holds. + src = join(dir, name); + } else { + const base = name.endsWith(STAGING_SUFFIX) ? name.slice(0, -STAGING_SUFFIX.length) : name.slice(0, -LINK_SUFFIX.length); + src = join(dir, base); + } + if (excludePaths.has(src)) continue; + strays.push({ path: src, entryPath: join(dir, name) }); + } + } + return strays; +} + +/** + * READ-ONLY detection of interrupted `apply` swaps, for `plan`. Never + * mutates anything. For every journal record found under + * `home/.cache/ide-agent-kit/model-tidy-journal`, reports the step reached + * and a human-readable description of the on-disk state, WITHOUT touching + * it. Also reports (but never touches) any `*.tidy-moving` / `*.tidy-link` + * sibling that has no matching journal record at all — those are not this + * tool's to interpret; only recoverInterruptedMoves() (a separate, explicit, + * mutating step) acts on journaled units, and only after validating them. + */ +export function detectInterruptedMoves(home, options = {}) { + const journalDir = options.journalDir || defaultJournalDir(home); + const findings = []; + const journaledPaths = new Set(); + + for (const { file, record, corrupt, reason } of listJournalRecords(journalDir)) { + if (corrupt || !record) { + findings.push({ path: null, journalFile: file, status: 'journal-unreadable', note: `journal file ${file} is unreadable: ${reason || 'unknown reason'} — the unit it may describe is left alone until this is resolved` }); + continue; + } + journaledPaths.add(record.sourcePath); + if (record.quarantinePath) journaledPaths.add(record.quarantinePath); + const srcExists = existsSync(record.sourcePath); + let srcIsSymlink = false; + try { + srcIsSymlink = lstatSync(record.sourcePath).isSymbolicLink(); + } catch { + // does not exist — srcIsSymlink stays false + } + const stagedExists = existsSync(record.stagedPath); + const linkExists = existsSync(record.linkPath); + findings.push({ + path: record.sourcePath, + journalFile: file, + step: record.step, + status: 'interrupted', + note: `journal step '${record.step}' — source ${srcExists ? (srcIsSymlink ? 'exists as a symlink' : 'exists as a real directory') : 'missing'}, ` + + `staged copy ${stagedExists ? 'present' : 'absent'}, pending link ${linkExists ? 'present' : 'absent'}. Run the 'recover' subcommand to resolve.` + }); + } + + for (const stray of strayJournalSuffixedPaths(home, journaledPaths)) { + findings.push({ + path: stray.path, + journalFile: null, + status: 'orphan-no-journal', + note: `found ${stray.entryPath} with no matching journal record — not this tool's to interpret, left alone` + }); + } + + return findings; +} + +/** + * MUTATING recovery for interrupted `apply` swaps. Only runs from the + * explicit `recover` subcommand, or at the very start of `apply` (i.e. + * only when `--apply` was actually given — `plan` never calls this). + * + * Acts ONLY on units that have a journal record — a directory merely + * *named* `*.tidy-moving` or `*.tidy-link` with no journal entry is never + * touched, regardless of how it looks. For a journaled unit, validates + * that whatever currently exists on disk (the source as a real directory, + * and/or the staged copy) matches the journal's recorded manifest before + * trusting the record enough to act; a mismatch is reported and left + * alone rather than guessed at. + */ +export function recoverInterruptedMoves(home, options = {}) { + const journalDir = options.journalDir || defaultJournalDir(home); + const journalOpts = { fsyncSync: options.journalFsyncSync, renameSync: options.journalRenameSync, platform: options.platform }; + const recovered = []; + const journaledPaths = new Set(); + + for (const { file, record, corrupt, reason: unreadableReason } of listJournalRecords(journalDir)) { + if (corrupt || !record) { + recovered.push({ path: null, journalFile: file, action: 'left-alone', note: `journal file ${file} is unreadable: ${unreadableReason || 'unknown reason'} — left alone, never acted on` }); + continue; + } + const { sourcePath, stagedPath, linkPath, targetPath, manifest, step } = record; + journaledPaths.add(sourcePath); + if (record.quarantinePath) journaledPaths.add(record.quarantinePath); + + try { + const srcExists = existsSync(sourcePath); + let srcIsSymlink = false; + try { + srcIsSymlink = lstatSync(sourcePath).isSymbolicLink(); + } catch { + // missing — srcIsSymlink stays false + } + const stagedExists = existsSync(stagedPath); + const linkExists = existsSync(linkPath); + + // A symlink is either already in place, or one verified rename away + // from being in place: the ONLY safe way to decide anything here — + // including whether it's even safe to COMPLETE that pending rename + // — is finalizeSwapOrRestore's own decision table (target manifest, + // symlink realpath, staged manifest, all re-checked now). It is + // NOT safe to complete the rename speculatively before that + // decision: row 4 (nothing verified good anywhere) must rename + // NOTHING, so finalizeSwapOrRestore performs the pending rename + // itself, only inside the row-1/row-2 branches that decide it's + // warranted. + if ((srcExists && srcIsSymlink) || (!srcExists && stagedExists && linkExists)) { + const result = finalizeSwapOrRestore(journalDir, record, journalOpts); + if (result.ok && result.quarantined) { + // Row 2: live path is correct and complete — preserved, + // untouched. The damaged backup was quarantined, not deleted. + // The journal record was just rewritten with this quarantine + // path, so exclude it from the stray scan below too. + journaledPaths.add(result.quarantinePath); + recovered.push({ + path: sourcePath, + journalFile: file, + action: 'completed-partial-staging-quarantined', + note: `${result.reason} (quarantined at ${result.quarantinePath})` + }); + } else if (result.ok) { + // Row 1. + recovered.push({ path: sourcePath, journalFile: file, action: 'completed-swap-and-cleaned', note: 'completed the pending symlink swap (or finished cleanup of one already in place) and removed the staged original — target and symlink both re-verified against the journal manifest first' }); + } else if (result.row === 4) { + // Row 4: nothing verified good anywhere — touched nothing. + recovered.push({ + path: sourcePath, + journalFile: file, + action: 'left-alone-nothing-verified-good', + note: `${result.reason} — every path left exactly as found (source: ${result.paths.sourcePath}, staged: ${result.paths.stagedPath}, target: ${result.paths.targetPath}); recover by hand` + }); + } else { + // Row 3. + recovered.push({ + path: sourcePath, + journalFile: file, + action: result.restored ? 'restored-after-failed-verification' : 'left-alone-after-failed-verification', + note: `refused to delete the staged original (${result.reason}); ${result.restored ? 'restored the original to the source path' : 'left current state for manual inspection'} and marked the journal record failed` + }); + } + continue; + } + + // Below here, no symlink has ever been placed at sourcePath — the + // delete-the-staged-original decision never applies, so the plainer + // manifest checks are enough. + const stagedOk = !stagedExists || verifyManifest(stagedPath, manifest); + const srcOk = !srcExists || verifyManifest(sourcePath, manifest); + if (!stagedOk || !srcOk) { + recovered.push({ path: sourcePath, journalFile: file, action: 'left-alone', note: `on-disk content (step recorded as '${step}') does not match the journaled manifest; left alone for manual inspection` }); + continue; + } + + if (!srcExists && stagedExists && !linkExists) { + renameSync(stagedPath, sourcePath); + removeJournalRecord(journalDir, sourcePath); + recovered.push({ path: sourcePath, journalFile: file, action: 'restored-original', note: 'restored the original directory (no verified pending symlink existed to trust instead)' }); + } else if (srcExists && !srcIsSymlink && linkExists && !stagedExists) { + const unlinkResult = unlinkOwnedSymlink(linkPath, targetPath); + if (!unlinkResult.ok) { + try { + writeJournalRecordSync(journalDir, { ...record, step: 'failed', failureReason: unlinkResult.reason }); + } catch { + // best effort + } + recovered.push({ path: sourcePath, journalFile: file, action: 'left-alone', note: `refused to remove the pending link (${unlinkResult.reason}); the original itself was never touched` }); + continue; + } + removeJournalRecord(journalDir, sourcePath); + recovered.push({ path: sourcePath, journalFile: file, action: 'removed-stray-link', note: 'the original was never touched; removed the unused pending symlink' }); + } else if (srcExists && !srcIsSymlink && !linkExists && !stagedExists) { + removeJournalRecord(journalDir, sourcePath); + recovered.push({ path: sourcePath, journalFile: file, action: 'removed-stale-journal', note: 'the original was never touched; cleared the journal record' }); + } else { + recovered.push({ path: sourcePath, journalFile: file, action: 'left-alone', note: `unrecognized on-disk state for a journaled unit (step='${step}'); left alone for manual inspection` }); + } + } catch (e) { + recovered.push({ path: sourcePath, journalFile: file, action: 'error', note: `recovery failed: ${e.message}` }); + } + } + + for (const stray of strayJournalSuffixedPaths(home, journaledPaths)) { + recovered.push({ + path: stray.path, + journalFile: null, + action: 'left-alone', + note: `found ${stray.entryPath} with no matching journal record — not this tool's to touch` + }); + } + + return recovered; +} + +/** + * Apply a plan: move every selected unit to `target`, unit by unit. Each + * unit is copied, verified, and only THEN is its source directory swapped + * for a symlink via swapToSymlink()'s journaled sequence (see its + * docstring). A failure at copy or verify leaves the source fully in + * place, untouched. Once the swap itself starts, the original data is + * preserved and recoverable at every step; the original path is + * unavailable for the instant between the two renames, and until recovery + * runs if a crash lands there — it is not continuously available. One + * unit's failure does not roll back units that already succeeded earlier + * in the same run; the process still exits non-zero overall. + * + * Recovery (recoverInterruptedMoves) runs once, here, at the very start — + * i.e. only when `--apply` was actually given. `planRun` never calls it; + * `plan` only detects and reports interrupted state (detectInterruptedMoves) + * without mutating anything. + */ +export function applyRun(options) { + const { plan, target, home } = options; + const copyFn = options.copyFn || copyUnitPureNode; + const verifyFn = options.verifyFn || verifyUnit; + const validateTargetFn = options.validateTarget || validateTarget; + const symlinkFn = options.symlinkFn; // test-only injection; real default is symlinkSync inside swapToSymlink + const recoverFn = options.recover || recoverInterruptedMoves; + const journalDir = options.journalDir || defaultJournalDir(home); + // GAP 2 (round 6) test-only injection points: the real journal fsync, + // the real journal rename, and process.platform, all overridable so + // tests can exercise the Linux-vs-other-platform durability rule + // without branching on the real platform inside the test itself. + const journalFsyncSync = options.journalFsyncSync; + const journalRenameSync = options.journalRenameSync; + const journalPlatform = options.platform; + + // GAP (round 8): this preflight MUST run before recoverFn below, not + // after. Recovery is itself a mutation (it promotes valid interrupted + // units to symlinks and deletes their staged originals), and the whole + // point of this check is a GLOBAL refusal to mutate anything at all + // while any journal file's content can't be trusted — including units + // that recovery would otherwise have happily fixed. Running it after + // recovery meant apply could report "unresolved journal state, refusing" + // while having already mutated a perfectly valid unit moments earlier. + // recoverInterruptedMoves never touches an unreadable/corrupt record + // either way (it always reports and leaves those alone), so the set of + // unreadable files this check sees is identical whether it runs before + // or after recovery — only WHEN it's allowed to act differs. + const preflightUnreadable = listJournalRecords(journalDir).filter(r => r.corrupt || !r.record); + if (preflightUnreadable.length > 0) { + const files = preflightUnreadable.map(r => r.file); + const reason = `unresolved journal state: ${files.join(', ')}; run recover, or resolve by hand`; + return { ok: false, error: reason, moved: [], errors: [{ error: reason }], recovered: [] }; + } + + // Self-heal any interrupted swap from a previous crash before this run's + // own pre-checks and copy/verify/swap loop — see recoverInterruptedMoves. + // Only reached here, in apply, never from planRun. Only reached at all + // once the preflight above confirms every journal file is trustworthy. + const recovered = recoverFn(home, { journalDir, journalFsyncSync, journalRenameSync, platform: journalPlatform }); + + const validation = validateTargetFn(target, home); + if (!validation.ok) { + return { ok: false, error: validation.error, moved: [], errors: [{ error: validation.error }], recovered }; + } + + const byGroup = new Map(); + for (const r of plan.selected) { + if (!byGroup.has(r.groupId)) byGroup.set(r.groupId, []); + byGroup.get(r.groupId).push(r); + } + + const moved = []; + const errors = []; + const durabilityNotes = []; // any 'durability degraded: ' notes from journal writes during this run + + for (const [groupId, members] of byGroup) { + const sourceAbsPaths = members.map(m => m.path); + + const leftover = sourceAbsPaths.filter(src => existsSync(src + STAGING_SUFFIX) || existsSync(src + LINK_SUFFIX)); + if (leftover.length > 0) { + errors.push({ + groupId, + paths: sourceAbsPaths, + step: 'pre-check', + error: `refusing: a leftover from a previous run still exists for ${leftover.join(', ')} — run the 'recover' subcommand first` + }); + continue; + } + + try { + copyFn(sourceAbsPaths, home, target); + } catch (e) { + errors.push({ groupId, paths: sourceAbsPaths, step: 'copy', error: e.message }); + continue; + } + + const verification = verifyFn(sourceAbsPaths, home, target); + if (!verification.ok) { + errors.push({ groupId, paths: sourceAbsPaths, step: 'verify', error: 'checksum/size verification failed', mismatches: verification.mismatches }); + continue; // source untouched — verification failed BEFORE any deletion + } + + // Verification passed for the whole unit: safe to swap every member. + let swapFailed = null; + const swapped = []; + for (const src of sourceAbsPaths) { + const rel = relative(home, src); + const dst = join(target, rel); + try { + const finalizeResult = swapToSymlink(src, dst, { + symlinkSync: symlinkFn, + journalDir, + journalFsyncSync, + journalRenameSync, + platform: journalPlatform, + beforeLink: options.beforeLink, + afterLink: options.afterLink, + afterStage: options.afterStage, + afterSwap: options.afterSwap + }); + const entry = { source: src, target: dst }; + if (finalizeResult && finalizeResult.quarantined) { + entry.quarantined = true; + entry.quarantinePath = finalizeResult.quarantinePath; + entry.note = finalizeResult.reason; + } + if (finalizeResult && finalizeResult.durabilityNotes && finalizeResult.durabilityNotes.length > 0) { + entry.durabilityNotes = finalizeResult.durabilityNotes; + durabilityNotes.push(...finalizeResult.durabilityNotes); + } + swapped.push(entry); + } catch (e) { + swapFailed = { path: src, error: e.message }; + break; + } + } + if (swapFailed) { + errors.push({ groupId, paths: sourceAbsPaths, step: 'swap', error: swapFailed.error, partiallySwapped: swapped }); + } else { + moved.push({ groupId, members: swapped, sizeBytes: members[0].groupSizeBytes }); + } + } + + return { ok: errors.length === 0, moved, errors, recovered, durabilityNotes }; +} + +// --------------------------------------------------------------------------- +// JSON run log +// --------------------------------------------------------------------------- + +export function writeRunLog(logDir, record) { + mkdirSync(logDir, { recursive: true }); + const line = JSON.stringify(record) + '\n'; + const file = join(logDir, `${new Date().toISOString().slice(0, 10)}.jsonl`); + appendFileSync(file, line); + return file; +} diff --git a/systemd/model-tidy.service b/systemd/model-tidy.service new file mode 100644 index 0000000..c316fdb --- /dev/null +++ b/systemd/model-tidy.service @@ -0,0 +1,20 @@ +[Unit] +Description=model-tidy: plan (and optionally apply) idle-model cleanup +Wants=network-online.target +After=docker.service + +[Service] +Type=oneshot +# PLAN MODE ONLY by default — never moves, deletes, or symlinks anything. +# To let the nightly timer actually move models, edit this line to add +# "apply --apply --target /mnt/your-target" AFTER you have reviewed several +# plan runs' output/logs yourself. Do not do this unattended on day one. +ExecStart=/usr/bin/env node %h/ide-agent-kit/bin/model-tidy.mjs plan +# Room reporting is opt-in and OFF here by default. To turn it on, add: +# --report-to-room --room thinkoff-development --config %h/ide-agent-kit/ide-agent-kit.json +# to the ExecStart line above. +Nice=19 +IOSchedulingClass=idle + +[Install] +WantedBy=default.target diff --git a/systemd/model-tidy.timer b/systemd/model-tidy.timer new file mode 100644 index 0000000..b212e4a --- /dev/null +++ b/systemd/model-tidy.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Nightly model-tidy plan run + +[Timer] +OnCalendar=*-*-* 03:30:00 +RandomizedDelaySec=900 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/test/model-tidy.test.mjs b/test/model-tidy.test.mjs new file mode 100644 index 0000000..384b707 --- /dev/null +++ b/test/model-tidy.test.mjs @@ -0,0 +1,1895 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +import { describe, it, afterEach } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { + mkdtempSync, mkdirSync, writeFileSync, symlinkSync, linkSync, rmSync, + existsSync, readFileSync, utimesSync, lstatSync, readdirSync, readlinkSync +} from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { spawnSync } from 'node:child_process'; +import { createHash, randomBytes } from 'node:crypto'; +import { + planRun, applyRun, discoverCandidates, loadKeepList, computeHardlinkGroups, + validateTarget, copyUnitPureNode, recoverInterruptedMoves, detectInterruptedMoves, + findProcessUsers, sha256File, HASH_CHUNK_BYTES +} from '../src/model-tidy.mjs'; + +const MODEL_TIDY_MODULE_URL = new URL('../src/model-tidy.mjs', import.meta.url).href; + +/** + * Run applyRun in a CHILD process with one option forced to hard-exit + * (process.exit, not a throw — a try/catch in the parent's process cannot + * intercept this, which is exactly the real-world crash swapToSymlink's + * design has to survive: SIGKILL, OOM kill, power loss). Mirrors codexmb's + * review probe technique verbatim: serialize the plan, embed an absolute + * import URL, embed a literal crashing arrow function as one applyRun + * option keyed by `crashOptionKey` ('symlinkFn', 'afterStage', or + * 'afterSwap'). + */ +function runApplyInChildWithCrash(plan, home, target, crashOptionKey) { + const code = `import {applyRun} from ${JSON.stringify(MODEL_TIDY_MODULE_URL)}; applyRun({plan:${JSON.stringify(plan)},home:${JSON.stringify(home)},target:${JSON.stringify(target)},validateTarget:()=>({ok:true}),${crashOptionKey}:()=>process.exit(77)});`; + return spawnSync(process.execPath, ['--input-type=module', '-e', code], { encoding: 'utf8' }); +} + +const tempPaths = []; + +afterEach(() => { + while (tempPaths.length > 0) { + const path = tempPaths.pop(); + rmSync(path, { recursive: true, force: true }); + } +}); + +function tempDir(prefix) { + const dir = mkdtempSync(join(tmpdir(), prefix)); + tempPaths.push(dir); + return dir; +} + +function writeFile(path, content = 'x'.repeat(1024)) { + mkdirSync(join(path, '..'), { recursive: true }); + writeFileSync(path, content); +} + +function daysAgo(n) { + return new Date(Date.now() - n * 86400000); +} + +function touch(path, when) { + utimesSync(path, when, when); +} + +/** + * Build a fixture home tree: + * .cache/huggingface/hub/models--idle-model/ <- idle HF cache model (selected) + * blobs/ real file, old mtime + * snapshots/rev1/file.safetensors -> ../../blobs/ symlink + * .cache/huggingface/hub/models--recent-model/ <- too-recent, skipped + * .cache/huggingface/hub/models--already-tidied <- already a symlink, skipped + * models/keep-me/ <- KEEP-listed, skipped + * models/hardlink-a/file.bin <-hardlinked-> models/hardlink-b/file.bin + * (both old mtime; both must be selected together, as one unit) + */ +function buildFixture() { + const home = tempDir('model-tidy-fixture-'); + const old = daysAgo(30); + const recent = daysAgo(1); + + // idle HF cache model + const idleRoot = join(home, '.cache/huggingface/hub/models--idle-model'); + const blobPath = join(idleRoot, 'blobs', 'abc123'); + writeFile(blobPath, 'idle-model-bytes'.repeat(100)); + touch(blobPath, old); + mkdirSync(join(idleRoot, 'snapshots', 'rev1'), { recursive: true }); + symlinkSync(join('..', '..', 'blobs', 'abc123'), join(idleRoot, 'snapshots', 'rev1', 'model.safetensors')); + + // too-recent HF cache model + const recentRoot = join(home, '.cache/huggingface/hub/models--recent-model'); + const recentBlob = join(recentRoot, 'blobs', 'def456'); + writeFile(recentBlob, 'recent-model-bytes'.repeat(100)); + touch(recentBlob, recent); + + // already-tidied: a symlink standing in for a models--* dir + const tidiedTargetDir = join(home, '.cache', 'elsewhere-target'); + mkdirSync(tidiedTargetDir, { recursive: true }); + writeFile(join(tidiedTargetDir, 'blobs', 'file'), 'already moved'); + symlinkSync(tidiedTargetDir, join(home, '.cache/huggingface/hub', 'models--already-tidied')); + + // KEEP-listed dir under ~/models + const keepRoot = join(home, 'models', 'keep-me'); + writeFile(join(keepRoot, 'weights.gguf'), 'keep-me-bytes'.repeat(100)); + touch(join(keepRoot, 'weights.gguf'), old); + + // hardlink pair under ~/models — must move together or not at all + const hardlinkA = join(home, 'models', 'hardlink-a'); + const hardlinkB = join(home, 'models', 'hardlink-b'); + mkdirSync(hardlinkA, { recursive: true }); + mkdirSync(hardlinkB, { recursive: true }); + writeFile(join(hardlinkA, 'file.bin'), 'shared-hardlinked-bytes'.repeat(100)); + touch(join(hardlinkA, 'file.bin'), old); + linkSync(join(hardlinkA, 'file.bin'), join(hardlinkB, 'file.bin')); + + const keepFile = join(home, 'model-tidy.keep'); + writeFileSync(keepFile, `${keepRoot}\n`); + + return { home, keepFile, idleRoot, recentRoot, keepRoot, hardlinkA, hardlinkB }; +} + +const noProcessUsers = () => ({ checked: true, users: [] }); +const dockerNotInUse = () => ({ available: true, inUse: false, containers: [] }); +const fixedDiskFree = () => 100 * 2 ** 30; + +describe('discoverCandidates', () => { + it('finds hf-cache, models-dir entries', () => { + const { home } = buildFixture(); + const candidates = discoverCandidates(home); + const paths = candidates.map(c => c.path); + assert.ok(paths.some(p => p.endsWith('models--idle-model'))); + assert.ok(paths.some(p => p.endsWith('models--recent-model'))); + assert.ok(paths.some(p => p.endsWith('models--already-tidied'))); + assert.ok(paths.some(p => p.endsWith('keep-me'))); + assert.ok(paths.some(p => p.endsWith('hardlink-a'))); + assert.ok(paths.some(p => p.endsWith('hardlink-b'))); + }); +}); + +describe('loadKeepList', () => { + it('parses non-blank, non-comment lines', () => { + const dir = tempDir('model-tidy-keep-'); + const f = join(dir, 'keep'); + writeFileSync(f, '# comment\n\n/some/path\n~/models/x\n'); + const entries = loadKeepList(f, '/home/petrus'); + assert.deepEqual(entries, ['/some/path', '/home/petrus/models/x']); + }); + + it('returns [] for a missing file', () => { + assert.deepEqual(loadKeepList('/no/such/file', '/home/petrus'), []); + }); +}); + +describe('computeHardlinkGroups', () => { + it('groups candidates that share an inode across different roots', () => { + const { home, hardlinkA, hardlinkB, idleRoot } = buildFixture(); + const candidates = [ + { id: hardlinkA, path: hardlinkA }, + { id: hardlinkB, path: hardlinkB }, + { id: idleRoot, path: idleRoot } + ]; + const { groupOf } = computeHardlinkGroups(candidates); + assert.equal(groupOf(hardlinkA), groupOf(hardlinkB)); + assert.notEqual(groupOf(hardlinkA), groupOf(idleRoot)); + }); +}); + +describe('planRun', () => { + it('selects exactly the idle dir and the complete hardlink set, with correct skip reasons for everything else', () => { + const { home, keepFile, idleRoot, recentRoot, keepRoot, hardlinkA, hardlinkB } = buildFixture(); + + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + + const selectedPaths = plan.selected.map(r => r.path).sort(); + assert.deepEqual(selectedPaths, [hardlinkA, hardlinkB, idleRoot].sort()); + + const byPath = new Map(plan.skipped.map(r => [r.path, r])); + + const tidied = [...byPath.keys()].find(p => p.endsWith('models--already-tidied')); + assert.ok(tidied, 'already-tidied dir should be in skipped list'); + assert.match(byPath.get(tidied).reason, /already a symlink/); + + const recent = [...byPath.keys()].find(p => p === recentRoot); + assert.ok(recent, 'recent dir should be in skipped list'); + assert.match(byPath.get(recent).reason, /min-idle-days/); + + const keep = [...byPath.keys()].find(p => p === keepRoot); + assert.ok(keep, 'keep-listed dir should be in skipped list'); + assert.match(byPath.get(keep).reason, /KEEP list/); + + assert.equal(plan.selected.length, 3); + assert.equal(plan.skipped.length, 3); + }); + + it('skips a whole hardlink set if either member is in use', () => { + const { home, keepFile, hardlinkA, hardlinkB } = buildFixture(); + // Note: touching hardlinkB's file.bin mtime would also move hardlinkA's + // mtime, since they are literally the same inode — that's not a useful + // way to make just one member fail independently. Use a per-path check + // (process-in-use) instead, which is genuinely independent per path. + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: (path) => (path === hardlinkB + ? { checked: true, users: [{ pid: '9999', via: 'fd', target: join(path, 'file.bin') }] } + : { checked: true, users: [] }), + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + + const selectedPaths = plan.selected.map(r => r.path); + assert.ok(!selectedPaths.includes(hardlinkA)); + assert.ok(!selectedPaths.includes(hardlinkB)); + const skippedA = plan.skipped.find(r => r.path === hardlinkA); + assert.match(skippedA.reason, /hardlinked to .*which is not moving/); + assert.match(skippedA.reason, /in use/); + const skippedB = plan.skipped.find(r => r.path === hardlinkB); + assert.match(skippedB.reason, /in use: pid 9999/); + }); + + it('fails closed for EVERY candidate on the box when docker is unreadable, not just the HF cache', () => { + // Regression test for the fail-open defect: an unreadable docker used + // to only fail-safe ~/.cache/huggingface, leaving ~/models/* and ad-hoc + // dirs selectable even though a container could bind-mount anything. + const { home, keepFile, idleRoot, recentRoot, keepRoot, hardlinkA } = buildFixture(); + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: () => ({ available: false, inUse: false, reason: 'permission denied' }), + diskFreeBytes: fixedDiskFree + }); + + assert.equal(plan.selected.length, 0, 'nothing should be selected while docker is unreadable'); + + const skippedIdle = plan.skipped.find(r => r.path === idleRoot); + assert.ok(skippedIdle); + assert.match(skippedIdle.reason, /in-use status unverified: docker not readable/); + + // The bug: hardlinkA lives under ~/models, NOT under ~/.cache/huggingface. + // It must ALSO be unverified now, box-wide. + const skippedHardlinkA = plan.skipped.find(r => r.path === hardlinkA); + assert.ok(skippedHardlinkA); + assert.match(skippedHardlinkA.reason, /unverified/); + + // Rule priority is unchanged: KEEP list still wins before the docker + // check is ever reached. + const skippedKeep = plan.skipped.find(r => r.path === keepRoot); + assert.match(skippedKeep.reason, /on KEEP list/); + + assert.match(plan.summaryLine, /\d+ unverified/); + assert.ok(plan.unverifiedCount >= 3, `expected several unverified candidates, got ${plan.unverifiedCount}`); + }); + + it('positive control: with a readable process list and docker, the idle dir is still selected', () => { + const { home, keepFile, idleRoot } = buildFixture(); + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: () => ({ checked: true, users: [] }), + listDockerBindUsers: () => ({ available: true, inUse: false, containers: [] }), + diskFreeBytes: fixedDiskFree + }); + const selectedIdle = plan.selected.find(r => r.path === idleRoot); + assert.ok(selectedIdle, 'idle dir should be selected when everything is verifiably readable and idle'); + assert.equal(plan.unverifiedCount, 0); + }); + + it('fails closed when the process check cannot be fully completed (e.g. EACCES on one pid)', () => { + const home = tempDir('model-tidy-unverified-'); + const idleA = join(home, 'models', 'idle-a'); + const idleB = join(home, 'models', 'idle-b'); + writeFile(join(idleA, 'weights.gguf'), 'a'.repeat(2048)); + writeFile(join(idleB, 'weights.gguf'), 'b'.repeat(2048)); + touch(join(idleA, 'weights.gguf'), daysAgo(30)); + touch(join(idleB, 'weights.gguf'), daysAgo(30)); + + const plan = planRun({ + home, + minIdleDays: 14, + listProcessUsers: () => ({ + checked: false, + users: [], + note: 'could not read /proc for pid(s) 4242 (permission denied) — cannot rule out those processes using this path' + }), + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + + assert.equal(plan.selected.length, 0, 'nothing should be selected when process status cannot be verified'); + assert.equal(plan.skipped.length, 2); + for (const r of plan.skipped) { + assert.match(r.reason, /in-use status unverified: could not read \/proc/); + } + assert.match(plan.summaryLine, /2 unverified/); + assert.equal(plan.unverifiedCount, 2); + }); + + it('skips a dir with an open file handle (simulated process check)', () => { + const { home, keepFile, idleRoot } = buildFixture(); + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: (path) => (path === idleRoot + ? { checked: true, users: [{ pid: '4242', via: 'fd', target: join(path, 'blobs/abc123') }] } + : { checked: true, users: [] }), + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + const skipped = plan.skipped.find(r => r.path === idleRoot); + assert.ok(skipped); + assert.match(skipped.reason, /in use: pid 4242/); + }); + + it('skips a dir bind-mounted into a running container', () => { + const { home, keepFile, idleRoot } = buildFixture(); + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: (path) => (path === idleRoot + ? { available: true, inUse: true, containers: [{ container: 'vllm-serve', source: path, destination: '/root/.cache/huggingface' }] } + : { available: true, inUse: false, containers: [] }), + diskFreeBytes: fixedDiskFree + }); + const skipped = plan.skipped.find(r => r.path === idleRoot); + assert.ok(skipped); + assert.match(skipped.reason, /bind-mounted into running container vllm-serve/); + }); + + it('caps total moved bytes by --max-gb, deferring smaller units', () => { + const { home, keepFile } = buildFixture(); + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + maxGb: 0, // too small for anything + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + assert.equal(plan.selected.length, 0); + const deferred = plan.skipped.filter(r => /max-gb.*cap/.test(r.reason)); + assert.ok(deferred.length > 0); + }); + + it('skips an idle dir hardlinked to a KEEP-listed dir, even though both are otherwise idle-eligible', () => { + // Regression test for hardlink-scan scope: the KEEP-listed copy lives + // outside the plain "idle" candidate set (it's filtered out by the + // KEEP rule), so the hardlink group it belongs to must still be + // detected and the whole group skipped — never just the KEEP-listed + // half, which would leave the idle-looking half free to move and + // silently double disk usage. + const { home, keepFile, idleRoot, keepRoot } = buildFixture(); + // Make idleRoot's blob and keepRoot's file share an inode. + const idleBlobDir = join(idleRoot, 'blobs'); + const idleBlobName = readdirSync(idleBlobDir)[0]; + rmSync(join(keepRoot, 'weights.gguf')); + linkSync(join(idleBlobDir, idleBlobName), join(keepRoot, 'weights.gguf')); + + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + + const selectedPaths = plan.selected.map(r => r.path); + assert.ok(!selectedPaths.includes(idleRoot), 'idle dir must not move while hardlinked to a KEEP-listed copy'); + + const skippedIdle = plan.skipped.find(r => r.path === idleRoot); + assert.ok(skippedIdle); + assert.match(skippedIdle.reason, /hardlinked to .*which is not moving/); + assert.match(skippedIdle.reason, /on KEEP list/); + }); + + it('GAP 1 regression: refuses a candidate hardlinked to a file OUTSIDE every discovered root (codexmb probe, reproduced exactly)', () => { + // Exact reproduction of the review's probe.mjs: link placed directly + // under the fixture root, ABOVE home/models, so no discovered + // candidate root ever contains that second path — scanning wider + // cannot see it. Only comparing st_nlink against what we actually + // observed can catch this. + const root = tempDir('model-tidy-gap1-'); + const home = join(root, 'home'); + const src = join(home, 'models', 'idle'); + mkdirSync(src, { recursive: true }); + writeFileSync(join(src, 'weights.gguf'), 'fixture-only'); + utimesSync(join(src, 'weights.gguf'), new Date(0), new Date(0)); + linkSync(join(src, 'weights.gguf'), join(root, 'outside-discovery.gguf')); + + const plan = planRun({ + home, + listProcessUsers: () => ({ checked: true, users: [] }), + listDockerBindUsers: () => ({ available: true, inUse: false, containers: [] }), + diskFreeBytes: () => 0 + }); + + assert.ok(!plan.selected.some(r => r.path === src), 'must not select a candidate with an invisible extra hardlink'); + const skipped = plan.skipped.find(r => r.path === src); + assert.ok(skipped); + assert.match(skipped.reason, /hardlinked 2 times, only 1 links found under scanned roots; refusing incomplete set/); + }); + + it('GAP 1 positive control: an idle dir whose hardlinks are ALL inside the scanned tree is still selected as a unit', () => { + const { home, keepFile, hardlinkA, hardlinkB } = buildFixture(); + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + const selectedPaths = plan.selected.map(r => r.path); + assert.ok(selectedPaths.includes(hardlinkA)); + assert.ok(selectedPaths.includes(hardlinkB)); + }); +}); + +// tmpdir()-based fixtures and their "target" dir are normally on the SAME +// filesystem on a dev box, so validateTarget()'s cross-filesystem refusal +// (tested for real below, with no mocking) would otherwise block every +// apply test here. The apply-flow tests inject a bypass for that one check +// so they can exercise copy/verify/symlink-swap in isolation; the +// cross-filesystem rule itself is covered by its own unmocked test. +const bypassCrossFsCheck = () => ({ ok: true }); + +describe('validateTarget', () => { + it('refuses a target on the same filesystem as the source (real check, no mocking)', () => { + const home = tempDir('model-tidy-home-'); + const target = tempDir('model-tidy-target-'); // same tmpfs as home on a dev box + const result = validateTarget(target, home); + assert.equal(result.ok, false); + assert.match(result.error, /same filesystem/); + }); + + it('refuses a target that does not exist', () => { + const home = tempDir('model-tidy-home-'); + const result = validateTarget(join(home, 'nonexistent'), home); + assert.equal(result.ok, false); + }); +}); + +describe('applyRun', () => { + it('copies, verifies, and replaces the source with a symlink for every selected unit', () => { + const { home, keepFile, idleRoot, hardlinkA, hardlinkB } = buildFixture(); + const target = tempDir('model-tidy-target-'); + + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + + const result = applyRun({ plan, target, home, validateTarget: bypassCrossFsCheck }); + + assert.equal(result.ok, true, JSON.stringify(result.errors)); + assert.equal(result.moved.length, 2); // idle-model unit + hardlink unit + + for (const root of [idleRoot, hardlinkA, hardlinkB]) { + assert.ok(lstatSync(root).isSymbolicLink(), `${root} should now be a symlink`); + } + + // Content is reachable through the symlink and byte-identical. + const blobViaSymlink = readdirSync(join(idleRoot, 'blobs'))[0]; + assert.ok(existsSync(join(idleRoot, 'blobs', blobViaSymlink))); + + // The hardlink relationship survived the move: both copies at the + // target still share one inode (moving them independently would have + // doubled disk usage, which is exactly what this guards against). + const aStat = lstatSync(join(hardlinkA, 'file.bin')); + const bStat = lstatSync(join(hardlinkB, 'file.bin')); + assert.equal(aStat.ino, bStat.ino); + assert.ok(aStat.nlink >= 2); + }); + + it('negative control: a corrupted target file makes apply refuse and leaves the source untouched', () => { + const { home, keepFile, idleRoot } = buildFixture(); + const target = tempDir('model-tidy-target-'); + + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + // Isolate to just the idle-model unit for a focused assertion. + plan.selected = plan.selected.filter(r => r.path === idleRoot); + + // Copy for real, then simulate corruption discovered after the copy + // (e.g. a bad transfer / bitrot) by overwriting the blob at the target. + function corruptAfterCopy(sourceAbsPaths, homeDir, targetRoot) { + copyUnitPureNode(sourceAbsPaths, homeDir, targetRoot); + for (const src of sourceAbsPaths) { + const rel = src.slice(homeDir.length + 1); + const targetBlobsDir = join(targetRoot, rel, 'blobs'); + if (existsSync(targetBlobsDir)) { + const [blobName] = readdirSync(targetBlobsDir); + const original = readFileSync(join(targetBlobsDir, blobName), 'utf8'); + // Same length as the original so this exercises the checksum + // comparison specifically, not just the (cheaper) size check. + writeFileSync(join(targetBlobsDir, blobName), 'X'.repeat(original.length)); + } + } + } + + const result = applyRun({ + plan, + target, + home, + validateTarget: bypassCrossFsCheck, + copyFn: corruptAfterCopy + }); + + assert.equal(result.ok, false); + assert.equal(result.moved.length, 0); + assert.equal(result.errors.length, 1); + assert.equal(result.errors[0].step, 'verify'); + assert.ok(result.errors[0].mismatches.some(m => /checksum mismatch/.test(m.reason))); + + // Source must be completely untouched: still a real directory, not a + // symlink, with its original (uncorrupted) content intact. + assert.equal(lstatSync(idleRoot).isSymbolicLink(), false); + assert.ok(existsSync(idleRoot)); + const blobDir = join(idleRoot, 'blobs'); + const [blobName] = readdirSync(blobDir); + const content = readFileSync(join(blobDir, blobName), 'utf8'); + assert.equal(content, 'idle-model-bytes'.repeat(100)); + }); + + it('crash-window negative control: if symlink creation fails after the source was staged, the source is restored byte-identical', () => { + const { home, keepFile, idleRoot } = buildFixture(); + const target = tempDir('model-tidy-target-'); + + const originalBlobName = readdirSync(join(idleRoot, 'blobs'))[0]; + const originalContent = readFileSync(join(idleRoot, 'blobs', originalBlobName), 'utf8'); + + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + plan.selected = plan.selected.filter(r => r.path === idleRoot); + + // Inject a failing symlink function (the same seam applyRun exposes + // for copyFn/verifyFn/validateTarget) so swapToSymlink's real call + // fails right after the real rename has already happened — the exact + // crash window the fix closes. + let callCount = 0; + const failingSymlinkSync = () => { + callCount++; + throw new Error('simulated symlink failure (disk full / EIO / etc.)'); + }; + + const result = applyRun({ plan, target, home, validateTarget: bypassCrossFsCheck, symlinkFn: failingSymlinkSync }); + assert.equal(result.ok, false); + assert.equal(result.moved.length, 0); + assert.equal(result.errors.length, 1); + assert.equal(result.errors[0].step, 'swap'); + assert.ok(callCount > 0, 'the injected failing symlinkSync should have been invoked'); + + // The source must be back at its original path, not a symlink, and + // byte-identical — never left staged, never left half-swapped. + assert.equal(existsSync(idleRoot), true); + assert.equal(lstatSync(idleRoot).isSymbolicLink(), false); + assert.equal(existsSync(idleRoot + '.tidy-moving'), false); + const restoredContent = readFileSync(join(idleRoot, 'blobs', originalBlobName), 'utf8'); + assert.equal(restoredContent, originalContent); + }); + + it("refuses a unit with an UNJOURNALED leftover *.tidy-moving, without touching it (apply's own recovery pass only heals journaled units)", () => { + const { home, keepFile, idleRoot } = buildFixture(); + const target = tempDir('model-tidy-target-'); + const staging = `${idleRoot}.tidy-moving`; + + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + plan.selected = plan.selected.filter(r => r.path === idleRoot); + + // Simulate a leftover with NO journal record at all (e.g. a directory + // that just happens to be named like one of ours, or a journal file + // that was separately lost). Recovery must never guess about this — + // it should be reported and left alone, and apply's own pre-check + // should then refuse the unit outright. + mkdirSync(staging, { recursive: true }); + writeFileSync(join(staging, 'marker'), 'leftover-from-a-previous-crash'); + tempPaths.push(staging); + + let copyCalled = false; + const result = applyRun({ + plan, + target, + home, + validateTarget: bypassCrossFsCheck, + copyFn: () => { copyCalled = true; } + }); + + assert.equal(result.ok, false); + assert.equal(result.moved.length, 0); + assert.equal(result.errors.length, 1); + assert.equal(result.errors[0].step, 'pre-check'); + assert.match(result.errors[0].error, /leftover from a previous run/); + assert.match(result.errors[0].error, /recover/); + assert.equal(copyCalled, false, 'apply must refuse before even attempting to copy this unit'); + + // Neither the original nor the leftover staging dir were touched. + assert.ok(existsSync(idleRoot)); + assert.equal(lstatSync(idleRoot).isSymbolicLink(), false); + assert.ok(existsSync(staging)); + assert.equal(readFileSync(join(staging, 'marker'), 'utf8'), 'leftover-from-a-previous-crash'); + }); +}); + +describe('GAP 2: crash-safety across a hard process kill (not just a thrown exception)', () => { + function buildSingleIdleFixture() { + const home = tempDir('model-tidy-gap2-home-'); + const src = join(home, 'models', 'idle'); + mkdirSync(src, { recursive: true }); + writeFileSync(join(src, 'weights.gguf'), 'gap2-fixture-bytes'.repeat(50)); + utimesSync(join(src, 'weights.gguf'), daysAgo(30), daysAgo(30)); + return { home, src }; + } + + function planFor(home) { + return planRun({ + home, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + } + + it("codexmb probe, kept verbatim: a hard crash during the symlink step leaves the original intact, and the invariant holds after one recovery pass", () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + assert.ok(plan.selected.some(r => r.path === src)); + + const child = runApplyInChildWithCrash(plan, home, target, 'symlinkFn'); + assert.equal(child.status, 77); + + recoverInterruptedMoves(home); + + // codexmb's probe assertion, kept verbatim in spirit: after the crash + // and one recovery pass, either the original path exists, or a + // working symlink exists at src. Never neither. + let srcIsWorkingSymlink = false; + try { + srcIsWorkingSymlink = lstatSync(src).isSymbolicLink() && existsSync(src); + } catch { + srcIsWorkingSymlink = false; + } + const originalPathExists = existsSync(src) && !srcIsWorkingSymlink; + assert.ok(originalPathExists || srcIsWorkingSymlink, 'neither the original nor a working symlink exists at src'); + + // This specific crash point happens before any rename, so nothing + // should have moved at all: the original itself, untouched. + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), false); + assert.equal(existsSync(src + '.tidy-moving'), false); + assert.equal(existsSync(src + '.tidy-link'), false); + }); + + it('a hard crash between the two renames is healed by completing the pending symlink swap', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterStage'); + assert.equal(child.status, 77); + // Mid-crash snapshot: src renamed away, symlink not yet swapped in. + assert.equal(existsSync(src), false); + assert.ok(existsSync(`${src}.tidy-moving`)); + + const recovered = recoverInterruptedMoves(home); + assert.ok(recovered.some(r => r.path === src && r.action === 'completed-swap-and-cleaned'), JSON.stringify(recovered)); + + assert.equal(existsSync(src), true, 'src must exist after recovery'); + assert.equal(lstatSync(src).isSymbolicLink(), true, 'src must be a working symlink after recovery'); + assert.equal(existsSync(`${src}.tidy-moving`), false, 'staged original should be fully cleaned up'); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'gap2-fixture-bytes'.repeat(50)); + }); + + it('a hard crash after the second rename (before cleanup) is healed by finishing the cleanup', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterSwap'); + assert.equal(child.status, 77); + // Mid-crash snapshot: the swap already completed structurally — src is + // already a working symlink — only the old original's cleanup is + // pending. + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), true); + assert.ok(existsSync(`${src}.tidy-moving`)); + + const recovered = recoverInterruptedMoves(home); + assert.ok(recovered.some(r => r.path === src && r.action === 'completed-swap-and-cleaned'), JSON.stringify(recovered)); + + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), true); + assert.equal(existsSync(`${src}.tidy-moving`), false, 'staged original should be cleaned up after recovery'); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'gap2-fixture-bytes'.repeat(50)); + }); + + it('recovery is idempotent and a no-op on a healthy tree', () => { + const { home } = buildSingleIdleFixture(); + const first = recoverInterruptedMoves(home); + const second = recoverInterruptedMoves(home); + assert.deepEqual(first, []); + assert.deepEqual(second, []); + }); + + it('a hard crash right after the link is verified, before either rename, is healed with the original untouched', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterLink'); + assert.equal(child.status, 77); + assert.equal(existsSync(src), true, 'mid-crash: src untouched, no rename has happened yet'); + assert.equal(lstatSync(src).isSymbolicLink(), false); + assert.ok(existsSync(`${src}.tidy-link`)); + + const recovered = recoverInterruptedMoves(home); + assert.ok(recovered.some(r => r.path === src && r.action === 'removed-stray-link'), JSON.stringify(recovered)); + + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), false); + assert.equal(existsSync(`${src}.tidy-link`), false); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'gap2-fixture-bytes'.repeat(50)); + }); + + function snapshotTree(root) { + const snap = {}; + function recurse(dir) { + for (const name of readdirSync(dir)) { + const full = join(dir, name); + const lst = lstatSync(full); + if (lst.isSymbolicLink()) { + snap[full] = { type: 'symlink', linkTarget: readlinkSync(full) }; + } else if (lst.isDirectory()) { + recurse(full); + } else if (lst.isFile()) { + snap[full] = { type: 'file', size: lst.size, mtimeMs: lst.mtimeMs, sha256: sha256Sync(full) }; + } + } + } + recurse(root); + return snap; + } + function sha256Sync(path) { + return createHash('sha256').update(readFileSync(path)).digest('hex'); + } + + it("AMENDMENT: plan is read-only — an interrupted move is reported per unit, and every byte on disk (full tree, sizes and mtimes) is untouched by plan", () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const firstPlan = planFor(home); + const child = runApplyInChildWithCrash(firstPlan, home, target, 'afterStage'); + assert.equal(child.status, 77); + assert.equal(existsSync(src), false, 'mid-crash sanity check: src is gone, staged+link exist'); + + const before = snapshotTree(home); + const plan = planFor(home); + const after = snapshotTree(home); + + assert.deepEqual(after, before, 'plan must not change any byte on disk (sizes/mtimes of the full tree)'); + + assert.ok(plan.interrupted.some(f => f.path === src), 'plan must report the interrupted move'); + const finding = plan.interrupted.find(f => f.path === src); + assert.equal(finding.status, 'interrupted'); + assert.match(finding.note, /journal step 'staged'/); + + assert.ok(!plan.selected.some(r => r.path === src), 'the interrupted unit must never be selected'); + const skipped = plan.skipped.find(r => r.path === src); + assert.ok(skipped, 'the interrupted unit must be reported in skipped, not silently dropped'); + assert.match(skipped.reason, /interrupted move found/); + + // Confirm it is really still interrupted (plan really did nothing) — + // recovery, called separately, still has work to do. + const recovered = recoverInterruptedMoves(home); + assert.ok(recovered.some(r => r.path === src)); + }); + + it('AMENDMENT: recovery acts only on journaled units — a directory literally named *.tidy-moving with NO journal record is never touched', () => { + const { home } = buildSingleIdleFixture(); + const strayBase = join(home, 'models', 'someone-elses-thing'); + const stray = `${strayBase}.tidy-moving`; + mkdirSync(stray, { recursive: true }); + writeFileSync(join(stray, 'not-ours.txt'), 'this directory is just named like one of ours, nothing to do with model-tidy'); + tempPaths.push(stray); + const before = readFileSync(join(stray, 'not-ours.txt'), 'utf8'); + + const recovered = recoverInterruptedMoves(home); + + assert.equal(existsSync(stray), true, 'the stray directory must still exist, completely untouched'); + assert.equal(readFileSync(join(stray, 'not-ours.txt'), 'utf8'), before); + const finding = recovered.find(r => r.path === strayBase); + assert.ok(finding, 'the stray must be reported'); + assert.equal(finding.action, 'left-alone'); + assert.match(finding.note, /no matching journal record/); + }); + + it('AMENDMENT / decision-table row 2: recovery never silently discards a staged original whose own content no longer matches the journal — with the live path (target + link) still correct, it quarantines rather than deletes or restores over the good symlink', () => { + // NOTE: this scenario crashes 'afterStage' (between the two renames), + // but since the pending link was already verified BEFORE it was ever + // created, recovery completes that rename first — so by the time + // finalizeSwapOrRestore runs, target and link are both fine. Only the + // staged backup is damaged. That is row 2 of the decision table, not + // row 3: an earlier version of this fix used to restore the tampered + // staged copy over the good live symlink here, which was itself a + // (milder) data-loss bug — see the round-3 fix. + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterStage'); + assert.equal(child.status, 77); + const staging = `${src}.tidy-moving`; + assert.ok(existsSync(staging)); + const goodContent = 'gap2-fixture-bytes'.repeat(50); + + // Tamper with the staged original so it no longer matches what the + // journal recorded — recovery must not trust its own naming + // convention over the manifest, and must not silently throw this + // away just because a symlink elsewhere happens to resolve. + writeFileSync(join(staging, 'weights.gguf'), 'TAMPERED-CONTENT-DOES-NOT-MATCH-MANIFEST'); + + const recovered = recoverInterruptedMoves(home); + const finding = recovered.find(r => r.path === src); + assert.ok(finding); + assert.equal(finding.action, 'completed-partial-staging-quarantined'); + assert.match(finding.note, /staged original was damaged/); + + // The live path must be preserved exactly as it was: a working + // symlink to the good target. + assert.equal(lstatSync(src).isSymbolicLink(), true, 'the live symlink must be preserved, not replaced with the tampered staged copy'); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), goodContent); + + // The damaged staging must not have been silently deleted: it must + // now live at a quarantine path, still holding the tampered content + // for a human to inspect — never lost. + assert.equal(existsSync(staging), false, 'the staging path itself is gone (renamed to quarantine)'); + const quarantineDirs = readdirSync(join(home, 'models')).filter(n => n.includes('.tidy-quarantine-')); + assert.equal(quarantineDirs.length, 1); + const quarantinePath = join(home, 'models', quarantineDirs[0]); + assert.equal(readFileSync(join(quarantinePath, 'weights.gguf'), 'utf8'), 'TAMPERED-CONTENT-DOES-NOT-MATCH-MANIFEST'); + }); + + it("DATA-LOSS FIX, codexmb's round-2 probe kept verbatim: target corrupted after staging must not cost the last good original", () => { + // Direct port of /tmp/codex-pr128-round2.pjlier/probe.mjs: a thrown + // exception (not a hard kill) during afterStage interrupts the swap + // with the original safely staged; the target is then corrupted + // in-place; recovery must read GOOD from the source afterwards, never + // BADD, and must never report success for a unit whose target was + // corrupted. + const home = tempDir('model-tidy-probe2-home-'); + const src = join(home, 'models', 'idle'); + const target = tempDir('model-tidy-probe2-target-'); + mkdirSync(src, { recursive: true }); + writeFileSync(join(src, 'weights.gguf'), 'GOOD'); + utimesSync(join(src, 'weights.gguf'), new Date(0), new Date(0)); + + const plan = planRun({ + home, + listProcessUsers: () => ({ checked: true, users: [] }), + listDockerBindUsers: () => ({ available: true, inUse: false, containers: [] }), + diskFreeBytes: () => 0 + }); + + const interrupted = applyRun({ + plan, target, home, + validateTarget: () => ({ ok: true }), + afterStage: () => { throw new Error('fixture interruption'); } + }); + assert.equal(interrupted.ok, false, 'apply must report failure for the interrupted unit'); + + // Corrupt the target after the interruption, exactly like the probe. + writeFileSync(join(target, 'models', 'idle', 'weights.gguf'), 'BADD'); + + const recovery = recoverInterruptedMoves(home); + + const sourceBytes = readFileSync(join(src, 'weights.gguf'), 'utf8'); + const stagedOriginalExists = existsSync(`${src}.tidy-moving`); + + // codexmb's exact probe assertions: source reads GOOD (never BADD), + // and the unit is reported failed with the target flagged. + assert.equal(sourceBytes, 'GOOD', 'source must read the last GOOD content, never the corrupted target content'); + // "staged original either restored to the source path or retained" — + // both are acceptable; what's NOT acceptable is stagedOriginalExists + // being false while sourceBytes came out wrong. Since sourceBytes is + // confirmed GOOD above, either outcome for stagedOriginalExists is + // fine as long as the unit was reported as failed, checked next. + void stagedOriginalExists; + + assert.equal(recovery.length, 1); + assert.equal(recovery[0].path, src); + assert.notEqual(recovery[0].action, 'completed-swap-and-cleaned', 'must never report success for a corrupted target'); + assert.match(recovery[0].note, /target content does not match the journaled manifest/); + + // Running recovery again must not somehow make it worse (idempotent + // refusal, not idempotent data loss). + const secondPass = recoverInterruptedMoves(home); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'GOOD'); + }); + + it('mirror case: target is fine but the source symlink points elsewhere — recover must not delete the staged original', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterSwap'); + assert.equal(child.status, 77); + // Mid-crash: swap already completed correctly (src is a symlink to + // the real target), staged original still pending cleanup. + assert.equal(lstatSync(src).isSymbolicLink(), true); + const staging = `${src}.tidy-moving`; + assert.ok(existsSync(staging)); + const goodContent = readFileSync(join(src, 'weights.gguf'), 'utf8'); + + // Simulate tampering: replace the correct symlink with one pointing + // somewhere unrelated. The target itself, and the staged original, + // are both still completely fine. + const decoy = tempDir('model-tidy-gap2-decoy-'); + writeFileSync(join(decoy, 'weights.gguf'), 'DECOY-NOT-THE-REAL-TARGET'); + rmSync(src); + symlinkSync(decoy, src); + + const recovered = recoverInterruptedMoves(home); + assert.equal(recovered.length, 1, 'exactly one journaled unit is in play here'); + const match = recovered[0]; + assert.notEqual(match.action, 'completed-swap-and-cleaned', 'must never delete the staged original on the strength of an unrelated link resolving'); + assert.match(match.note, /source is not a symlink whose realpath resolves exactly to the recorded target path/); + + // The staged original must not have been discarded: it's back at src + // (restored) or still sitting at the staging path — never gone. + const restoredAtSrc = existsSync(src) && !lstatSync(src).isSymbolicLink() && readFileSync(join(src, 'weights.gguf'), 'utf8') === goodContent; + const stillStaged = existsSync(staging) && readFileSync(join(staging, 'weights.gguf'), 'utf8') === goodContent; + assert.ok(restoredAtSrc || stillStaged, 'the good staged original must survive, either restored to src or left at the staging path'); + }); + + it('positive control: target and symlink both correct — recover completes the swap and cleans up', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterStage'); + assert.equal(child.status, 77); + + const recovered = recoverInterruptedMoves(home); + assert.equal(recovered.length, 1); + assert.equal(recovered[0].action, 'completed-swap-and-cleaned'); + + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), true); + assert.equal(existsSync(`${src}.tidy-moving`), false); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'gap2-fixture-bytes'.repeat(50)); + }); + + it("the SAME corrupt-target scenario against apply's OWN final cleanup path, not only recover", () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + const targetFile = join(target, 'models', 'idle', 'weights.gguf'); + + const result = applyRun({ + plan, + target, + home, + validateTarget: bypassCrossFsCheck, + afterSwap: () => { + // Corrupt the target in the SAME run, right after the second + // rename but before apply's own finalize step runs. + writeFileSync(targetFile, 'CORRUPTED-DURING-THE-SAME-APPLY-RUN'); + } + }); + + assert.equal(result.ok, false, 'apply must report failure for this unit rather than silently succeed'); + assert.equal(result.moved.length, 0); + assert.equal(result.errors.length, 1); + assert.equal(result.errors[0].step, 'swap'); + assert.match(result.errors[0].error, /target content does not match the journaled manifest/); + + // The last good original must not have been lost: either restored to + // src as a real directory, or still present at the staging path. + const restoredAtSrc = existsSync(src) && !lstatSync(src).isSymbolicLink(); + const stillStaged = existsSync(`${src}.tidy-moving`); + assert.ok(restoredAtSrc || stillStaged, 'the good original must survive apply refusing to finalize onto a corrupted target'); + if (restoredAtSrc) { + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'gap2-fixture-bytes'.repeat(50)); + } + }); + + it("decision-table row 3, reproducing codexmb's round-3 probe's own 'bad-target' scenario (afterSwap crash, not afterStage): a corrupted target still triggers restore-from-staged, never quarantine", () => { + // Exact mechanics of round-3's probe: crash at afterSwap (src is + // ALREADY a symlink, staged still pending cleanup), then corrupt the + // target's content. linkOk only checks that the symlink's realpath + // equals the recorded target PATH — it does not, and must not, imply + // the target's CONTENT is still correct. targetOk must independently + // fail here, landing on row 3 (restore), not row 1 or row 2. + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterSwap'); + assert.equal(child.status, 77); + assert.equal(lstatSync(src).isSymbolicLink(), true, 'mid-crash: the swap already completed structurally'); + const staging = `${src}.tidy-moving`; + assert.ok(existsSync(staging)); + + writeFileSync(join(target, 'models', 'idle', 'weights.gguf'), 'BADD-TARGET-CONTENT'); + + const recovered = recoverInterruptedMoves(home); + const finding = recovered.find(r => r.path === src); + assert.ok(finding); + assert.equal(finding.action, 'restored-after-failed-verification'); + assert.match(finding.note, /target content does not match the journaled manifest/); + + assert.equal(lstatSync(src).isSymbolicLink(), false, 'row 3 restores the real directory, it does not quarantine or preserve a symlink to a corrupted target'); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'gap2-fixture-bytes'.repeat(50)); + assert.equal(existsSync(staging), false); + }); + + it('decision-table row 4: nothing verified good anywhere (target corrupted AND staging partial) — recovery deletes nothing, renames nothing, and leaves every path exactly as found', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterStage'); + assert.equal(child.status, 77); + const staging = `${src}.tidy-moving`; + assert.ok(existsSync(staging)); + + // Damage BOTH copies: the target (live) and the staged backup. + writeFileSync(join(target, 'models', 'idle', 'weights.gguf'), 'BADD-TARGET'); + rmSync(join(staging, 'weights.gguf')); + + // Snapshot only the DATA paths (source tree + target tree), not + // model-tidy's own journal bookkeeping under home/.cache — the + // journal is EXPECTED to record this failure (step + // 'failed-both-copies-damaged'); what must stay byte-for-byte + // unchanged is the actual model data. + const snapshotData = () => ({ ...snapshotTree(join(home, 'models')), ...snapshotTree(target) }); + + const before = snapshotData(); + const recovered = recoverInterruptedMoves(home); + const after = snapshotData(); + + assert.deepEqual(after, before, 'row 4 must delete nothing and rename nothing — every data path (sizes and content hashes) must be byte-for-byte unchanged'); + + const finding = recovered.find(r => r.path === src); + assert.ok(finding); + assert.equal(finding.action, 'left-alone-nothing-verified-good'); + assert.match(finding.note, /target content does not match the journaled manifest/); + assert.match(finding.note, /no staged original exists to fall back on|staged original itself no longer matches/); + assert.match(finding.note, new RegExp(src.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')), 'the report must name the affected paths so a human can recover by hand'); + + // A second recovery pass must be equally inert — refusing is stable, + // not just a one-time fluke. + const secondPass = recoverInterruptedMoves(home); + assert.deepEqual(snapshotData(), before); + assert.equal(secondPass.find(r => r.path === src).action, 'left-alone-nothing-verified-good'); + }); + + function journalDirFor(home) { + return join(home, '.cache', 'ide-agent-kit', 'model-tidy-journal'); + } + function findJournalFile(home) { + const dir = journalDirFor(home); + const names = readdirSync(dir).filter(n => n.endsWith('.json')); + assert.equal(names.length, 1, `expected exactly one journal file, found ${names.length}: ${names.join(', ')}`); + return join(dir, names[0]); + } + function snapshotData(home, target) { + return { ...snapshotTree(join(home, 'models')), ...snapshotTree(target) }; + } + + describe('BLOCKER 1: never recursive-delete a path assumed to be a symlink (unlinkOwnedSymlink)', () => { + it('site 1 (finalizeSwapOrRestore row 3, sourcePath obstacle): a REAL DIRECTORY with a sentinel file sitting where a symlink was expected is never touched', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + // afterSwap: src is already a symlink, staged still pending cleanup. + const child = runApplyInChildWithCrash(plan, home, target, 'afterSwap'); + assert.equal(child.status, 77); + assert.equal(lstatSync(src).isSymbolicLink(), true); + const staging = `${src}.tidy-moving`; + assert.ok(existsSync(staging)); + + // Force row 3/4 by corrupting the target, THEN replace src (the + // symlink) with a real directory holding a sentinel file — as if + // something else had put a real directory exactly where model-tidy + // expected to find (and remove) its own symlink. + writeFileSync(join(target, 'models', 'idle', 'weights.gguf'), 'BADD'); + rmSync(src); + mkdirSync(src, { recursive: true }); + writeFileSync(join(src, 'SENTINEL.txt'), 'do not delete me'); + + const before = snapshotData(home, target); + const recovered = recoverInterruptedMoves(home); + const after = snapshotData(home, target); + + assert.equal(existsSync(join(src, 'SENTINEL.txt')), true, 'the sentinel file must survive'); + assert.equal(readFileSync(join(src, 'SENTINEL.txt'), 'utf8'), 'do not delete me'); + assert.deepEqual(after, before, 'nothing on disk may change when a real directory sits where a symlink was expected'); + + const finding = recovered.find(r => r.path === src); + assert.ok(finding); + assert.notEqual(finding.action, 'completed-swap-and-cleaned'); + assert.notEqual(finding.action, 'completed-partial-staging-quarantined'); + }); + + it('site 2 (finalizeSwapOrRestore row 3, pending-link obstacle): a REAL DIRECTORY with a sentinel file at the .tidy-link path is refused, not recursively removed', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + // afterStage: src missing, staged + pending link both present. + const child = runApplyInChildWithCrash(plan, home, target, 'afterStage'); + assert.equal(child.status, 77); + const staging = `${src}.tidy-moving`; + const link = `${src}.tidy-link`; + assert.ok(existsSync(staging)); + assert.equal(lstatSync(link).isSymbolicLink(), true); + + // Replace the verified pending link with a real directory + sentinel. + rmSync(link); + mkdirSync(link, { recursive: true }); + writeFileSync(join(link, 'SENTINEL.txt'), 'do not delete me'); + + const before = snapshotData(home, target); + const recovered = recoverInterruptedMoves(home); + const after = snapshotData(home, target); + + assert.equal(existsSync(join(link, 'SENTINEL.txt')), true, 'the sentinel file must survive'); + assert.equal(readFileSync(join(link, 'SENTINEL.txt'), 'utf8'), 'do not delete me'); + assert.deepEqual(after, before, 'nothing on disk may change when a real directory sits where the pending link was expected'); + assert.equal(existsSync(staging), true, 'the staged original must still be there too — nothing was renamed'); + + const finding = recovered.find(r => r.path === src); + assert.ok(finding); + assert.match(finding.note, /found a directory instead|expected an owned symlink/); + }); + + it('site 3 (recoverInterruptedMoves stray-link branch): a REAL DIRECTORY with a sentinel file at the .tidy-link path is refused, original untouched source survives', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + // afterLink: src is STILL the real, untouched original; only a + // verified pending link exists so far (crashed before the first + // rename), so recovery routes through the "remove stray link, leave + // the never-touched original alone" branch. + const child = runApplyInChildWithCrash(plan, home, target, 'afterLink'); + assert.equal(child.status, 77); + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), false); + const link = `${src}.tidy-link`; + assert.equal(lstatSync(link).isSymbolicLink(), true); + const originalContent = readFileSync(join(src, 'weights.gguf'), 'utf8'); + + rmSync(link); + mkdirSync(link, { recursive: true }); + writeFileSync(join(link, 'SENTINEL.txt'), 'do not delete me'); + + const before = snapshotData(home, target); + const recovered = recoverInterruptedMoves(home); + const after = snapshotData(home, target); + + assert.equal(existsSync(join(link, 'SENTINEL.txt')), true, 'the sentinel file must survive'); + assert.deepEqual(after, before, 'nothing on disk may change'); + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), false); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), originalContent, 'the original, never touched by the crash, must remain exactly as it was'); + + const finding = recovered.find(r => r.path === src); + assert.ok(finding); + assert.equal(finding.action, 'left-alone'); + assert.match(finding.note, /found a directory instead|expected an owned symlink/); + }); + + it('a symlink at the pending-link path pointing somewhere OTHER than the recorded target must be refused, never unlinked', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterStage'); + assert.equal(child.status, 77); + const staging = `${src}.tidy-moving`; + const link = `${src}.tidy-link`; + assert.ok(existsSync(staging)); + assert.equal(lstatSync(link).isSymbolicLink(), true); + + // Re-point the pending link at an unrelated decoy directory instead + // of the journal's recorded target. The target itself stays GOOD. + const decoy = tempDir('model-tidy-gap2-decoy-'); + writeFileSync(join(decoy, 'weights.gguf'), 'DECOY-NOT-THE-REAL-TARGET'); + rmSync(link); + symlinkSync(decoy, link); + + const before = snapshotData(home, target); + const recovered = recoverInterruptedMoves(home); + const after = snapshotData(home, target); + + // The wrong-target symlink itself must survive, unlinked-not: + assert.equal(lstatSync(link).isSymbolicLink(), true); + assert.equal(readlinkSync(link), decoy); + assert.deepEqual(after, before, 'nothing on disk may change when the pending link points to the wrong place'); + assert.equal(existsSync(staging), true, 'the staged original must still be there — nothing was renamed'); + assert.equal(existsSync(src), false, 'src must still be missing — never populated from the wrong-target link'); + + const finding = recovered.find(r => r.path === src); + assert.ok(finding); + assert.match(finding.note, /resolves to .*not the recorded target/); + }); + }); + + describe('BLOCKER 2: journal durability — unreadable journal records are reported and left alone, never acted on', () => { + it('(a) a journal record truncated to HALF its bytes is reported unreadable by plan and recover, and every path is left unchanged', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan1 = planFor(home); + const child = runApplyInChildWithCrash(plan1, home, target, 'afterStage'); + assert.equal(child.status, 77); + + const journalFile = findJournalFile(home); + const original = readFileSync(journalFile, 'utf8'); + writeFileSync(journalFile, original.slice(0, Math.floor(original.length / 2))); + + const before = snapshotData(home, target); + + const plan2 = planFor(home); + const finding = plan2.interrupted.find(f => f.journalFile === journalFile); + assert.ok(finding, 'plan must report the unreadable journal file'); + assert.equal(finding.status, 'journal-unreadable'); + assert.match(finding.note, /truncated|not valid JSON/); + + assert.deepEqual(snapshotData(home, target), before, 'plan must not mutate anything, including for an unreadable journal'); + + const recovered = recoverInterruptedMoves(home); + const recoverFinding = recovered.find(r => r.journalFile === journalFile); + assert.ok(recoverFinding); + assert.equal(recoverFinding.action, 'left-alone'); + assert.match(recoverFinding.note, /unreadable/); + + assert.deepEqual(snapshotData(home, target), before, 'recover must not act on an unreadable journal record either'); + }); + + it('(a) a journal record truncated to ZERO bytes (empty file) is reported unreadable, and every path is left unchanged', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan1 = planFor(home); + const child = runApplyInChildWithCrash(plan1, home, target, 'afterStage'); + assert.equal(child.status, 77); + + const journalFile = findJournalFile(home); + writeFileSync(journalFile, ''); + + const before = snapshotData(home, target); + + const plan2 = planFor(home); + const finding = plan2.interrupted.find(f => f.journalFile === journalFile); + assert.ok(finding); + assert.equal(finding.status, 'journal-unreadable'); + assert.match(finding.note, /empty/); + assert.deepEqual(snapshotData(home, target), before); + + const recovered = recoverInterruptedMoves(home); + const recoverFinding = recovered.find(r => r.journalFile === journalFile); + assert.ok(recoverFinding); + assert.equal(recoverFinding.action, 'left-alone'); + assert.match(recoverFinding.note, /unreadable/); + assert.deepEqual(snapshotData(home, target), before); + }); + + it('(b) a stray .tmp-* journal file (left over from an interrupted journal WRITE) is never parsed as a record, and is reported', () => { + const { home } = buildSingleIdleFixture(); + const journalDir = journalDirFor(home); + mkdirSync(journalDir, { recursive: true }); + const strayTmp = join(journalDir, 'deadbeefdeadbeef.json.tmp-99999-abc123def456'); + writeFileSync(strayTmp, JSON.stringify({ + sourcePath: join(home, 'models', 'not-a-real-unit'), + stagedPath: 'x', linkPath: 'y', targetPath: 'z', manifest: [], step: 'pending' + })); + + const detected = detectInterruptedMoves(home); + const detectedFinding = detected.find(f => f.journalFile === strayTmp); + assert.ok(detectedFinding, 'plan-side detection must report the stray .tmp file'); + assert.match(detectedFinding.note, /incomplete journal write|\.tmp/); + + const recovered = recoverInterruptedMoves(home); + const finding = recovered.find(r => r.journalFile === strayTmp); + assert.ok(finding, 'recover must report the stray .tmp file too'); + assert.equal(finding.action, 'left-alone'); + assert.match(finding.note, /incomplete journal write|\.tmp/); + assert.equal(finding.path, null, 'a stray .tmp write describes no confirmed unit — it was never parsed as a record'); + + // Never touched: still sitting there exactly as it was. + assert.equal(existsSync(strayTmp), true); + }); + + it('(c) positive control: a complete, valid journal record still drives recovery normally', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + const child = runApplyInChildWithCrash(plan, home, target, 'afterStage'); + assert.equal(child.status, 77); + + const journalFile = findJournalFile(home); + const parsed = JSON.parse(readFileSync(journalFile, 'utf8')); + assert.equal(parsed.sourcePath, src, 'sanity: this is a real, complete, readable record'); + + const detected = detectInterruptedMoves(home); + assert.equal(detected.length, 1); + assert.equal(detected[0].status, 'interrupted'); + + const recovered = recoverInterruptedMoves(home); + assert.equal(recovered.length, 1); + assert.equal(recovered[0].action, 'completed-swap-and-cleaned'); + assert.equal(lstatSync(src).isSymbolicLink(), true); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'gap2-fixture-bytes'.repeat(50)); + }); + }); +}); + +describe('ROUND 6 ITEM 1: findProcessUsers fails closed on a real read-boundary error, not on every race', () => { + function buildFakeProcRoot() { + const procRoot = tempDir('model-tidy-fakeproc-'); + mkdirSync(join(procRoot, '1234', 'fd'), { recursive: true }); + writeFileSync(join(procRoot, '1234', 'fd', '3'), ''); + writeFileSync(join(procRoot, '1234', 'cmdline'), 'unrelated-process\0--flag\0'); + return procRoot; + } + + it('ENOENT at the fd-resolution boundary (fd vanished mid-scan) is ignorable — checked stays true, candidate remains selectable', () => { + const procRoot = buildFakeProcRoot(); + const result = findProcessUsers('/some/candidate/path', { + procRoot, + realpathSync: () => { + const e = new Error('no such file or directory'); + e.code = 'ENOENT'; + throw e; + } + }); + assert.equal(result.checked, true, 'ENOENT at the fd boundary must not fail the check closed'); + assert.equal(result.users.length, 0); + }); + + it('EACCES at the fd-resolution boundary marks that pid unverified and fails closed, naming the pid and the code', () => { + const procRoot = buildFakeProcRoot(); + const result = findProcessUsers('/some/candidate/path', { + procRoot, + realpathSync: () => { + const e = new Error('permission denied'); + e.code = 'EACCES'; + throw e; + } + }); + assert.equal(result.checked, false, 'EACCES at the fd boundary must fail the check closed'); + assert.match(result.note, /1234/, 'the note must name the pid'); + assert.match(result.note, /EACCES/, 'the note must name the error code'); + }); + + it('positive control: with no read-boundary errors at all, the check completes normally', () => { + const procRoot = buildFakeProcRoot(); + const result = findProcessUsers('/some/candidate/path', { + procRoot, + realpathSync: p => p // resolves to itself — never matches the candidate path, never throws + }); + assert.equal(result.checked, true); + assert.equal(result.users.length, 0); + }); + + it('end-to-end: planRun skips the candidate with a reason naming the pid and code when the process check is unverified', () => { + const home = tempDir('model-tidy-r6-home-'); + const src = join(home, 'models', 'idle'); + mkdirSync(src, { recursive: true }); + writeFileSync(join(src, 'weights.gguf'), 'x'.repeat(1024)); + utimesSync(join(src, 'weights.gguf'), daysAgo(30), daysAgo(30)); + + const plan = planRun({ + home, + minIdleDays: 14, + listProcessUsers: () => ({ checked: false, users: [], note: '4242 EACCES — cannot rule out that process using this path' }), + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + const skipped = plan.skipped.find(r => r.path === src); + assert.ok(skipped); + assert.match(skipped.reason, /in-use status unverified: 4242 EACCES/); + }); +}); + +describe('ROUND 6 ITEM 2: sha256File hashes with bounded memory (streaming chunks), never a whole-file read', () => { + it('hashes a file LARGER than the chunk size identically to a whole-buffer digest', () => { + const dir = tempDir('model-tidy-hash-'); + const file = join(dir, 'big.bin'); + const size = HASH_CHUNK_BYTES + 3 * 1024 * 1024; // > 1 full chunk, plus a partial tail + const content = randomBytes(size); + writeFileSync(file, content); + + const streamed = sha256File(file); + const wholeBuffer = createHash('sha256').update(content).digest('hex'); + assert.equal(streamed, wholeBuffer, 'the chunked hash must exactly match the whole-buffer hash'); + }); + + it('never allocates a buffer larger than the chunk size while hashing (proves bounded memory, not a hidden readFileSync)', () => { + const dir = tempDir('model-tidy-hash-'); + const file = join(dir, 'big2.bin'); + const size = HASH_CHUNK_BYTES * 2 + 12345; // several chunks, plus a partial tail + const content = randomBytes(size); + writeFileSync(file, content); + + const originalAllocUnsafe = Buffer.allocUnsafe; + const originalAlloc = Buffer.alloc; + const seenSizes = []; + function guard(n) { + seenSizes.push(n); + if (n > HASH_CHUNK_BYTES) { + Buffer.allocUnsafe = originalAllocUnsafe; + Buffer.alloc = originalAlloc; + throw new Error(`sha256File allocated ${n} bytes, more than the ${HASH_CHUNK_BYTES}-byte chunk limit — it read some or all of the file into memory at once`); + } + } + Buffer.allocUnsafe = function (n) { + guard(n); + return originalAllocUnsafe.call(Buffer, n); + }; + Buffer.alloc = function (n, ...rest) { + guard(n); + return originalAlloc.call(Buffer, n, ...rest); + }; + + let hash; + try { + hash = sha256File(file); + } finally { + Buffer.allocUnsafe = originalAllocUnsafe; + Buffer.alloc = originalAlloc; + } + + assert.equal(hash, createHash('sha256').update(content).digest('hex')); + assert.ok(seenSizes.length >= 1, 'sanity: at least one allocation was observed'); + assert.ok(seenSizes.every(n => n <= HASH_CHUNK_BYTES), `every allocation must be <= ${HASH_CHUNK_BYTES} bytes; saw ${JSON.stringify(seenSizes)}`); + }); +}); + +describe('ROUND 6 ITEM 3: swapToSymlink pre-swap verification failure never recursively deletes an assumed symlink', () => { + it('sentinel test: symlinkFn creates a REAL DIRECTORY at the link path instead of a symlink — it and a sentinel file inside it survive, apply fails, and the journal record says failed', () => { + const home = tempDir('model-tidy-r6-item3-home-'); + const target = tempDir('model-tidy-r6-item3-target-'); + const src = join(home, 'models', 'idle'); + mkdirSync(src, { recursive: true }); + writeFileSync(join(src, 'weights.gguf'), 'r6-item3-bytes'.repeat(50)); + utimesSync(join(src, 'weights.gguf'), daysAgo(30), daysAgo(30)); + + const plan = planRun({ + home, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + assert.ok(plan.selected.some(r => r.path === src)); + + const link = `${src}.tidy-link`; + const result = applyRun({ + plan, + target, + home, + validateTarget: bypassCrossFsCheck, + symlinkFn: (dst, linkPath) => { + // Simulate something creating a real directory exactly where + // model-tidy expected to create — and, on verification failure, + // remove — its own symlink. + mkdirSync(linkPath, { recursive: true }); + writeFileSync(join(linkPath, 'SENTINEL.txt'), 'do not delete me'); + } + }); + + assert.equal(result.ok, false); + assert.equal(result.errors.length, 1); + assert.equal(result.errors[0].step, 'swap'); + assert.match(result.errors[0].error, /pre-swap symlink verification failed/); + assert.match(result.errors[0].error, /could not be safely removed/); + + // The directory and its sentinel file must survive, completely + // untouched — never rmSync'd. + assert.equal(existsSync(link), true); + assert.equal(lstatSync(link).isSymbolicLink(), false); + assert.equal(existsSync(join(link, 'SENTINEL.txt')), true); + assert.equal(readFileSync(join(link, 'SENTINEL.txt'), 'utf8'), 'do not delete me'); + + // The original source itself must also be untouched — swapToSymlink + // never got past the pre-swap step, so nothing was ever renamed away. + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), false); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'r6-item3-bytes'.repeat(50)); + + // The journal record must say failed, not be silently cleared. + const journalDir = join(home, '.cache', 'ide-agent-kit', 'model-tidy-journal'); + const names = readdirSync(journalDir).filter(n => n.endsWith('.json')); + assert.equal(names.length, 1); + const record = JSON.parse(readFileSync(join(journalDir, names[0]), 'utf8')); + assert.equal(record.step, 'failed'); + assert.match(record.failureReason, /could not be safely removed/); + }); +}); + +describe('ROUND 7', () => { + function journalDirFor(home) { + return join(home, '.cache', 'ide-agent-kit', 'model-tidy-journal'); + } + function findJournalFile(home) { + const dir = journalDirFor(home); + const names = readdirSync(dir).filter(n => n.endsWith('.json') && !n.includes('.tmp-')); + assert.equal(names.length, 1, `expected exactly one journal file, found ${names.length}: ${names.join(', ')}`); + return join(dir, names[0]); + } + function buildIdleFixture(prefix) { + const home = tempDir(prefix || 'model-tidy-r7-home-'); + const src = join(home, 'models', 'idle'); + mkdirSync(src, { recursive: true }); + writeFileSync(join(src, 'weights.gguf'), 'GOOD'.repeat(20)); + utimesSync(join(src, 'weights.gguf'), daysAgo(30), daysAgo(30)); + return { home, src }; + } + function planFor(home) { + return planRun({ + home, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + } + function snapshotTree(root) { + const snap = {}; + function recurse(dir) { + for (const name of readdirSync(dir)) { + const full = join(dir, name); + const lst = lstatSync(full); + if (lst.isSymbolicLink()) { + snap[full] = { type: 'symlink', linkTarget: readlinkSync(full) }; + } else if (lst.isDirectory()) { + recurse(full); + } else if (lst.isFile()) { + snap[full] = { type: 'file', size: lst.size, sha256: createHash('sha256').update(readFileSync(full)).digest('hex') }; + } + } + } + recurse(root); + return snap; + } + function escapeRe(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + } + + describe('GAP 1: an unreadable journal record must still protect its unit, recovered from the FILENAME', () => { + it("codexmb's probe verbatim: interrupt beforeLink, empty the journal record, rerun plan — the source must NOT be selected, and the reason names the record", () => { + const { home, src } = buildIdleFixture(); + const target = tempDir('model-tidy-r7-target-'); + const opts = { home, listProcessUsers: noProcessUsers, listDockerBindUsers: dockerNotInUse, diskFreeBytes: fixedDiskFree }; + const interrupted = applyRun({ + home, + target, + plan: planRun(opts), + validateTarget: () => ({ ok: true }), + beforeLink: () => { throw new Error('fixture interruption'); } + }); + assert.equal(interrupted.ok, false); + + const journalFile = findJournalFile(home); + writeFileSync(journalFile, ''); + + const next = planRun(opts); + assert.ok(!next.selected.some(r => r.path === src), 'the source must not be selected while its journal record is unreadable'); + const skipped = next.skipped.find(r => r.path === src); + assert.ok(skipped, 'the source must be reported in skipped, recovered from the journal filename'); + assert.match(skipped.reason, /interrupted move with unreadable journal record/); + assert.match(skipped.reason, new RegExp(escapeRe(journalFile)), 'the reason must name the record file'); + }); + + it('a stray journal file with an UNPARSEABLE name: plan selects nothing, and apply refuses before any mutation', () => { + const { home } = buildIdleFixture(); + const target = tempDir('model-tidy-r7-target-'); + const journalDir = journalDirFor(home); + mkdirSync(journalDir, { recursive: true }); + const strayFile = join(journalDir, 'not-a-real-hash-key.json'); + writeFileSync(strayFile, '{ this is not valid json'); + + const before = snapshotTree(home); + + const plan = planFor(home); + assert.equal(plan.selected.length, 0, 'plan must select nothing while an unresolvable journal file exists'); + assert.ok(plan.globalJournalBlockReason); + assert.match(plan.globalJournalBlockReason, /unresolved journal state/); + assert.match(plan.globalJournalBlockReason, new RegExp(escapeRe(strayFile))); + assert.match(plan.summaryLine, /REFUSING TO SELECT ANYTHING THIS RUN/); + + assert.deepEqual(snapshotTree(home), before, 'plan must not mutate anything'); + + const result = applyRun({ plan, target, home, validateTarget: bypassCrossFsCheck }); + assert.equal(result.ok, false); + assert.match(result.error, /unresolved journal state/); + assert.match(result.error, new RegExp(escapeRe(strayFile))); + + assert.deepEqual(snapshotTree(home), before, 'apply must not mutate anything either'); + assert.equal(readdirSync(target).length, 0, 'apply must refuse before even attempting to copy anything into target'); + }); + + it('positive control: a healthy, journal-free run still selects the idle dir', () => { + const { home, src } = buildIdleFixture(); + const plan = planFor(home); + assert.ok(plan.selected.some(r => r.path === src)); + assert.equal(plan.globalJournalBlockReason, null); + }); + }); + + describe('GAP 2: a directory-fsync failure must never be silently swallowed', () => { + // Scoped to the SOURCE tree only, not model-tidy's own journal + // bookkeeping under home/.cache (a durability failure legitimately + // leaves journal-directory debris: a written-but-not-fsynced record, + // or an un-renamed temp file) and not the target dir (the copy step + // runs, and legitimately succeeds, BEFORE the swap step where the + // journal write under test happens — target ending up with a copy is + // expected, not a violation; what must never change is the source). + function snapshotSource(home) { + return snapshotTree(join(home, 'models')); + } + + it('EIO on the journal-directory fsync, on a SIMULATED Linux platform: apply aborts, source untouched (full tree identical), report names the error', () => { + const { home, src } = buildIdleFixture('model-tidy-r7-gap2a-home-'); + const target = tempDir('model-tidy-r7-gap2a-target-'); + const plan = planFor(home); + assert.ok(plan.selected.some(r => r.path === src)); + + const before = snapshotSource(home); + + // Within one writeJournalRecordSync call, fsyncSync is called on the + // temp-file fd first, then the directory fd second — fail every + // SECOND call (the directory fsync) and let the first (temp-file) + // succeed, so this test isolates the directory-fsync failure + // specifically. The very first journal write (step 'pending') + // happens before any source mutation, so its directory-fsync + // failure aborts before anything is touched. + let callCount = 0; + const result = applyRun({ + plan, + target, + home, + validateTarget: bypassCrossFsCheck, + platform: 'linux', + journalFsyncSync: () => { + callCount++; + if (callCount % 2 === 0) { + throw Object.assign(new Error('input/output error'), { code: 'EIO' }); + } + } + }); + + assert.equal(result.ok, false); + assert.equal(result.moved.length, 0); + assert.ok(result.errors.length >= 1); + assert.match(result.errors[0].error, /EIO/); + assert.match(result.errors[0].error, /journal directory/); + // The copy+verify step runs (and succeeds) BEFORE the swap step + // where this journal write happens, so target legitimately holds a + // copy — that's expected, not a violation. What must never change, + // durability failure or not, is the SOURCE. + assert.deepEqual(snapshotSource(home), before, 'the source must be completely untouched'); + assert.equal(lstatSync(src).isSymbolicLink(), false, 'the source must still be a real directory, never swapped for a symlink'); + }); + + it('ENOTSUP on the journal-directory fsync, on a non-Linux platform: proceeds with a "durability degraded" note', () => { + const { home, src } = buildIdleFixture('model-tidy-r7-gap2b-home-'); + const target = tempDir('model-tidy-r7-gap2b-target-'); + const plan = planFor(home); + + let callCount = 0; + const result = applyRun({ + plan, + target, + home, + validateTarget: bypassCrossFsCheck, + platform: 'darwin', // explicitly injected, never the real process.platform + journalFsyncSync: () => { + callCount++; + if (callCount % 2 === 0) { + throw Object.assign(new Error('operation not supported'), { code: 'ENOTSUP' }); + } + } + }); + + assert.equal(result.ok, true, JSON.stringify(result.errors)); + assert.equal(result.moved.length, 1); + assert.equal(lstatSync(src).isSymbolicLink(), true); + assert.ok(result.durabilityNotes.length > 0, 'a durability-degraded note must be reported, not silently dropped'); + assert.ok(result.durabilityNotes.every(n => n.includes('durability degraded: ENOTSUP'))); + }); + + it('EIO on the journal TEMP-FILE fsync aborts, on any platform', () => { + const { home, src } = buildIdleFixture('model-tidy-r7-gap2c-home-'); + const target = tempDir('model-tidy-r7-gap2c-target-'); + const plan = planFor(home); + + const before = snapshotSource(home); + + // Fail the FIRST fsyncSync call within each write (the temp file), + // regardless of platform — a temp-file fsync failure is never + // tolerated anywhere, unlike the directory fsync on non-Linux. + let callCount = 0; + const result = applyRun({ + plan, + target, + home, + validateTarget: bypassCrossFsCheck, + platform: 'darwin', + journalFsyncSync: () => { + callCount++; + if (callCount % 2 === 1) { + throw Object.assign(new Error('input/output error'), { code: 'EIO' }); + } + } + }); + + assert.equal(result.ok, false); + assert.equal(result.moved.length, 0); + assert.ok(result.errors.length >= 1); + assert.match(result.errors[0].error, /EIO/); + assert.match(result.errors[0].error, /temp file/); + assert.deepEqual(snapshotSource(home), before, 'the source must be completely untouched'); + }); + }); +}); + +describe('ROUND 8: the unresolved-journal preflight must run BEFORE any automatic recovery, not after', () => { + function journalDirFor(home) { + return join(home, '.cache', 'ide-agent-kit', 'model-tidy-journal'); + } + function buildIdleFixture(prefix) { + const home = tempDir(prefix || 'model-tidy-r8-home-'); + const src = join(home, 'models', 'idle'); + mkdirSync(src, { recursive: true }); + writeFileSync(join(src, 'weights.gguf'), 'GOOD'); + utimesSync(join(src, 'weights.gguf'), new Date(0), new Date(0)); + return { home, src }; + } + function opts(home) { + return { + home, + listProcessUsers: () => ({ checked: true, users: [] }), + listDockerBindUsers: () => ({ available: true, inUse: false, containers: [] }), + diskFreeBytes: () => 0 + }; + } + function snapshotTree(root) { + const snap = {}; + function recurse(dir) { + for (const name of readdirSync(dir)) { + const full = join(dir, name); + const lst = lstatSync(full); + if (lst.isSymbolicLink()) { + snap[full] = { type: 'symlink', linkTarget: readlinkSync(full) }; + } else if (lst.isDirectory()) { + recurse(full); + } else if (lst.isFile()) { + snap[full] = { type: 'file', size: lst.size, sha256: createHash('sha256').update(readFileSync(full)).digest('hex') }; + } + } + } + recurse(root); + return snap; + } + /** Build the mixed fixture: one VALID interrupted unit (afterStage, so + * its staging exists and its source is currently missing) plus one + * completely unrelated, unreadable journal file (empty unknown.json). */ + function buildMixedFixture() { + const { home, src } = buildIdleFixture('model-tidy-r8-mixed-home-'); + const target = tempDir('model-tidy-r8-mixed-target-'); + const o = opts(home); + const interrupted = applyRun({ + home, target, plan: planRun(o), validateTarget: () => ({ ok: true }), + afterStage: () => { throw new Error('fixture interruption'); } + }); + assert.equal(interrupted.ok, false, 'sanity: the fixture interruption must have happened'); + const journalDir = journalDirFor(home); + writeFileSync(join(journalDir, 'unknown.json'), ''); + return { home, src, target, o }; + } + + it("codexmb's probe verbatim: apply on a mixed fixture (one valid afterStage interruption + one empty unknown.json) refuses with the unresolved-journal error, and the full tree is byte-identical before/after — the valid unit's staging is still present and its source is not yet a symlink", () => { + const { home, src, target, o } = buildMixedFixture(); + const staging = `${src}.tidy-moving`; + assert.equal(existsSync(src), false, 'sanity: the valid unit is mid-swap, source currently missing'); + assert.equal(existsSync(staging), true, 'sanity: its staged original is present'); + + const before = snapshotTree(home); + + const result = applyRun({ home, target, plan: planRun(o), validateTarget: () => ({ ok: true }) }); + + assert.equal(result.ok, false); + assert.match(result.error, /unresolved journal state/); + assert.deepEqual(result.recovered, [], 'apply must not have run any recovery at all — not even for the valid unit'); + + assert.deepEqual(snapshotTree(home), before, 'the ENTIRE tree, including the valid interrupted unit, must be byte-identical before and after — apply touched nothing'); + assert.equal(existsSync(src), false, 'the valid unit must still be mid-swap: source still missing'); + assert.equal(existsSync(staging), true, 'the valid unit\'s staged original must still be present, not deleted'); + }); + + it('the lone-corrupt case (no valid interrupted unit at all) still refuses', () => { + const { home } = buildIdleFixture('model-tidy-r8-lone-home-'); + const target = tempDir('model-tidy-r8-lone-target-'); + const journalDir = journalDirFor(home); + mkdirSync(journalDir, { recursive: true }); + writeFileSync(join(journalDir, 'unknown.json'), ''); + + const plan = planRun(opts(home)); + const result = applyRun({ home, target, plan, validateTarget: () => ({ ok: true }) }); + + assert.equal(result.ok, false); + assert.match(result.error, /unresolved journal state/); + assert.deepEqual(result.recovered, []); + }); + + it('positive control: a valid interruption with NO unreadable records still recovers and proceeds normally', () => { + const { home, src } = buildIdleFixture('model-tidy-r8-positive-home-'); + const target = tempDir('model-tidy-r8-positive-target-'); + const o = opts(home); + const interrupted = applyRun({ + home, target, plan: planRun(o), validateTarget: () => ({ ok: true }), + afterStage: () => { throw new Error('fixture interruption'); } + }); + assert.equal(interrupted.ok, false); + assert.equal(existsSync(src), false); + + // No corrupt/unreadable journal file this time — just re-run apply. + const result = applyRun({ home, target, plan: planRun(o), validateTarget: () => ({ ok: true }) }); + + assert.equal(result.ok, true, JSON.stringify(result.errors)); + assert.equal(result.recovered.length, 1); + assert.equal(result.recovered[0].action, 'completed-swap-and-cleaned'); + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), true); + }); + + it("recover (the explicit subcommand's underlying function) on the mixed fixture heals the valid unit and reports the unreadable one — recover's own behavior is unchanged by this fix", () => { + const { home, src } = buildMixedFixture(); + const staging = `${src}.tidy-moving`; + + const recovered = recoverInterruptedMoves(home); + + const validFinding = recovered.find(r => r.path === src); + assert.ok(validFinding, 'the valid unit must be reported'); + assert.equal(validFinding.action, 'completed-swap-and-cleaned'); + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), true); + assert.equal(existsSync(staging), false); + + const unreadableFinding = recovered.find(r => r.journalFile && r.journalFile.endsWith('unknown.json')); + assert.ok(unreadableFinding, 'the unreadable record must be reported too'); + assert.equal(unreadableFinding.action, 'left-alone'); + assert.match(unreadableFinding.note, /unreadable/); + }); +});