From 104d7c9cf51924ae72ca21cfeffa83412208857c Mon Sep 17 00:00:00 2001 From: Petrus Pennanen Date: Mon, 21 Sep 2026 08:33:50 +0200 Subject: [PATCH 1/9] Add model-tidy: safe plan/apply tool for idle LLM model cleanup Moves idle local model directories (HF cache models--*, ~/models/*, ad-hoc project dirs) off a full GPU-box drive onto another mount, leaving a symlink behind so existing paths and docker bind mounts keep working. - plan (default, read-only) vs apply (needs --apply AND a verified, writable, cross-filesystem --target). - Selection rules run in order with an explicit skip reason each: KEEP list, process/fd + serving-process cmdline in-use checks, running-container docker bind mounts (fail-safe to "all of ~/.cache/huggingface is in use" if docker is unreadable), --min-idle-days, already-a-symlink, and hardlink sets that must move together as one unit or not at all. - apply copies with a hand-written pure-Node recursive copy that explicitly recreates hardlinks via dev:ino tracking (not a shelled rsync, to keep the repo's zero-dependency Node-only posture and keep tests deterministic), verifies every file byte-for-byte (size + SHA-256) before ever deleting the source, then symlinks and re-verifies. Any failure leaves the source untouched. - systemd/model-tidy.{service,timer} for a nightly plan-only run (not installed by this change); docs/model-tidy.md covers install, the selection rules, and a "not verified" list. - test/model-tidy.test.mjs: fixture-based plan test (idle dir + complete hardlink set selected, correct skip reasons for KEEP/ recent/already-tidied/broken-hardlink-set/docker-unreadable/ process-in-use/max-gb-capped) plus an apply test and a negative control (corrupted target checksum -> refuse, source untouched). Never run against a real machine; --dry-run-remote is read-only plan-mode-only ssh helper, off by default. Co-Authored-By: Claude Fable 5.1 --- README.md | 1 + bin/model-tidy.mjs | 209 +++++++++ config/model-tidy.keep.example | 16 + docs/model-tidy.md | 224 +++++++++ package.json | 4 +- src/model-tidy.mjs | 802 +++++++++++++++++++++++++++++++++ systemd/model-tidy.service | 20 + systemd/model-tidy.timer | 10 + test/model-tidy.test.mjs | 381 ++++++++++++++++ 9 files changed, 1666 insertions(+), 1 deletion(-) create mode 100755 bin/model-tidy.mjs create mode 100644 config/model-tidy.keep.example create mode 100644 docs/model-tidy.md create mode 100644 src/model-tidy.mjs create mode 100644 systemd/model-tidy.service create mode 100644 systemd/model-tidy.timer create mode 100644 test/model-tidy.test.mjs diff --git a/README.md b/README.md index 8a6c1b1..fddec50 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ Run allowlisted commands in a named tmux session, capture output + exit code. 11. **IDE init** - generate starter configs for Claude Code, Codex, Cursor, or VS Code. 12. **ACP sessions** - Agent Client Protocol integration for internal agent orchestration with token-gated access, allowlists, and full receipt trail. 13. **Background consolidation** - optional `light / REM / deep` pass over recent queue items, with append-only sidecars and no effect on the foreground room loop by default. +14. **model-tidy** - plan/apply tool for moving idle local LLM model directories off a full GPU-box drive onto another mount, leaving a symlink behind. Read-only `plan` by default; `apply` needs an explicit flag plus a verified, cross-filesystem target. See [docs/model-tidy.md](docs/model-tidy.md). No dependencies. Node.js ≥ 18 only. diff --git a/bin/model-tidy.mjs b/bin/model-tidy.mjs new file mode 100755 index 0000000..e57ea36 --- /dev/null +++ b/bin/model-tidy.mjs @@ -0,0 +1,209 @@ +#!/usr/bin/env node +// SPDX-License-Identifier: AGPL-3.0-only + +/** + * model-tidy CLI. + * + * model-tidy plan [--home ] [--keep-file ] [--min-idle-days N] + * [--max-gb N] [--json] [--log-dir ] + * [--report-to-room] [--room ] [--config ] + * + * model-tidy apply --apply --target [same options as plan] + * + * model-tidy --dry-run-remote [--remote-home ] + * ssh's to and runs `plan` there, read-only. Never copies, + * deletes, or symlinks anything. If the remote has no `node`, this + * refuses and exits non-zero rather than installing one. + * + * See docs/model-tidy.md for full documentation. + */ + +import { execFileSync, spawnSync } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { homedir } from 'node:os'; +import { planRun, applyRun, writeRunLog } from '../src/model-tidy.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = join(__dirname, '..'); +const DEFAULT_LOG_DIR = join(homedir(), '.cache', 'ide-agent-kit', 'model-tidy-logs'); +const DEFAULT_KEEP_FILE = join(homedir(), '.config', 'ide-agent-kit', 'model-tidy.keep'); + +function parseArgs(argv) { + const out = { _: [] }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg.startsWith('--')) { + const eq = arg.indexOf('='); + if (eq !== -1) { + out[arg.slice(2, eq)] = arg.slice(eq + 1); + continue; + } + const key = arg.slice(2); + const next = argv[i + 1]; + if (next === undefined || next.startsWith('--')) { + out[key] = true; + } else { + out[key] = next; + i++; + } + } else { + out._.push(arg); + } + } + return out; +} + +function printPlan(plan, opts) { + if (opts.json) { + console.log(JSON.stringify(plan, null, 2)); + return; + } + console.log(plan.summaryLine); + console.log(''); + console.log(`Would move (${plan.selected.length}):`); + const printedGroups = new Set(); + for (const r of plan.selected) { + const gib = (r.sizeBytes / 2 ** 30).toFixed(2); + console.log(` [move] ${r.path} (${gib} GiB, ${r.kind}) — ${r.reason}`); + printedGroups.add(r.groupId); + } + console.log(''); + console.log(`Skipped (${plan.skipped.length}):`); + for (const r of plan.skipped) { + const gib = (r.sizeBytes / 2 ** 30).toFixed(2); + console.log(` [skip] ${r.path} (${gib} GiB, ${r.kind}) — ${r.reason}`); + } +} + +function reportToRoom(summaryLine, args) { + const room = args.room; + if (!room) { + console.error('--report-to-room requires --room '); + return false; + } + let apiKey = process.env.IAK_API_KEY; + if (!apiKey && args.config && existsSync(args.config)) { + try { + const cfg = JSON.parse(readFileSync(args.config, 'utf8')); + apiKey = cfg?.poller?.api_key; + } catch (e) { + console.error(`could not read --config ${args.config}: ${e.message}`); + } + } + if (!apiKey) { + console.error('--report-to-room: no API key (set IAK_API_KEY or pass --config pointing at a poller config with poller.api_key)'); + return false; + } + const payload = JSON.stringify({ room, body: summaryLine }); + try { + execFileSync('curl', ['-sS', '-X', 'POST', 'https://groupmind.one/api/v1/messages', + '-H', `X-API-Key: ${apiKey}`, '-H', 'Content-Type: application/json', '-d', payload], + { timeout: 15000 }); + return true; + } catch (e) { + console.error(`--report-to-room: post failed: ${e.message}`); + return false; + } +} + +function runDryRunRemote(host, args) { + console.log(`[dry-run-remote] checking node on ${host}...`); + const check = spawnSync('ssh', ['-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8', host, 'command -v node'], { encoding: 'utf8' }); + if (check.status !== 0 || !check.stdout.trim()) { + console.error(`[dry-run-remote] ${host}: node not found on remote. Refusing to install anything.`); + console.error('[dry-run-remote] Install Node.js >= 18 on the remote host, then re-run --dry-run-remote.'); + return 1; + } + const nodeVersionOut = check.stdout.trim(); + console.log(`[dry-run-remote] ${host}: found node at ${nodeVersionOut}`); + + const remoteDir = `/tmp/.model-tidy-dryrun-${process.pid}-${Date.now()}`; + console.log(`[dry-run-remote] copying tool to ${host}:${remoteDir} (temp, read-only run, self-cleaned)...`); + const mkdir = spawnSync('ssh', ['-o', 'BatchMode=yes', host, `mkdir -p '${remoteDir}/bin' '${remoteDir}/src'`], { encoding: 'utf8' }); + if (mkdir.status !== 0) { + console.error(`[dry-run-remote] mkdir on remote failed: ${mkdir.stderr}`); + return 1; + } + const scpBin = spawnSync('scp', ['-q', join(REPO_ROOT, 'bin', 'model-tidy.mjs'), `${host}:${remoteDir}/bin/model-tidy.mjs`], { encoding: 'utf8' }); + const scpSrc = spawnSync('scp', ['-q', join(REPO_ROOT, 'src', 'model-tidy.mjs'), `${host}:${remoteDir}/src/model-tidy.mjs`], { encoding: 'utf8' }); + if (scpBin.status !== 0 || scpSrc.status !== 0) { + console.error(`[dry-run-remote] scp failed: ${scpBin.stderr || ''} ${scpSrc.stderr || ''}`); + spawnSync('ssh', [host, `rm -rf '${remoteDir}'`]); + return 1; + } + + const remoteHome = args['remote-home'] || '$HOME'; + const remoteCmd = `node '${remoteDir}/bin/model-tidy.mjs' plan --home "${remoteHome}"`; + console.log(`[dry-run-remote] running (read-only): ${remoteCmd}`); + const run = spawnSync('ssh', ['-o', 'BatchMode=yes', host, remoteCmd], { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }); + console.log(run.stdout || ''); + if (run.stderr) console.error(run.stderr); + + spawnSync('ssh', [host, `rm -rf '${remoteDir}'`]); + console.log(`[dry-run-remote] cleaned up ${host}:${remoteDir}`); + return run.status === 0 ? 0 : (run.status ?? 1); +} + +function main() { + const argv = process.argv.slice(2); + const args = parseArgs(argv); + const mode = args._[0] || (args.apply ? 'apply' : 'plan'); + + if (args['dry-run-remote']) { + process.exit(runDryRunRemote(args['dry-run-remote'], args)); + return; + } + + const home = args.home || homedir(); + const keepFile = args['keep-file'] || (existsSync(DEFAULT_KEEP_FILE) ? DEFAULT_KEEP_FILE : undefined); + const minIdleDays = args['min-idle-days'] !== undefined ? Number(args['min-idle-days']) : 14; + const maxGb = args['max-gb'] !== undefined ? Number(args['max-gb']) : Infinity; + const logDir = args['log-dir'] || DEFAULT_LOG_DIR; + + const plan = planRun({ home, keepFile, minIdleDays, maxGb }); + + const logFile = writeRunLog(logDir, { + mode, + ...plan, + argv + }); + + if (mode === 'plan') { + printPlan(plan, args); + console.log(`\n(log: ${logFile})`); + if (args['report-to-room']) reportToRoom(plan.summaryLine, args); + process.exit(0); + } + + if (mode === 'apply') { + if (!args.apply) { + console.error('apply mode requires the explicit --apply flag'); + process.exit(2); + } + if (!args.target) { + console.error('apply mode requires --target '); + process.exit(2); + } + printPlan(plan, args); + const result = applyRun({ plan, target: args.target, home }); + writeRunLog(logDir, { mode: 'apply', target: args.target, ...result }); + if (!result.ok) { + console.error('\napply FAILED for one or more units — sources for failed units were left untouched:'); + for (const err of result.errors) { + console.error(` [error] group ${err.groupId} (${err.step}): ${err.error}`); + } + } + console.log(`\nmoved ${result.moved.length} unit(s), ${result.errors.length} error(s). (log dir: ${logDir})`); + if (args['report-to-room']) { + reportToRoom(`model-tidy apply: moved ${result.moved.length} unit(s), ${result.errors.length} error(s)`, args); + } + process.exit(result.ok ? 0 : 1); + } + + console.error(`unknown mode: ${mode} (expected "plan" or "apply")`); + process.exit(2); +} + +main(); diff --git a/config/model-tidy.keep.example b/config/model-tidy.keep.example new file mode 100644 index 0000000..425fc44 --- /dev/null +++ b/config/model-tidy.keep.example @@ -0,0 +1,16 @@ +# model-tidy KEEP list — example file, NOT active by default. +# +# One path or glob per line. Anything matching is never moved by +# model-tidy, in either plan or apply mode. '#' starts a comment; blank +# lines are ignored. A leading '~' is expanded to the home directory the +# tool is run with (--home, or $HOME by default). +# +# To activate: copy this file to ~/.config/ide-agent-kit/model-tidy.keep +# (the default --keep-file path) or pass --keep-file explicitly, then edit +# it to match what you actually want kept. +# +# Seeded from the owner's notes (do not delete without petrus's word): +~/models/glm53-flash-q6/UD-Q6_K_XL +~/models/GLM-5.3-Flash-exl3-2.05bpw +~/DeepSeek-v4.1-Flash-EXL3-2x-DGX-Sparks +~/models/ltx-2.5 diff --git a/docs/model-tidy.md b/docs/model-tidy.md new file mode 100644 index 0000000..017df6e --- /dev/null +++ b/docs/model-tidy.md @@ -0,0 +1,224 @@ +# model-tidy + +Moves idle local LLM model directories off a full internal drive onto a +target mount (external SSD / NAS share), leaving a symlink behind so every +path that already points at the model — a docker bind mount, a script, a +shell alias — keeps working without edits. + +Built for the asus1/asus2 GPU boxes (single NVMe, running near-full), +serving models with vLLM in docker containers bind-mounting +`~/.cache/huggingface`. + +## Why Node, not a shell/Python script + +The repo is Node-only ("No dependencies. Node.js >= 18 only.", per the main +README) and every other host-side tool here (`src/session-keepalive.mjs`, +`src/room-automation.mjs`, etc.) follows that pattern: logic in +`src/.mjs`, a thin CLI wrapper in `bin/`, tests under `test/` using +`node:test` with `mkdtempSync`/`afterEach` cleanup. `model-tidy` follows the +same shape (`src/model-tidy.mjs` + `bin/model-tidy.mjs` + +`test/model-tidy.test.mjs`) rather than introducing Python or a new shell +tool family. There is no existing systemd pattern in this repo (it ships +macOS launchd `.plist` files for its own daemons), but the target hosts are +Ubuntu, so `systemd/model-tidy.{service,timer}` are new but self-contained +units, not wired into the macOS installer. + +One deliberate deviation from the spec's suggested implementation: the copy +step is a small hand-written recursive Node copy +(`copyUnitPureNode` in `src/model-tidy.mjs`), not a shelled-out `rsync`. +Reasons: +- Keeps the zero-dependency, Node-only posture consistent with the rest of + the repo — no assumption that `rsync` is installed or that its flags + behave identically across the dev machine, CI, and asus1/asus2. +- `rsync -H` only preserves a hardlink relationship between files that are + named in the *same* `rsync` invocation. The pure-Node copy tracks a + `dev:ino -> target path` map explicitly across every member of a + hardlink unit and calls `fs.linkSync` for repeats — the same safety + property, verified directly by a test rather than relying on `rsync` + flag behavior. +- It makes the apply tests deterministic on any machine (no dependency on + `rsync` being present, or on its exact hardlink/`--checksum` semantics). + +Byte-for-byte verification (size + SHA-256 of every regular file, plus +symlink-target and file-count comparison) happens after the copy and before +any deletion, which is the same safety bar the spec's `rsync +--checksum-after` approach targets. + +## Modes + +### `plan` (default) — read-only, never touches disk + +``` +node bin/model-tidy.mjs plan [options] +``` + +Prints, for every discovered candidate: +- `[move]` — would be moved, with its size and unit reason +- `[skip]` — would NOT be moved, with the specific reason (see Selection + rules below) + +...then a one-line summary with real GiB sizes (bytes / 2^30) and a +"free before -> ~free after" estimate, and writes a JSON record to the log +dir (`--log-dir`, default `~/.cache/ide-agent-kit/model-tidy-logs/`). + +### `apply` — only with `--apply --target ` + +``` +node bin/model-tidy.mjs apply --apply --target /mnt/model-archive [options] +``` + +Refuses immediately, before touching anything, unless `--target`: +- exists and is a directory +- is writable +- is on a **different filesystem device** than `--home` (checked via + `stat().dev`, not by path string — a bind mount of the same device would + still be refused) + +For each selected unit (a single directory, or a whole hardlink set moved +together): +1. Copy every file into `--target`, preserving the relative-to-home path, + symlinks, and hardlink relationships. +2. Verify every file: same relative paths, same symlink targets, same byte + size AND SHA-256 for every regular file. +3. Only if verification passed for every file in the unit: delete the + source directory (`fs.rmSync`, not a shell `rm -rf`) and replace it with + a symlink to the target copy. +4. Re-verify: the symlink resolves (`realpathSync`) to the target copy. + +**Any failure at steps 1-2 leaves the source completely untouched** — the +delete in step 3 only ever runs after verification has already passed for +every file in that unit. A failure for one unit does not roll back units +that already succeeded earlier in the same run, but the process still +exits non-zero and prints exactly which unit failed and why. + +## Selection rules (in order, each with an explicit reason string) + +1. **KEEP list** — anything matching `--keep-file` (path or glob, `~` + expanded, `#` comments) is always skipped. See + `config/model-tidy.keep.example`. +2. **In use by a process** — skipped if: + - any process has one of the model's files open under `/proc/*/fd`, or + - a recognized serving process's command line references the path + (`vllm`, `llama-server`, `llama.cpp`, `sglang`, `exllama`, `tabby`, + `ollama`, `mlx`, `text-generation`). + Off Linux (no `/proc`), this check reports `checked: false` rather than + silently passing — see "Not verified" below. +3. **Bind-mounted into a running docker container** — reads + `docker inspect` of every running container's `Mounts`; if the + candidate path is the mount source (or under it), it's in use. + **Fail-safe:** if `docker` is not installed or not readable, the WHOLE + of `~/.cache/huggingface` is treated as in-use and the run says so — + nothing outside that tree is blanket-skipped by this rule. +4. **Too recent** — skipped if the newest mtime of any real file in the + directory is within `--min-idle-days` (default 14). +5. **Already tidied** — skipped if the candidate is already a symlink. +6. **Hardlink sets move as one unit** — candidates are grouped by shared + `dev:ino` across different candidate roots. A group is selected only if + *every* member independently passed rules 1-5; otherwise every member is + skipped with a reason naming which member failed and why (moving one + copy of a hardlinked pair to another filesystem breaks the hardlink and + doubles disk use — this is the whole point of the rule). + +Remaining candidates are sorted by size (desc) and capped by `--max-gb` +(whole units only — a unit is either fully included in this run's budget or +fully deferred to the next run). + +## Candidate discovery + +- `~/.cache/huggingface/hub/models--*` — the whole `models--X` directory is + one unit (its `snapshots//*` symlinks point into `blobs/`; moving a + snapshot without its blobs breaks it). +- `~/models/*` — each top-level entry is one unit. +- Ad-hoc project directories: a heuristic. A top-level entry under `--home` + is a candidate if it (or an immediate subdirectory, e.g. `.../model`) + contains a model marker: `config.json`, `tokenizer*.json`, or a file + ending in `.safetensors`, `.gguf`, `.bin`, `.pt`, `.pth`, `.exl2`, + `.exl3`, `.awq`, or `.gptq`. This is NOT exhaustive — see "Not verified". + +## Reporting + +- `plan` and `apply` both print a one-line summary suitable for pasting + into the room, and write a JSON record per run to `--log-dir` + (default `~/.cache/ide-agent-kit/model-tidy-logs/.jsonl`). +- `--report-to-room --room [--config ]` posts that summary + line via the same `groupmind.one` POST the rest of this repo uses + (`src/room-automation.mjs`'s `postMessage`). **Off by default.** Needs an + API key: `IAK_API_KEY` env var, or `poller.api_key` from `--config`'s + JSON. + +## Scheduling — systemd user timer (Linux, nightly, plan-only by default) + +Units are in `systemd/`. They are **not installed by this PR** — the owner +installs them explicitly on each box: + +``` +mkdir -p ~/.config/systemd/user && cp systemd/model-tidy.service systemd/model-tidy.timer ~/.config/systemd/user/ && systemctl --user daemon-reload && systemctl --user enable --now model-tidy.timer +``` + +The shipped `model-tidy.service` runs `plan` only — it never moves anything +until the `ExecStart` line is edited to add `apply --apply --target ...`, +which the unit's own comments call out explicitly. Do this only after +reviewing several plan runs' logs. + +Check it fired: `systemctl --user list-timers model-tidy.timer` and +`journalctl --user -u model-tidy.service`. + +## `--dry-run-remote ` + +``` +node bin/model-tidy.mjs --dry-run-remote asus1 [--remote-home /home/petrus] +``` + +SSHes to ``, checks for `node` there, and if found, `scp`s +`bin/model-tidy.mjs` + `src/model-tidy.mjs` into a throwaway +`/tmp/.model-tidy-dryrun-*` directory, runs `plan` there (read-only, no +`--apply`), prints the output, then `ssh`es back to `rm -rf` **only that +throwaway directory it just created** (never model data). If `node` is not +found on the remote, it refuses and tells you to install Node >= 18 first — +it never attempts to install anything itself. + +## Safety invariants + +- `plan` mode makes zero filesystem writes anywhere. It's the default. +- `apply` requires both `--apply` AND `--target ` — neither alone is + enough. +- `--target` must be an existing, writable directory on a different + filesystem device than `--home`, or apply refuses before touching + anything. +- A source directory is only ever deleted after its copy has been verified + byte-for-byte (size + SHA-256) at the target. There is no code path that + deletes before verifying. +- Deletion uses `fs.rmSync` (Node), never a shell `rm -rf`. +- A hardlink set moves as a unit or not at all — never partially. +- KEEP-listed paths are never touched by either mode. +- `--report-to-room` and any nightly `apply` are both opt-in and off by + default. + +## Not verified + +- **Never run against a real machine except `--dry-run-remote` in plan + mode.** No `apply` run has ever executed outside the test fixtures in + this PR. +- `asus1` / `asus2` have no `node` binary installed (checked via SSH during + this PR's review, read-only: `ssh asus1 command -v node` / + `ssh asus2 command -v node`, both empty). `--dry-run-remote` against both + hosts therefore stops at that check and prints an install instruction — + it does not, and did not, run the actual plan logic on either box. Real + candidate discovery, HF-cache sizing, docker-mount detection, and + process-scanning on asus1/asus2 are unverified until Node is installed + there (the owner's call, not done as part of this change). +- The ad-hoc project-directory heuristic (marker files at the top level or + one level deep) is written to match the two examples in the brief + (`~/DeepSeek-v4.1-Flash-EXL3-2x-DGX-Sparks/`, + `~/GLM-5.3-Flash-EXL3-2x-DGX-Sparks/model`) but has not been run against + the real directory layout on either box. +- The `/proc/*/fd` and cmdline process-in-use check only runs meaningfully + on Linux; it degrades to `checked:false` (not "confirmed idle") off + Linux, but that degraded path itself is only exercised by this dev + machine being macOS, not by a real Linux box lacking `/proc` access. +- `docker inspect`'s exact `Mounts[].Source` string format was assumed from + documented docker behavior, not confirmed against the actual + vLLM/docker-compose setup on asus1/asus2. +- No real hardlinked pair between `~/models/*` and the HF cache has been + inspected on either box — the fixture in `test/model-tidy.test.mjs` + constructs a synthetic one via `fs.linkSync`. diff --git a/package.json b/package.json index de944bc..1110509 100644 --- a/package.json +++ b/package.json @@ -7,11 +7,13 @@ "ide-agent-kit": "./bin/cli.mjs", "ide-agent-kit-mcp": "./bin/iak-mcp.mjs", "iak-pending": "./bin/iak-pending.mjs", - "iak-scan-history": "./scripts/scan-history-for-secrets.mjs" + "iak-scan-history": "./scripts/scan-history-for-secrets.mjs", + "ide-agent-kit-model-tidy": "./bin/model-tidy.mjs" }, "scripts": { "test": "node --test test/*.test.mjs packages/user-intent-kit/test/*.test.js", "scan:history": "node scripts/scan-history-for-secrets.mjs", + "model-tidy": "node bin/model-tidy.mjs", "start": "node bin/cli.mjs serve", "mcp": "node bin/iak-mcp.mjs", "relay": "node scripts/local-relay.mjs", diff --git a/src/model-tidy.mjs b/src/model-tidy.mjs new file mode 100644 index 0000000..734dc81 --- /dev/null +++ b/src/model-tidy.mjs @@ -0,0 +1,802 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +/** + * model-tidy — move idle local LLM model directories off a full internal + * drive onto a target mount (external SSD / NAS), leaving a symlink behind + * so every path that already points at the model keeps working. + * + * Two modes: + * plan (default) — read-only. Lists what WOULD move, and why every other + * candidate is skipped. Never touches disk. + * apply — only runs with --apply AND --target . Copies + * (verified byte-for-byte), then replaces the source + * directory with a symlink to the copy, then re-verifies + * the symlink resolves. Any failure at any step leaves + * the source untouched. + * + * Zero external dependencies (Node >= 18 only), matching the rest of + * ide-agent-kit. The copy step is a small hand-written recursive copy + * (not a shelled-out `rsync`) specifically so that hardlink relationships + * *within a moved unit* are recreated at the destination via fs.linkSync — + * a plain recursive copy (or `cp -a` without `-H` semantics) would silently + * double disk usage for a hardlinked model. See copyUnitPureNode(). + * + * Selection rules run in this order, each producing an explicit reason: + * 1. KEEP list match -> always skipped + * 2. open by a process, or referenced by a -> skipped, "in use" + * known serving process's command line + * 3. under the bind-mount source of a RUNNING -> skipped, "in use" + * docker container (fail-safe: if docker is + * unreadable, ~/.cache/huggingface is treated + * as in-use) + * 4. newest mtime within --min-idle-days -> skipped, "too recent" + * 5. already a symlink (previously tidied) -> skipped, "already tidied" + * 6. hardlink sets move as one unit: every member -> skipped, "hardlink set" + * must be a candidate or the whole set is skipped + * + * Remaining candidates are sorted by size (desc) and capped by --max-gb. + */ + +import { spawnSync } from 'node:child_process'; +import { + readdirSync, lstatSync, existsSync, readFileSync, readlinkSync, + realpathSync, symlinkSync, rmSync, mkdirSync, copyFileSync, linkSync, + statSync, appendFileSync, constants as FS_CONSTANTS, accessSync +} from 'node:fs'; +import { join, relative, sep } from 'node:path'; +import { createHash } from 'node:crypto'; +import { homedir } from 'node:os'; + +export const SERVING_PROCESS_NAMES = [ + 'vllm', 'llama-server', 'llama.cpp', 'sglang', 'exllama', + 'tabby', 'ollama', 'mlx', 'text-generation' +]; + +const MODEL_MARKER_FILES = new Set(['config.json', 'tokenizer.json', 'tokenizer_config.json']); +const MODEL_MARKER_EXT = ['.safetensors', '.gguf', '.bin', '.pt', '.pth', '.exl2', '.exl3', '.awq', '.gptq']; +const AD_HOC_EXCLUDE = new Set([ + 'Desktop', 'Downloads', 'Documents', 'Pictures', 'Movies', 'Music', 'Public', + 'node_modules', 'go', 'snap', 'models', '.cache', '.local', '.config', + '.ssh', '.npm', '.cargo', '.rustup', '.docker', '.git' +]); + +// --------------------------------------------------------------------------- +// Filesystem walking primitives +// --------------------------------------------------------------------------- + +/** Recursively list regular files under root. Symlinks are NOT followed or + * counted here (their target is counted when the target itself is walked, + * e.g. HF cache blobs/ vs snapshots/ symlinks). Returns [] on any read error. */ +export function walkFiles(root) { + const results = []; + function recurse(dir) { + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const full = join(dir, entry.name); + let st; + try { + st = lstatSync(full); + } catch { + continue; + } + if (st.isSymbolicLink()) continue; + if (st.isDirectory()) { + recurse(full); + continue; + } + if (st.isFile()) { + results.push({ path: full, size: st.size, mtimeMs: st.mtimeMs, dev: st.dev, ino: st.ino }); + } + } + } + recurse(root); + return results; +} + +/** Sum of real (non-symlink) file bytes under a directory. */ +export function dirSizeBytes(path, filesCache) { + const files = filesCache || walkFiles(path); + return files.reduce((sum, f) => sum + f.size, 0); +} + +/** Newest mtime (ms since epoch) of any real file under a directory. Falls + * back to the directory's own mtime if it has no real files (e.g. an + * all-symlink snapshots/ dir). */ +export function newestMtimeMs(path, filesCache) { + const files = filesCache || walkFiles(path); + if (files.length === 0) { + try { + return lstatSync(path).mtimeMs; + } catch { + return 0; + } + } + return files.reduce((max, f) => Math.max(max, f.mtimeMs), 0); +} + +export function isSymlink(path) { + try { + return lstatSync(path).isSymbolicLink(); + } catch { + return false; + } +} + +function hasModelMarkers(dir) { + let entries; + try { + entries = readdirSync(dir); + } catch { + return false; + } + for (const name of entries) { + if (MODEL_MARKER_FILES.has(name)) return true; + if (MODEL_MARKER_EXT.some(ext => name.endsWith(ext))) return true; + } + return false; +} + +// --------------------------------------------------------------------------- +// Candidate discovery +// --------------------------------------------------------------------------- + +/** + * Discover model-directory candidates under a home directory: + * - ~/.cache/huggingface/hub/models--* (whole dir is the unit; moving a + * snapshot without its blobs breaks it) + * - ~/models/* (each top-level entry is a unit) + * - ad-hoc project dirs: a top-level home entry (or its immediate `model` + * / marker-bearing subdir) that itself contains model marker files + * (config.json, tokenizer*.json, or a *.safetensors/*.gguf/... file). + * This is a heuristic, not exhaustive — see docs/model-tidy.md. + */ +export function discoverCandidates(home) { + const candidates = []; + const seen = new Set(); + + function add(path, kind) { + if (seen.has(path)) return; + seen.add(path); + candidates.push({ id: path, path, kind }); + } + + const hfHub = join(home, '.cache', 'huggingface', 'hub'); + if (existsSync(hfHub)) { + for (const entry of safeReaddir(hfHub)) { + if (entry.startsWith('models--')) { + add(join(hfHub, entry), 'hf-cache'); + } + } + } + + const modelsDir = join(home, 'models'); + if (existsSync(modelsDir)) { + for (const entry of safeReaddir(modelsDir)) { + add(join(modelsDir, entry), 'models-dir'); + } + } + + for (const entry of safeReaddir(home)) { + if (entry.startsWith('.')) continue; + if (AD_HOC_EXCLUDE.has(entry)) continue; + const full = join(home, entry); + let st; + try { + st = lstatSync(full); + } catch { + continue; + } + if (!st.isDirectory()) continue; + if (hasModelMarkers(full)) { + add(full, 'ad-hoc'); + continue; + } + // one level deep: e.g. ~/GLM-5.3-Flash-EXL3-2x-DGX-Sparks/model + for (const child of safeReaddir(full)) { + const childFull = join(full, child); + let cst; + try { + cst = lstatSync(childFull); + } catch { + continue; + } + if (cst.isDirectory() && hasModelMarkers(childFull)) { + add(childFull, 'ad-hoc'); + } + } + } + + return candidates; +} + +function safeReaddir(dir) { + try { + return readdirSync(dir); + } catch { + return []; + } +} + +// --------------------------------------------------------------------------- +// KEEP list +// --------------------------------------------------------------------------- + +/** Load a KEEP list file: one path or glob per line, '#' comments, blank + * lines ignored, leading '~' expanded to home. Missing file -> []. */ +export function loadKeepList(keepFilePath, home = homedir()) { + if (!keepFilePath || !existsSync(keepFilePath)) return []; + const lines = readFileSync(keepFilePath, 'utf8').split('\n'); + const out = []; + for (const raw of lines) { + const line = raw.trim(); + if (!line || line.startsWith('#')) continue; + out.push(line.startsWith('~') ? join(home, line.slice(1).replace(/^\//, '')) : line); + } + return out; +} + +function globToRegExp(glob) { + const escaped = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*').replace(/\?/g, '.'); + return new RegExp(`^${escaped}$`); +} + +/** True if `path` is on the KEEP list — exact match, ancestor match (a + * KEEP-listed directory contains this candidate), or glob match. */ +export function matchesKeepList(path, keepEntries) { + for (const entry of keepEntries) { + if (entry.includes('*') || entry.includes('?')) { + if (globToRegExp(entry).test(path)) return true; + continue; + } + if (path === entry || path.startsWith(entry + sep) || entry.startsWith(path + sep)) return true; + } + return false; +} + +// --------------------------------------------------------------------------- +// Process-in-use detection (Linux /proc) +// --------------------------------------------------------------------------- + +/** + * Detect processes using a path: (a) ANY process with the path open under + * /proc/*\/fd, (b) a recognized serving process (vllm, ollama, ...) whose + * command line references the path. Degrades to `{checked:false}` off Linux + * or without /proc read access — callers should treat that as "could not + * verify" rather than "confirmed idle". + */ +export function findProcessUsers(path, opts = {}) { + const procRoot = opts.procRoot || '/proc'; + if (!existsSync(procRoot)) { + return { checked: false, users: [], note: `${procRoot} not available (not Linux); process-in-use check skipped` }; + } + let pids; + try { + pids = readdirSync(procRoot).filter(n => /^[0-9]+$/.test(n)); + } catch (e) { + return { checked: false, users: [], note: `cannot list ${procRoot}: ${e.message}` }; + } + const users = []; + for (const pid of pids) { + try { + const fdDir = join(procRoot, pid, 'fd'); + for (const fd of readdirSync(fdDir)) { + try { + const target = realpathSync(join(fdDir, fd)); + if (target === path || target.startsWith(path + sep)) { + users.push({ pid, via: 'fd', target }); + break; + } + } catch { + // fd vanished mid-scan, or unreadable — ignore + } + } + } catch { + // /proc//fd unreadable (permission, or process exited) — ignore + } + try { + const cmdline = readFileSync(join(procRoot, pid, 'cmdline'), 'utf8').replace(/\0/g, ' ').trim(); + if (cmdline && cmdline.includes(path)) { + const lower = cmdline.toLowerCase(); + const servingProcess = SERVING_PROCESS_NAMES.some(n => lower.includes(n)); + users.push({ pid, via: 'cmdline', cmdline: cmdline.slice(0, 200), servingProcess }); + } + } catch { + // no permission to read cmdline — ignore + } + } + return { checked: true, users }; +} + +// --------------------------------------------------------------------------- +// Docker bind-mount detection +// --------------------------------------------------------------------------- + +/** + * Is `path` under the bind-mount source of a RUNNING docker container? + * `{available:false}` means docker itself could not be queried — callers + * must apply the fail-safe rule (treat ~/.cache/huggingface as in-use). + */ +export function findDockerBindUsers(path, opts = {}) { + const dockerBin = opts.dockerBin || 'docker'; + const ps = spawnSync(dockerBin, ['ps', '-q'], { encoding: 'utf8', timeout: 10000 }); + if (ps.error || ps.status !== 0) { + return { available: false, inUse: false, reason: `docker ps failed: ${ps.error?.message || ps.stderr?.trim() || `exit ${ps.status}`}` }; + } + const ids = ps.stdout.split('\n').map(s => s.trim()).filter(Boolean); + if (ids.length === 0) return { available: true, inUse: false, containers: [] }; + const inspect = spawnSync(dockerBin, ['inspect', ...ids], { encoding: 'utf8', timeout: 15000, maxBuffer: 64 * 1024 * 1024 }); + if (inspect.error || inspect.status !== 0) { + return { available: false, inUse: false, reason: `docker inspect failed: ${inspect.error?.message || inspect.stderr?.trim() || `exit ${inspect.status}`}` }; + } + let parsed; + try { + parsed = JSON.parse(inspect.stdout); + } catch (e) { + return { available: false, inUse: false, reason: `docker inspect returned unparseable JSON: ${e.message}` }; + } + const hits = []; + for (const c of parsed) { + for (const m of c.Mounts || []) { + if (m.Source && (path === m.Source || path.startsWith(m.Source + sep))) { + hits.push({ container: (c.Name || '').replace(/^\//, '') || (c.Id || '').slice(0, 12), source: m.Source, destination: m.Destination }); + } + } + } + return { available: true, inUse: hits.length > 0, containers: hits }; +} + +// --------------------------------------------------------------------------- +// Hardlink grouping +// --------------------------------------------------------------------------- + +/** + * Group candidates that share an inode (device+inode) across DIFFERENT + * candidate roots into a single move unit. Returns a union-find lookup plus + * a dedup'd byte size per unit (each shared inode counted once). + */ +export function computeHardlinkGroups(candidates) { + const filesByCandidate = new Map(); + const inodeMap = new Map(); // "dev:ino" -> Set(candidateId) + + for (const c of candidates) { + const files = walkFiles(c.path); + filesByCandidate.set(c.id, files); + for (const f of files) { + const key = `${f.dev}:${f.ino}`; + if (!inodeMap.has(key)) inodeMap.set(key, new Set()); + inodeMap.get(key).add(c.id); + } + } + + const parent = new Map(candidates.map(c => [c.id, c.id])); + function find(x) { + while (parent.get(x) !== x) { + parent.set(x, parent.get(parent.get(x))); + x = parent.get(x); + } + return x; + } + function union(a, b) { + const ra = find(a); + const rb = find(b); + if (ra !== rb) parent.set(ra, rb); + } + for (const ids of inodeMap.values()) { + if (ids.size > 1) { + const arr = [...ids]; + for (let i = 1; i < arr.length; i++) union(arr[0], arr[i]); + } + } + + const groupMembers = new Map(); // groupId -> [candidateId] + for (const c of candidates) { + const g = find(c.id); + if (!groupMembers.has(g)) groupMembers.set(g, []); + groupMembers.get(g).push(c.id); + } + + const groupSizeBytes = new Map(); + for (const [g, ids] of groupMembers) { + const seenInode = new Set(); + let total = 0; + for (const id of ids) { + for (const f of filesByCandidate.get(id)) { + const key = `${f.dev}:${f.ino}`; + if (seenInode.has(key)) continue; + seenInode.add(key); + total += f.size; + } + } + groupSizeBytes.set(g, total); + } + + return { groupOf: id => find(id), groupMembers, groupSizeBytes, filesByCandidate }; +} + +// --------------------------------------------------------------------------- +// Disk free space +// --------------------------------------------------------------------------- + +/** Free bytes on the filesystem containing `path`, via `df -Pk`. Returns + * null if `df` is unavailable or unparseable (caller should omit the + * before/after line rather than print a wrong number). */ +export function diskFreeBytes(path) { + const res = spawnSync('df', ['-Pk', path], { encoding: 'utf8', timeout: 10000 }); + if (res.error || res.status !== 0) return null; + const lines = res.stdout.trim().split('\n'); + if (lines.length < 2) return null; + const cols = lines[lines.length - 1].trim().split(/\s+/); + const availKb = Number(cols[3]); + if (!Number.isFinite(availKb)) return null; + return availKb * 1024; +} + +// --------------------------------------------------------------------------- +// Plan +// --------------------------------------------------------------------------- + +export function planRun(options = {}) { + const home = options.home || homedir(); + const minIdleDays = options.minIdleDays ?? 14; + const maxGb = options.maxGb ?? Infinity; + const keepEntries = options.keepEntries || loadKeepList(options.keepFile, home); + const listProcessUsers = options.listProcessUsers || findProcessUsers; + const listDockerBindUsers = options.listDockerBindUsers || findDockerBindUsers; + const getDiskFreeBytes = options.diskFreeBytes || diskFreeBytes; + const nowMs = options.now ? options.now.getTime() : Date.now(); + const hfCacheRoot = join(home, '.cache', 'huggingface'); + + const rawCandidates = options.candidates || discoverCandidates(home); + const { groupOf, groupMembers, groupSizeBytes, filesByCandidate } = computeHardlinkGroups(rawCandidates); + + const perCandidate = new Map(); + for (const c of rawCandidates) { + let skipReason = null; + + if (matchesKeepList(c.path, keepEntries)) { + skipReason = 'on KEEP list'; + } else if (isSymlink(c.path)) { + skipReason = 'already a symlink (tidied)'; + } else { + const procResult = listProcessUsers(c.path); + const fdHit = (procResult.users || []).find(u => u.via === 'fd'); + const cmdHit = (procResult.users || []).find(u => u.via === 'cmdline' && u.servingProcess); + const hit = fdHit || cmdHit; + if (hit) { + skipReason = hit.via === 'fd' + ? `in use: pid ${hit.pid} has a file open under this path` + : `in use: pid ${hit.pid} serving process references this path (${hit.cmdline})`; + } else { + const dockerResult = listDockerBindUsers(c.path); + if (dockerResult.available === false) { + if (c.path === hfCacheRoot || c.path.startsWith(hfCacheRoot + sep)) { + skipReason = `docker not readable (${dockerResult.reason}); treating ~/.cache/huggingface as in-use, fail-safe`; + } + } else if (dockerResult.inUse) { + const first = dockerResult.containers[0]; + skipReason = `in use: bind-mounted into running container ${first.container} (${first.source} -> ${first.destination})`; + } + if (!skipReason) { + const files = filesByCandidate.get(c.id); + const newest = newestMtimeMs(c.path, files); + const ageDays = (nowMs - newest) / 86400000; + if (ageDays < minIdleDays) { + skipReason = `modified ${ageDays.toFixed(1)}d ago, newer than --min-idle-days ${minIdleDays}`; + } + } + } + } + + perCandidate.set(c.id, { + id: c.id, + path: c.path, + kind: c.kind, + sizeBytes: dirSizeBytes(c.path, filesByCandidate.get(c.id)), + skipReason + }); + } + + const results = []; + for (const [gid, memberIds] of groupMembers) { + const members = memberIds.map(id => perCandidate.get(id)); + const failing = members.find(m => m.skipReason); + const isHardlinkSet = memberIds.length > 1; + if (failing) { + for (const m of members) { + results.push({ + ...m, + groupId: gid, + groupSizeBytes: groupSizeBytes.get(gid), + selected: false, + reason: isHardlinkSet + ? `hardlink set with ${failing.path}, which is skipped: ${failing.skipReason}` + : m.skipReason + }); + } + } else { + for (const m of members) { + results.push({ + ...m, + groupId: gid, + groupSizeBytes: groupSizeBytes.get(gid), + selected: true, + reason: isHardlinkSet ? 'idle (hardlink set, all members idle)' : 'idle' + }); + } + } + } + + // Cap by --max-gb: whole groups only, largest first. + const maxBytes = maxGb === Infinity ? Infinity : maxGb * 2 ** 30; + const idleGroupIds = [...new Set(results.filter(r => r.selected).map(r => r.groupId))] + .sort((a, b) => groupSizeBytes.get(b) - groupSizeBytes.get(a)); + let runningTotal = 0; + const cappedGroupIds = new Set(); + for (const gid of idleGroupIds) { + const size = groupSizeBytes.get(gid); + if (runningTotal + size <= maxBytes) { + runningTotal += size; + } else { + cappedGroupIds.add(gid); + } + } + for (const r of results) { + if (r.selected && cappedGroupIds.has(r.groupId)) { + r.selected = false; + r.reason = `deferred: --max-gb ${maxGb} cap reached for this run`; + } + } + + results.sort((a, b) => b.groupSizeBytes - a.groupSizeBytes || a.path.localeCompare(b.path)); + + const selected = results.filter(r => r.selected); + const skipped = results.filter(r => !r.selected); + const selectedGroupIds = new Set(selected.map(r => r.groupId)); + const totalSelectedBytes = [...selectedGroupIds].reduce((sum, gid) => sum + groupSizeBytes.get(gid), 0); + + const freeBeforeBytes = getDiskFreeBytes(home); + const freeAfterEstimateBytes = freeBeforeBytes == null ? null : freeBeforeBytes + totalSelectedBytes; + + const freeLine = freeBeforeBytes == null + ? 'free before/after: unknown (df unavailable)' + : `free before/after: ${(freeBeforeBytes / 2 ** 30).toFixed(1)} GiB -> ~${(freeAfterEstimateBytes / 2 ** 30).toFixed(1)} GiB`; + + const summaryLine = `model-tidy plan: ${selected.length} dir(s) in ${selectedGroupIds.size} unit(s), ` + + `${(totalSelectedBytes / 2 ** 30).toFixed(1)} GiB movable to target, ${skipped.length} skipped. ${freeLine}`; + + return { + home, + minIdleDays, + maxGb, + selected, + skipped, + totalSelectedBytes, + freeBeforeBytes, + freeAfterEstimateBytes, + freeLine, + summaryLine, + generatedAt: new Date(nowMs).toISOString() + }; +} + +// --------------------------------------------------------------------------- +// Apply +// --------------------------------------------------------------------------- + +function sha256File(path) { + const hash = createHash('sha256'); + hash.update(readFileSync(path)); + return hash.digest('hex'); +} + +/** + * Pure-Node recursive copy of one or more source directories into + * `targetRoot`, preserving relative-to-`home` layout, preserving symlinks + * verbatim, and recreating hardlinks *within this call* (a shared + * `inodeToTargetPath` map across all `sourceAbsPaths`) so a hardlinked unit + * stays a hardlinked unit at the destination instead of doubling in size. + */ +export function copyUnitPureNode(sourceAbsPaths, home, targetRoot) { + const inodeToTargetPath = new Map(); + + function copyDir(srcDir, dstDir) { + mkdirSync(dstDir, { recursive: true }); + for (const entry of readdirSync(srcDir, { withFileTypes: true })) { + const s = join(srcDir, entry.name); + const d = join(dstDir, entry.name); + const lst = lstatSync(s); + if (lst.isSymbolicLink()) { + symlinkSync(readlinkSync(s), d); + } else if (lst.isDirectory()) { + copyDir(s, d); + } else if (lst.isFile()) { + const key = `${lst.dev}:${lst.ino}`; + if (inodeToTargetPath.has(key)) { + linkSync(inodeToTargetPath.get(key), d); + } else { + copyFileSync(s, d); + inodeToTargetPath.set(key, d); + } + } + } + } + + for (const src of sourceAbsPaths) { + const rel = relative(home, src); + const dst = join(targetRoot, rel); + copyDir(src, dst); + } +} + +/** Byte-for-byte verification: same set of relative paths, same symlink + * targets, same file sizes AND sha256 for every regular file. */ +export function verifyUnit(sourceAbsPaths, home, targetRoot) { + const mismatches = []; + for (const src of sourceAbsPaths) { + const rel = relative(home, src); + const dst = join(targetRoot, rel); + const srcFiles = listAllEntries(src); + for (const s of srcFiles) { + const relToUnit = relative(src, s); + const d = join(dst, relToUnit); + const lst = lstatSync(s); + if (!existsSync(d)) { + mismatches.push({ path: s, reason: 'missing at target', target: d }); + continue; + } + const dlst = lstatSync(d); + if (lst.isSymbolicLink()) { + if (!dlst.isSymbolicLink() || readlinkSync(s) !== readlinkSync(d)) { + mismatches.push({ path: s, reason: 'symlink target mismatch', target: d }); + } + } else if (lst.isFile()) { + if (!dlst.isFile()) { + mismatches.push({ path: s, reason: 'not a regular file at target', target: d }); + continue; + } + if (lst.size !== dlst.size) { + mismatches.push({ path: s, reason: `size mismatch (${lst.size} vs ${dlst.size})`, target: d }); + continue; + } + const srcHash = sha256File(s); + const dstHash = sha256File(d); + if (srcHash !== dstHash) { + mismatches.push({ path: s, reason: 'checksum mismatch', target: d }); + } + } + } + } + return { ok: mismatches.length === 0, mismatches }; +} + +function listAllEntries(root) { + const out = []; + function recurse(dir) { + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const full = join(dir, entry.name); + out.push(full); + let st; + try { + st = lstatSync(full); + } catch { + continue; + } + if (st.isDirectory() && !st.isSymbolicLink()) recurse(full); + } + } + recurse(root); + return out; +} + +/** Validate --target: must exist, be a directory, be writable, and be on a + * different filesystem device than `home`. */ +export function validateTarget(target, home) { + if (!target) return { ok: false, error: '--target is required with --apply' }; + if (!existsSync(target)) return { ok: false, error: `--target ${target} does not exist` }; + const st = statSync(target); + if (!st.isDirectory()) return { ok: false, error: `--target ${target} is not a directory` }; + try { + accessSync(target, FS_CONSTANTS.W_OK); + } catch { + return { ok: false, error: `--target ${target} is not writable` }; + } + const homeSt = statSync(home); + if (st.dev === homeSt.dev) { + return { ok: false, error: `--target ${target} is on the same filesystem (dev ${st.dev}) as ${home}; refusing (must be a different mount)` }; + } + return { ok: true }; +} + +/** + * Apply a plan: move every selected unit to `target`, unit by unit. Each + * unit is copied, verified, and only THEN does its source directory get + * replaced with a symlink — so a failure at any step for a unit leaves that + * unit's source completely untouched. One unit's failure does not roll back + * units that already succeeded; the run still exits non-zero overall. + */ +export function applyRun(options) { + const { plan, target, home } = options; + const copyFn = options.copyFn || copyUnitPureNode; + const verifyFn = options.verifyFn || verifyUnit; + const validateTargetFn = options.validateTarget || validateTarget; + + const validation = validateTargetFn(target, home); + if (!validation.ok) { + return { ok: false, error: validation.error, moved: [], errors: [{ error: validation.error }] }; + } + + const byGroup = new Map(); + for (const r of plan.selected) { + if (!byGroup.has(r.groupId)) byGroup.set(r.groupId, []); + byGroup.get(r.groupId).push(r); + } + + const moved = []; + const errors = []; + + for (const [groupId, members] of byGroup) { + const sourceAbsPaths = members.map(m => m.path); + try { + copyFn(sourceAbsPaths, home, target); + } catch (e) { + errors.push({ groupId, paths: sourceAbsPaths, step: 'copy', error: e.message }); + continue; + } + + const verification = verifyFn(sourceAbsPaths, home, target); + if (!verification.ok) { + errors.push({ groupId, paths: sourceAbsPaths, step: 'verify', error: 'checksum/size verification failed', mismatches: verification.mismatches }); + continue; // source untouched — verification failed BEFORE any deletion + } + + // Verification passed for the whole unit: safe to swap every member. + let swapFailed = null; + const swapped = []; + for (const src of sourceAbsPaths) { + const rel = relative(home, src); + const dst = join(target, rel); + try { + rmSync(src, { recursive: true }); + symlinkSync(dst, src); + const real = realpathSync(src); + if (real !== realpathSync(dst) || !lstatSync(src).isSymbolicLink()) { + throw new Error(`post-symlink verification failed for ${src}`); + } + swapped.push({ source: src, target: dst }); + } catch (e) { + swapFailed = { path: src, error: e.message }; + break; + } + } + if (swapFailed) { + errors.push({ groupId, paths: sourceAbsPaths, step: 'swap', error: swapFailed.error, partiallySwapped: swapped }); + } else { + moved.push({ groupId, members: swapped, sizeBytes: members[0].groupSizeBytes }); + } + } + + return { ok: errors.length === 0, moved, errors }; +} + +// --------------------------------------------------------------------------- +// JSON run log +// --------------------------------------------------------------------------- + +export function writeRunLog(logDir, record) { + mkdirSync(logDir, { recursive: true }); + const line = JSON.stringify(record) + '\n'; + const file = join(logDir, `${new Date().toISOString().slice(0, 10)}.jsonl`); + appendFileSync(file, line); + return file; +} diff --git a/systemd/model-tidy.service b/systemd/model-tidy.service new file mode 100644 index 0000000..c316fdb --- /dev/null +++ b/systemd/model-tidy.service @@ -0,0 +1,20 @@ +[Unit] +Description=model-tidy: plan (and optionally apply) idle-model cleanup +Wants=network-online.target +After=docker.service + +[Service] +Type=oneshot +# PLAN MODE ONLY by default — never moves, deletes, or symlinks anything. +# To let the nightly timer actually move models, edit this line to add +# "apply --apply --target /mnt/your-target" AFTER you have reviewed several +# plan runs' output/logs yourself. Do not do this unattended on day one. +ExecStart=/usr/bin/env node %h/ide-agent-kit/bin/model-tidy.mjs plan +# Room reporting is opt-in and OFF here by default. To turn it on, add: +# --report-to-room --room thinkoff-development --config %h/ide-agent-kit/ide-agent-kit.json +# to the ExecStart line above. +Nice=19 +IOSchedulingClass=idle + +[Install] +WantedBy=default.target diff --git a/systemd/model-tidy.timer b/systemd/model-tidy.timer new file mode 100644 index 0000000..b212e4a --- /dev/null +++ b/systemd/model-tidy.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Nightly model-tidy plan run + +[Timer] +OnCalendar=*-*-* 03:30:00 +RandomizedDelaySec=900 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/test/model-tidy.test.mjs b/test/model-tidy.test.mjs new file mode 100644 index 0000000..106d99b --- /dev/null +++ b/test/model-tidy.test.mjs @@ -0,0 +1,381 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +import { describe, it, afterEach } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { + mkdtempSync, mkdirSync, writeFileSync, symlinkSync, linkSync, rmSync, + existsSync, readFileSync, utimesSync, lstatSync, readdirSync +} from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { + planRun, applyRun, discoverCandidates, loadKeepList, computeHardlinkGroups, + validateTarget, copyUnitPureNode +} from '../src/model-tidy.mjs'; + +const tempPaths = []; + +afterEach(() => { + while (tempPaths.length > 0) { + const path = tempPaths.pop(); + rmSync(path, { recursive: true, force: true }); + } +}); + +function tempDir(prefix) { + const dir = mkdtempSync(join(tmpdir(), prefix)); + tempPaths.push(dir); + return dir; +} + +function writeFile(path, content = 'x'.repeat(1024)) { + mkdirSync(join(path, '..'), { recursive: true }); + writeFileSync(path, content); +} + +function daysAgo(n) { + return new Date(Date.now() - n * 86400000); +} + +function touch(path, when) { + utimesSync(path, when, when); +} + +/** + * Build a fixture home tree: + * .cache/huggingface/hub/models--idle-model/ <- idle HF cache model (selected) + * blobs/ real file, old mtime + * snapshots/rev1/file.safetensors -> ../../blobs/ symlink + * .cache/huggingface/hub/models--recent-model/ <- too-recent, skipped + * .cache/huggingface/hub/models--already-tidied <- already a symlink, skipped + * models/keep-me/ <- KEEP-listed, skipped + * models/hardlink-a/file.bin <-hardlinked-> models/hardlink-b/file.bin + * (both old mtime; both must be selected together, as one unit) + */ +function buildFixture() { + const home = tempDir('model-tidy-fixture-'); + const old = daysAgo(30); + const recent = daysAgo(1); + + // idle HF cache model + const idleRoot = join(home, '.cache/huggingface/hub/models--idle-model'); + const blobPath = join(idleRoot, 'blobs', 'abc123'); + writeFile(blobPath, 'idle-model-bytes'.repeat(100)); + touch(blobPath, old); + mkdirSync(join(idleRoot, 'snapshots', 'rev1'), { recursive: true }); + symlinkSync(join('..', '..', 'blobs', 'abc123'), join(idleRoot, 'snapshots', 'rev1', 'model.safetensors')); + + // too-recent HF cache model + const recentRoot = join(home, '.cache/huggingface/hub/models--recent-model'); + const recentBlob = join(recentRoot, 'blobs', 'def456'); + writeFile(recentBlob, 'recent-model-bytes'.repeat(100)); + touch(recentBlob, recent); + + // already-tidied: a symlink standing in for a models--* dir + const tidiedTargetDir = join(home, '.cache', 'elsewhere-target'); + mkdirSync(tidiedTargetDir, { recursive: true }); + writeFile(join(tidiedTargetDir, 'blobs', 'file'), 'already moved'); + symlinkSync(tidiedTargetDir, join(home, '.cache/huggingface/hub', 'models--already-tidied')); + + // KEEP-listed dir under ~/models + const keepRoot = join(home, 'models', 'keep-me'); + writeFile(join(keepRoot, 'weights.gguf'), 'keep-me-bytes'.repeat(100)); + touch(join(keepRoot, 'weights.gguf'), old); + + // hardlink pair under ~/models — must move together or not at all + const hardlinkA = join(home, 'models', 'hardlink-a'); + const hardlinkB = join(home, 'models', 'hardlink-b'); + mkdirSync(hardlinkA, { recursive: true }); + mkdirSync(hardlinkB, { recursive: true }); + writeFile(join(hardlinkA, 'file.bin'), 'shared-hardlinked-bytes'.repeat(100)); + touch(join(hardlinkA, 'file.bin'), old); + linkSync(join(hardlinkA, 'file.bin'), join(hardlinkB, 'file.bin')); + + const keepFile = join(home, 'model-tidy.keep'); + writeFileSync(keepFile, `${keepRoot}\n`); + + return { home, keepFile, idleRoot, recentRoot, keepRoot, hardlinkA, hardlinkB }; +} + +const noProcessUsers = () => ({ checked: true, users: [] }); +const dockerNotInUse = () => ({ available: true, inUse: false, containers: [] }); +const fixedDiskFree = () => 100 * 2 ** 30; + +describe('discoverCandidates', () => { + it('finds hf-cache, models-dir entries', () => { + const { home } = buildFixture(); + const candidates = discoverCandidates(home); + const paths = candidates.map(c => c.path); + assert.ok(paths.some(p => p.endsWith('models--idle-model'))); + assert.ok(paths.some(p => p.endsWith('models--recent-model'))); + assert.ok(paths.some(p => p.endsWith('models--already-tidied'))); + assert.ok(paths.some(p => p.endsWith('keep-me'))); + assert.ok(paths.some(p => p.endsWith('hardlink-a'))); + assert.ok(paths.some(p => p.endsWith('hardlink-b'))); + }); +}); + +describe('loadKeepList', () => { + it('parses non-blank, non-comment lines', () => { + const dir = tempDir('model-tidy-keep-'); + const f = join(dir, 'keep'); + writeFileSync(f, '# comment\n\n/some/path\n~/models/x\n'); + const entries = loadKeepList(f, '/home/petrus'); + assert.deepEqual(entries, ['/some/path', '/home/petrus/models/x']); + }); + + it('returns [] for a missing file', () => { + assert.deepEqual(loadKeepList('/no/such/file', '/home/petrus'), []); + }); +}); + +describe('computeHardlinkGroups', () => { + it('groups candidates that share an inode across different roots', () => { + const { home, hardlinkA, hardlinkB, idleRoot } = buildFixture(); + const candidates = [ + { id: hardlinkA, path: hardlinkA }, + { id: hardlinkB, path: hardlinkB }, + { id: idleRoot, path: idleRoot } + ]; + const { groupOf } = computeHardlinkGroups(candidates); + assert.equal(groupOf(hardlinkA), groupOf(hardlinkB)); + assert.notEqual(groupOf(hardlinkA), groupOf(idleRoot)); + }); +}); + +describe('planRun', () => { + it('selects exactly the idle dir and the complete hardlink set, with correct skip reasons for everything else', () => { + const { home, keepFile, idleRoot, recentRoot, keepRoot, hardlinkA, hardlinkB } = buildFixture(); + + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + + const selectedPaths = plan.selected.map(r => r.path).sort(); + assert.deepEqual(selectedPaths, [hardlinkA, hardlinkB, idleRoot].sort()); + + const byPath = new Map(plan.skipped.map(r => [r.path, r])); + + const tidied = [...byPath.keys()].find(p => p.endsWith('models--already-tidied')); + assert.ok(tidied, 'already-tidied dir should be in skipped list'); + assert.match(byPath.get(tidied).reason, /already a symlink/); + + const recent = [...byPath.keys()].find(p => p === recentRoot); + assert.ok(recent, 'recent dir should be in skipped list'); + assert.match(byPath.get(recent).reason, /min-idle-days/); + + const keep = [...byPath.keys()].find(p => p === keepRoot); + assert.ok(keep, 'keep-listed dir should be in skipped list'); + assert.match(byPath.get(keep).reason, /KEEP list/); + + assert.equal(plan.selected.length, 3); + assert.equal(plan.skipped.length, 3); + }); + + it('skips a whole hardlink set if either member is not idle', () => { + const { home, keepFile, hardlinkA, hardlinkB } = buildFixture(); + touch(join(hardlinkB, 'file.bin'), daysAgo(1)); // make B recent + + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + + const selectedPaths = plan.selected.map(r => r.path); + assert.ok(!selectedPaths.includes(hardlinkA)); + assert.ok(!selectedPaths.includes(hardlinkB)); + const skippedA = plan.skipped.find(r => r.path === hardlinkA); + assert.match(skippedA.reason, /hardlink set/); + }); + + it('treats ~/.cache/huggingface as in-use when docker is unreadable', () => { + const { home, keepFile, idleRoot } = buildFixture(); + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: () => ({ available: false, inUse: false, reason: 'permission denied' }), + diskFreeBytes: fixedDiskFree + }); + const skippedIdle = plan.skipped.find(r => r.path === idleRoot); + assert.ok(skippedIdle, 'HF cache dir should be fail-safe skipped when docker is unreadable'); + assert.match(skippedIdle.reason, /docker not readable/); + }); + + it('skips a dir with an open file handle (simulated process check)', () => { + const { home, keepFile, idleRoot } = buildFixture(); + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: (path) => (path === idleRoot + ? { checked: true, users: [{ pid: '4242', via: 'fd', target: join(path, 'blobs/abc123') }] } + : { checked: true, users: [] }), + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + const skipped = plan.skipped.find(r => r.path === idleRoot); + assert.ok(skipped); + assert.match(skipped.reason, /in use: pid 4242/); + }); + + it('skips a dir bind-mounted into a running container', () => { + const { home, keepFile, idleRoot } = buildFixture(); + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: (path) => (path === idleRoot + ? { available: true, inUse: true, containers: [{ container: 'vllm-serve', source: path, destination: '/root/.cache/huggingface' }] } + : { available: true, inUse: false, containers: [] }), + diskFreeBytes: fixedDiskFree + }); + const skipped = plan.skipped.find(r => r.path === idleRoot); + assert.ok(skipped); + assert.match(skipped.reason, /bind-mounted into running container vllm-serve/); + }); + + it('caps total moved bytes by --max-gb, deferring smaller units', () => { + const { home, keepFile } = buildFixture(); + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + maxGb: 0, // too small for anything + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + assert.equal(plan.selected.length, 0); + const deferred = plan.skipped.filter(r => /max-gb.*cap/.test(r.reason)); + assert.ok(deferred.length > 0); + }); +}); + +// tmpdir()-based fixtures and their "target" dir are normally on the SAME +// filesystem on a dev box, so validateTarget()'s cross-filesystem refusal +// (tested for real below, with no mocking) would otherwise block every +// apply test here. The apply-flow tests inject a bypass for that one check +// so they can exercise copy/verify/symlink-swap in isolation; the +// cross-filesystem rule itself is covered by its own unmocked test. +const bypassCrossFsCheck = () => ({ ok: true }); + +describe('validateTarget', () => { + it('refuses a target on the same filesystem as the source (real check, no mocking)', () => { + const home = tempDir('model-tidy-home-'); + const target = tempDir('model-tidy-target-'); // same tmpfs as home on a dev box + const result = validateTarget(target, home); + assert.equal(result.ok, false); + assert.match(result.error, /same filesystem/); + }); + + it('refuses a target that does not exist', () => { + const home = tempDir('model-tidy-home-'); + const result = validateTarget(join(home, 'nonexistent'), home); + assert.equal(result.ok, false); + }); +}); + +describe('applyRun', () => { + it('copies, verifies, and replaces the source with a symlink for every selected unit', () => { + const { home, keepFile, idleRoot, hardlinkA, hardlinkB } = buildFixture(); + const target = tempDir('model-tidy-target-'); + + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + + const result = applyRun({ plan, target, home, validateTarget: bypassCrossFsCheck }); + + assert.equal(result.ok, true, JSON.stringify(result.errors)); + assert.equal(result.moved.length, 2); // idle-model unit + hardlink unit + + for (const root of [idleRoot, hardlinkA, hardlinkB]) { + assert.ok(lstatSync(root).isSymbolicLink(), `${root} should now be a symlink`); + } + + // Content is reachable through the symlink and byte-identical. + const blobViaSymlink = readdirSync(join(idleRoot, 'blobs'))[0]; + assert.ok(existsSync(join(idleRoot, 'blobs', blobViaSymlink))); + + // The hardlink relationship survived the move: both copies at the + // target still share one inode (moving them independently would have + // doubled disk usage, which is exactly what this guards against). + const aStat = lstatSync(join(hardlinkA, 'file.bin')); + const bStat = lstatSync(join(hardlinkB, 'file.bin')); + assert.equal(aStat.ino, bStat.ino); + assert.ok(aStat.nlink >= 2); + }); + + it('negative control: a corrupted target file makes apply refuse and leaves the source untouched', () => { + const { home, keepFile, idleRoot } = buildFixture(); + const target = tempDir('model-tidy-target-'); + + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + // Isolate to just the idle-model unit for a focused assertion. + plan.selected = plan.selected.filter(r => r.path === idleRoot); + + // Copy for real, then simulate corruption discovered after the copy + // (e.g. a bad transfer / bitrot) by overwriting the blob at the target. + function corruptAfterCopy(sourceAbsPaths, homeDir, targetRoot) { + copyUnitPureNode(sourceAbsPaths, homeDir, targetRoot); + for (const src of sourceAbsPaths) { + const rel = src.slice(homeDir.length + 1); + const targetBlobsDir = join(targetRoot, rel, 'blobs'); + if (existsSync(targetBlobsDir)) { + const [blobName] = readdirSync(targetBlobsDir); + const original = readFileSync(join(targetBlobsDir, blobName), 'utf8'); + // Same length as the original so this exercises the checksum + // comparison specifically, not just the (cheaper) size check. + writeFileSync(join(targetBlobsDir, blobName), 'X'.repeat(original.length)); + } + } + } + + const result = applyRun({ + plan, + target, + home, + validateTarget: bypassCrossFsCheck, + copyFn: corruptAfterCopy + }); + + assert.equal(result.ok, false); + assert.equal(result.moved.length, 0); + assert.equal(result.errors.length, 1); + assert.equal(result.errors[0].step, 'verify'); + assert.ok(result.errors[0].mismatches.some(m => /checksum mismatch/.test(m.reason))); + + // Source must be completely untouched: still a real directory, not a + // symlink, with its original (uncorrupted) content intact. + assert.equal(lstatSync(idleRoot).isSymbolicLink(), false); + assert.ok(existsSync(idleRoot)); + const blobDir = join(idleRoot, 'blobs'); + const [blobName] = readdirSync(blobDir); + const content = readFileSync(join(blobDir, blobName), 'utf8'); + assert.equal(content, 'idle-model-bytes'.repeat(100)); + }); +}); From 0f6b682e919d1d6f7f26e8df917a4b06dafd8ed1 Mon Sep 17 00:00:00 2001 From: Petrus Pennanen Date: Mon, 21 Sep 2026 08:59:46 +0200 Subject: [PATCH 2/9] Fix 3 safety defects from static review: fail-open checks, apply crash window, hardlink scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses an independent static review (codexmb) of commit 104d7c9: 1. FAIL-OPEN in plan. findProcessUsers silently swallowed per-pid permission failures and still reported checked:true; an unreadable docker only fail-safed ~/.cache/huggingface, leaving ~/models/* and ad-hoc dirs selectable. Now: any candidate whose in-use status can't be fully verified (any unreadable pid, /proc unreadable, docker present but not inspectable) is skipped with "in-use status unverified: ", never treated as idle for lack of evidence. An unreadable docker now taints every candidate on the box. The plan summary line reports "N unverified". 2. CRASH WINDOW in apply. The old sequence did rmSync(src) then symlinkSync(dst, src) — a crash between those two calls left the source gone with no symlink. Now: renameSync(src, src+'.tidy-moving') -> symlinkSync(dst, src) -> re-verify -> rmSync(staging). Any failure after the rename removes the half-made symlink and renames staging back to src, so a unit's source ends up either fully in place or fully swapped to a working symlink, never neither. A leftover '.tidy-moving' from a previous interrupted run is detected up front and that unit is refused untouched. 3. HARDLINK SCAN SCOPE. Clarified that computeHardlinkGroups already scans every discovered candidate (KEEP-listed, recent, and in-use dirs included, not just the plain-idle subset), and fixed the skip message to name the actual blocking member ("hardlinked to , which is not moving ('s own reason)") instead of a confusing self-referential wrapper. Added a regression test proving an idle dir hardlinked to a KEEP-listed copy is skipped as a unit. Also, while touching the same code paths: docker bind-mount detection now also catches a mount SOURCE nested under a candidate (not just the reverse), --target must be an absolute path, and the CLI rejects a non-finite/negative --min-idle-days instead of silently disabling the freshness guard via NaN. Both are from the same automated review but outside the three defects above. 6 new tests (fail-closed process/docker checks with a positive control, hardlink-to-KEEP-listed skip, crash-window rollback, leftover-staging refusal) plus one existing test's fixture fixed (it relied on touching a hardlinked file's mtime, which — being the same inode — moved both copies' mtime and no longer exercised what it claimed to). Tests: 19/19 model-tidy (new tests pass standalone and in the full model-tidy run), 678/678 full repo suite. Not done: the streaming-checksum (large-file readFileSync) and lsof-fallback findings from the same review are out of scope for this pass and documented in docs/model-tidy.md's "Not verified" section. Still never run against a real machine. Co-Authored-By: Claude Fable 5.1 --- bin/model-tidy.mjs | 8 ++ docs/model-tidy.md | 123 ++++++++++++++------ src/model-tidy.mjs | 240 +++++++++++++++++++++++++++++---------- test/model-tidy.test.mjs | 211 ++++++++++++++++++++++++++++++++-- 4 files changed, 479 insertions(+), 103 deletions(-) diff --git a/bin/model-tidy.mjs b/bin/model-tidy.mjs index e57ea36..8b14142 100755 --- a/bin/model-tidy.mjs +++ b/bin/model-tidy.mjs @@ -159,7 +159,15 @@ function main() { const home = args.home || homedir(); const keepFile = args['keep-file'] || (existsSync(DEFAULT_KEEP_FILE) ? DEFAULT_KEEP_FILE : undefined); const minIdleDays = args['min-idle-days'] !== undefined ? Number(args['min-idle-days']) : 14; + if (!Number.isFinite(minIdleDays) || minIdleDays < 0) { + console.error(`--min-idle-days must be a finite, non-negative number (got ${JSON.stringify(args['min-idle-days'])}) — refusing, since a NaN here would silently disable the freshness guard`); + process.exit(2); + } const maxGb = args['max-gb'] !== undefined ? Number(args['max-gb']) : Infinity; + if (!Number.isFinite(maxGb) && args['max-gb'] !== undefined) { + console.error(`--max-gb must be a finite number (got ${JSON.stringify(args['max-gb'])})`); + process.exit(2); + } const logDir = args['log-dir'] || DEFAULT_LOG_DIR; const plan = planRun({ home, keepFile, minIdleDays, maxGb }); diff --git a/docs/model-tidy.md b/docs/model-tidy.md index 017df6e..a0cec76 100644 --- a/docs/model-tidy.md +++ b/docs/model-tidy.md @@ -76,48 +76,79 @@ Refuses immediately, before touching anything, unless `--target`: For each selected unit (a single directory, or a whole hardlink set moved together): +0. Refuse if a `.tidy-moving` directory already exists (see step 3) + — that means a previous `apply` was interrupted mid-swap for this unit, + and this run will not guess which of the two paths is authoritative. 1. Copy every file into `--target`, preserving the relative-to-home path, symlinks, and hardlink relationships. 2. Verify every file: same relative paths, same symlink targets, same byte size AND SHA-256 for every regular file. -3. Only if verification passed for every file in the unit: delete the - source directory (`fs.rmSync`, not a shell `rm -rf`) and replace it with - a symlink to the target copy. -4. Re-verify: the symlink resolves (`realpathSync`) to the target copy. - -**Any failure at steps 1-2 leaves the source completely untouched** — the -delete in step 3 only ever runs after verification has already passed for -every file in that unit. A failure for one unit does not roll back units -that already succeeded earlier in the same run, but the process still -exits non-zero and prints exactly which unit failed and why. +3. Only if verification passed for every file in the unit, swap the source + for a symlink via a crash-safe staged sequence: + `rename(source, source + '.tidy-moving')` (atomic on the same + filesystem) → `symlink(target-copy, source)` → verify the symlink + resolves (`realpathSync`) to the target copy → remove + `source + '.tidy-moving'`. +4. If anything in step 3 fails after the rename, the symlink (if partially + created) is removed and `source + '.tidy-moving'` is renamed straight + back to `source`, so the unit ends up exactly as it started. + +**The safety claim is: a failure at any step leaves a unit's source EITHER +with its original completely untouched (any failure in steps 0-2, or a +step-3 failure that rolled back) OR with a working symlink to a verified +copy (a step-3 swap that completed) — never neither, and never deleted +without a verified copy existing both at the target and reachable through +the symlink.** A failure for one unit does not roll back units that already +succeeded earlier in the same run, but the process still exits non-zero and +prints exactly which unit failed, at which step, and why. ## Selection rules (in order, each with an explicit reason string) 1. **KEEP list** — anything matching `--keep-file` (path or glob, `~` expanded, `#` comments) is always skipped. See `config/model-tidy.keep.example`. -2. **In use by a process** — skipped if: +2. **In use by a process, or unverifiable** — skipped if: - any process has one of the model's files open under `/proc/*/fd`, or - a recognized serving process's command line references the path (`vllm`, `llama-server`, `llama.cpp`, `sglang`, `exllama`, `tabby`, `ollama`, `mlx`, `text-generation`). - Off Linux (no `/proc`), this check reports `checked: false` rather than - silently passing — see "Not verified" below. -3. **Bind-mounted into a running docker container** — reads - `docker inspect` of every running container's `Mounts`; if the - candidate path is the mount source (or under it), it's in use. - **Fail-safe:** if `docker` is not installed or not readable, the WHOLE - of `~/.cache/huggingface` is treated as in-use and the run says so — - nothing outside that tree is blanket-skipped by this rule. + **Fail-closed:** this check has to succeed for *every* pid on the box to + count as verified. Off Linux (no `/proc`), if `/proc` itself can't be + listed, or if even one pid's `fd` directory or `cmdline` can't be read + (a permission failure, not the process simply having exited mid-scan — + that's a normal race and not a failure), the candidate is skipped with + `in-use status unverified: ` rather than treated as idle for lack + of evidence. In practice, on a typical non-root Linux host with other + users' or root's processes running, this makes the tool quite + conservative unless it runs with enough privilege to read every pid's + `/proc` entry — that is intentional: an unreadable process is exactly + the case where we cannot prove a model is idle. +3. **Bind-mounted into (or containing) the bind-mount source of a running + docker container, or unverifiable** — reads `docker inspect` of every + running container's `Mounts`; a candidate is in use if it is at or under + a mount source, OR if a mount source is at or under the candidate (a + container mounting a subdirectory of a larger candidate still makes + that whole candidate unsafe to move). **Fail-closed:** if `docker` is + not installed, not running, or its `ps`/`inspect` output can't be read, + EVERY candidate on the box is skipped with `in-use status unverified: + docker not readable: ` — not just `~/.cache/huggingface`. A + container can bind-mount anything; only being able to enumerate every + running container's mounts makes any candidate provably idle. 4. **Too recent** — skipped if the newest mtime of any real file in the - directory is within `--min-idle-days` (default 14). + directory is within `--min-idle-days` (default 14). The CLI rejects a + non-finite or negative `--min-idle-days` up front (e.g. a typo like + `--min-idle-days fourteen`) rather than let a stray `NaN` silently + disable this check. 5. **Already tidied** — skipped if the candidate is already a symlink. 6. **Hardlink sets move as one unit** — candidates are grouped by shared - `dev:ino` across different candidate roots. A group is selected only if + `dev:ino`, scanning every discovered model root regardless of its own + KEEP/in-use/recent status (so a hardlink to a KEEP-listed copy, or to a + directory that's in use, is still detected). A group is selected only if *every* member independently passed rules 1-5; otherwise every member is - skipped with a reason naming which member failed and why (moving one - copy of a hardlinked pair to another filesystem breaks the hardlink and - doubles disk use — this is the whole point of the rule). + skipped with `hardlinked to , which is not moving ('s own + reason)` (moving one copy of a hardlinked pair to another filesystem + breaks the hardlink and doubles disk use — this is the whole point of + the rule). Remaining candidates are sorted by size (desc) and capped by `--max-gb` (whole units only — a unit is either fully included in this run's budget or @@ -182,14 +213,23 @@ it never attempts to install anything itself. - `plan` mode makes zero filesystem writes anywhere. It's the default. - `apply` requires both `--apply` AND `--target ` — neither alone is enough. -- `--target` must be an existing, writable directory on a different - filesystem device than `--home`, or apply refuses before touching - anything. -- A source directory is only ever deleted after its copy has been verified - byte-for-byte (size + SHA-256) at the target. There is no code path that - deletes before verifying. -- Deletion uses `fs.rmSync` (Node), never a shell `rm -rf`. -- A hardlink set moves as a unit or not at all — never partially. +- `--target` must be an absolute, existing, writable directory on a + different filesystem device than `--home`, or apply refuses before + touching anything. +- A source directory's original copy is only ever removed after (a) its + copy has been verified byte-for-byte (size + SHA-256) at the target, and + (b) the replacement symlink at the source has itself been created and + verified to resolve to that copy. There is no code path that removes the + original before both of those have happened — see the staged + rename/symlink/verify/cleanup sequence above. A crash between the rename + and the final cleanup leaves a `.tidy-moving` directory, which a + later `apply` run detects and refuses to touch until it's resolved by + hand. +- A hardlink set moves as a unit or not at all — never partially — and the + hardlink scan covers every discovered model root, not just the ones that + already passed every other rule. +- Process- and docker-in-use checks are fail-closed: anything that could + not be fully verified is treated as in-use, never as idle. - KEEP-listed paths are never touched by either mode. - `--report-to-room` and any nightly `apply` are both opt-in and off by default. @@ -222,3 +262,22 @@ it never attempts to install anything itself. - No real hardlinked pair between `~/models/*` and the HF cache has been inspected on either box — the fixture in `test/model-tidy.test.mjs` constructs a synthetic one via `fs.linkSync`. +- **Practical consequence of the fail-closed process check, not verified + on a real box:** on a typical multi-process Linux host, at least some + pids (root's, other users', kernel-adjacent processes) will be + unreadable to a non-root `model-tidy` process. As written, that makes + EVERY run report every non-KEEP/non-tidied candidate as unverified unless + `model-tidy` runs with enough privilege to read every pid's `/proc` + entry, or unless asus1/asus2 turn out to be single-user boxes where + `model-tidy`'s own user can read every relevant pid. This has not been + checked against the real process list on either box, so it's unknown + whether the tool would select anything at all there today. +- Two findings from the automated Codex review are known and NOT addressed + in this pass (out of scope for the three defects above, tracked here + instead of silently dropped): (1) `verifyUnit`'s checksum step + (`sha256File`) reads each file whole via `readFileSync` rather than + streaming — for real multi-GiB `.safetensors`/`.gguf` shards this could + exhaust memory or exceed Node's Buffer limits, making `apply` fail at + the verify step for large real models even though the copy itself + succeeded; (2) there is no `lsof`-based fallback for the process-in-use + check, only `/proc`. diff --git a/src/model-tidy.mjs b/src/model-tidy.mjs index 734dc81..24d1e8a 100644 --- a/src/model-tidy.mjs +++ b/src/model-tidy.mjs @@ -10,9 +10,13 @@ * candidate is skipped. Never touches disk. * apply — only runs with --apply AND --target . Copies * (verified byte-for-byte), then replaces the source - * directory with a symlink to the copy, then re-verifies - * the symlink resolves. Any failure at any step leaves - * the source untouched. + * directory with a symlink to the copy via a crash-safe + * rename/symlink/verify/cleanup sequence (swapToSymlink). + * A failure at any step leaves the unit EITHER with its + * original untouched (copy/verify failures, or a swap + * failure that rolled back) OR with a working symlink + * to a verified copy (a swap that completed) — never + * neither, and never a bare deletion with no symlink. * * Zero external dependencies (Node >= 18 only), matching the rest of * ide-agent-kit. The copy step is a small hand-written recursive copy @@ -21,18 +25,23 @@ * a plain recursive copy (or `cp -a` without `-H` semantics) would silently * double disk usage for a hardlinked model. See copyUnitPureNode(). * - * Selection rules run in this order, each producing an explicit reason: + * Selection rules run in this order, each producing an explicit reason. + * Rules 2 and 3 are FAIL-CLOSED: if either check cannot be fully completed + * for a candidate, it is skipped as unverified rather than treated as idle + * on absence of evidence. * 1. KEEP list match -> always skipped - * 2. open by a process, or referenced by a -> skipped, "in use" - * known serving process's command line - * 3. under the bind-mount source of a RUNNING -> skipped, "in use" - * docker container (fail-safe: if docker is - * unreadable, ~/.cache/huggingface is treated - * as in-use) + * 2. open by a process, or referenced by a -> skipped, "in use", or + * known serving process's command line "in-use status unverified" + * if /proc can't be fully read for every pid if unverifiable + * 3. under (or containing) the bind-mount source -> skipped, "in use", or + * of a RUNNING docker container; if docker is "in-use status unverified" + * unreadable, EVERY candidate on the box is if docker is unreadable + * unverified, not just the HF cache * 4. newest mtime within --min-idle-days -> skipped, "too recent" * 5. already a symlink (previously tidied) -> skipped, "already tidied" - * 6. hardlink sets move as one unit: every member -> skipped, "hardlink set" - * must be a candidate or the whole set is skipped + * 6. hardlink sets move as one unit: every member -> skipped, "hardlinked to + * must independently clear rules 1-5 or the , which is not moving" + * whole set is skipped * * Remaining candidates are sorted by size (desc) and capped by --max-gb. */ @@ -41,9 +50,9 @@ import { spawnSync } from 'node:child_process'; import { readdirSync, lstatSync, existsSync, readFileSync, readlinkSync, realpathSync, symlinkSync, rmSync, mkdirSync, copyFileSync, linkSync, - statSync, appendFileSync, constants as FS_CONSTANTS, accessSync + statSync, appendFileSync, constants as FS_CONSTANTS, accessSync, renameSync } from 'node:fs'; -import { join, relative, sep } from 'node:path'; +import { join, relative, sep, isAbsolute } from 'node:path'; import { createHash } from 'node:crypto'; import { homedir } from 'node:os'; @@ -265,9 +274,17 @@ export function matchesKeepList(path, keepEntries) { /** * Detect processes using a path: (a) ANY process with the path open under * /proc/*\/fd, (b) a recognized serving process (vllm, ollama, ...) whose - * command line references the path. Degrades to `{checked:false}` off Linux - * or without /proc read access — callers should treat that as "could not - * verify" rather than "confirmed idle". + * command line references the path. + * + * Returns `{checked:false}` whenever the check could NOT be completed for + * every process on the box — off Linux (no /proc), if /proc itself can't be + * listed, or if even a single pid's fd directory or cmdline is unreadable + * (a permission failure, not the process having simply exited mid-scan, + * which is expected and not a failure). A permission failure on ONE pid + * means we cannot rule out THAT pid holding any of our candidates open, so + * it taints the whole result rather than being silently skipped — callers + * MUST treat `checked:false` as "could not verify", never as "confirmed + * idle". */ export function findProcessUsers(path, opts = {}) { const procRoot = opts.procRoot || '/proc'; @@ -281,10 +298,23 @@ export function findProcessUsers(path, opts = {}) { return { checked: false, users: [], note: `cannot list ${procRoot}: ${e.message}` }; } const users = []; + const unreadablePids = []; for (const pid of pids) { + let fdOk = true; try { const fdDir = join(procRoot, pid, 'fd'); - for (const fd of readdirSync(fdDir)) { + let fds; + try { + fds = readdirSync(fdDir); + } catch (e) { + // ENOENT here means the process exited between the pid listing and + // this read — a benign race, not a verification failure. Anything + // else (EACCES/EPERM, or an unexpected error) means we genuinely + // could not check this pid's open files. + if (e.code !== 'ENOENT') fdOk = false; + fds = []; + } + for (const fd of fds) { try { const target = realpathSync(join(fdDir, fd)); if (target === path || target.startsWith(path + sep)) { @@ -292,12 +322,14 @@ export function findProcessUsers(path, opts = {}) { break; } } catch { - // fd vanished mid-scan, or unreadable — ignore + // this one fd vanished mid-scan (ENOENT) — not a verification failure } } } catch { - // /proc//fd unreadable (permission, or process exited) — ignore + fdOk = false; } + + let cmdlineOk = true; try { const cmdline = readFileSync(join(procRoot, pid, 'cmdline'), 'utf8').replace(/\0/g, ' ').trim(); if (cmdline && cmdline.includes(path)) { @@ -305,9 +337,21 @@ export function findProcessUsers(path, opts = {}) { const servingProcess = SERVING_PROCESS_NAMES.some(n => lower.includes(n)); users.push({ pid, via: 'cmdline', cmdline: cmdline.slice(0, 200), servingProcess }); } - } catch { - // no permission to read cmdline — ignore + } catch (e) { + if (e.code !== 'ENOENT') cmdlineOk = false; } + + if (!fdOk || !cmdlineOk) unreadablePids.push(pid); + } + + if (unreadablePids.length > 0) { + const shown = unreadablePids.slice(0, 5).join(', '); + const more = unreadablePids.length > 5 ? `, +${unreadablePids.length - 5} more` : ''; + return { + checked: false, + users, + note: `could not read /proc for pid(s) ${shown}${more} (permission denied) — cannot rule out those processes using this path` + }; } return { checked: true, users }; } @@ -317,9 +361,13 @@ export function findProcessUsers(path, opts = {}) { // --------------------------------------------------------------------------- /** - * Is `path` under the bind-mount source of a RUNNING docker container? + * Is `path` under, or does it CONTAIN, the bind-mount source of a RUNNING + * docker container? Both directions matter: `path` under the mount source + * means the whole candidate is served; the mount source under `path` means + * moving `path` would carry an actively-mounted subdirectory away with it. * `{available:false}` means docker itself could not be queried — callers - * must apply the fail-safe rule (treat ~/.cache/huggingface as in-use). + * must fail closed (see planRun: an unreadable docker means every + * candidate on the box is treated as unverified, not just this one). */ export function findDockerBindUsers(path, opts = {}) { const dockerBin = opts.dockerBin || 'docker'; @@ -342,7 +390,7 @@ export function findDockerBindUsers(path, opts = {}) { const hits = []; for (const c of parsed) { for (const m of c.Mounts || []) { - if (m.Source && (path === m.Source || path.startsWith(m.Source + sep))) { + if (m.Source && (path === m.Source || path.startsWith(m.Source + sep) || m.Source.startsWith(path + sep))) { hits.push({ container: (c.Name || '').replace(/^\//, '') || (c.Id || '').slice(0, 12), source: m.Source, destination: m.Destination }); } } @@ -449,7 +497,6 @@ export function planRun(options = {}) { const listDockerBindUsers = options.listDockerBindUsers || findDockerBindUsers; const getDiskFreeBytes = options.diskFreeBytes || diskFreeBytes; const nowMs = options.now ? options.now.getTime() : Date.now(); - const hfCacheRoot = join(home, '.cache', 'huggingface'); const rawCandidates = options.candidates || discoverCandidates(home); const { groupOf, groupMembers, groupSizeBytes, filesByCandidate } = computeHardlinkGroups(rawCandidates); @@ -463,30 +510,38 @@ export function planRun(options = {}) { } else if (isSymlink(c.path)) { skipReason = 'already a symlink (tidied)'; } else { + // Rules 2-3 (process / docker in-use) are fail-closed: if either + // check could not be fully completed, we cannot prove the candidate + // is idle, so it is skipped as unverified rather than allowed + // through on absence of evidence. An unreadable docker taints EVERY + // candidate on the box, not just the HF cache — a container could + // bind-mount anything. const procResult = listProcessUsers(c.path); - const fdHit = (procResult.users || []).find(u => u.via === 'fd'); - const cmdHit = (procResult.users || []).find(u => u.via === 'cmdline' && u.servingProcess); - const hit = fdHit || cmdHit; - if (hit) { - skipReason = hit.via === 'fd' - ? `in use: pid ${hit.pid} has a file open under this path` - : `in use: pid ${hit.pid} serving process references this path (${hit.cmdline})`; + if (procResult.checked === false) { + skipReason = `in-use status unverified: ${procResult.note || 'process check could not be completed'}`; } else { - const dockerResult = listDockerBindUsers(c.path); - if (dockerResult.available === false) { - if (c.path === hfCacheRoot || c.path.startsWith(hfCacheRoot + sep)) { - skipReason = `docker not readable (${dockerResult.reason}); treating ~/.cache/huggingface as in-use, fail-safe`; + const fdHit = (procResult.users || []).find(u => u.via === 'fd'); + const cmdHit = (procResult.users || []).find(u => u.via === 'cmdline' && u.servingProcess); + const hit = fdHit || cmdHit; + if (hit) { + skipReason = hit.via === 'fd' + ? `in use: pid ${hit.pid} has a file open under this path` + : `in use: pid ${hit.pid} serving process references this path (${hit.cmdline})`; + } else { + const dockerResult = listDockerBindUsers(c.path); + if (dockerResult.available === false) { + skipReason = `in-use status unverified: docker not readable (${dockerResult.reason})`; + } else if (dockerResult.inUse) { + const first = dockerResult.containers[0]; + skipReason = `in use: bind-mounted into running container ${first.container} (${first.source} -> ${first.destination})`; } - } else if (dockerResult.inUse) { - const first = dockerResult.containers[0]; - skipReason = `in use: bind-mounted into running container ${first.container} (${first.source} -> ${first.destination})`; - } - if (!skipReason) { - const files = filesByCandidate.get(c.id); - const newest = newestMtimeMs(c.path, files); - const ageDays = (nowMs - newest) / 86400000; - if (ageDays < minIdleDays) { - skipReason = `modified ${ageDays.toFixed(1)}d ago, newer than --min-idle-days ${minIdleDays}`; + if (!skipReason) { + const files = filesByCandidate.get(c.id); + const newest = newestMtimeMs(c.path, files); + const ageDays = (nowMs - newest) / 86400000; + if (ageDays < minIdleDays) { + skipReason = `modified ${ageDays.toFixed(1)}d ago, newer than --min-idle-days ${minIdleDays}`; + } } } } @@ -513,8 +568,8 @@ export function planRun(options = {}) { groupId: gid, groupSizeBytes: groupSizeBytes.get(gid), selected: false, - reason: isHardlinkSet - ? `hardlink set with ${failing.path}, which is skipped: ${failing.skipReason}` + reason: isHardlinkSet && m.id !== failing.id + ? `hardlinked to ${failing.path}, which is not moving (${failing.skipReason})` : m.skipReason }); } @@ -558,6 +613,7 @@ export function planRun(options = {}) { const skipped = results.filter(r => !r.selected); const selectedGroupIds = new Set(selected.map(r => r.groupId)); const totalSelectedBytes = [...selectedGroupIds].reduce((sum, gid) => sum + groupSizeBytes.get(gid), 0); + const unverifiedCount = skipped.filter(r => r.reason.includes('unverified')).length; const freeBeforeBytes = getDiskFreeBytes(home); const freeAfterEstimateBytes = freeBeforeBytes == null ? null : freeBeforeBytes + totalSelectedBytes; @@ -567,7 +623,8 @@ export function planRun(options = {}) { : `free before/after: ${(freeBeforeBytes / 2 ** 30).toFixed(1)} GiB -> ~${(freeAfterEstimateBytes / 2 ** 30).toFixed(1)} GiB`; const summaryLine = `model-tidy plan: ${selected.length} dir(s) in ${selectedGroupIds.size} unit(s), ` + - `${(totalSelectedBytes / 2 ** 30).toFixed(1)} GiB movable to target, ${skipped.length} skipped. ${freeLine}`; + `${(totalSelectedBytes / 2 ** 30).toFixed(1)} GiB movable to target, ${skipped.length} skipped ` + + `(${unverifiedCount} unverified). ${freeLine}`; return { home, @@ -576,6 +633,7 @@ export function planRun(options = {}) { selected, skipped, totalSelectedBytes, + unverifiedCount, freeBeforeBytes, freeAfterEstimateBytes, freeLine, @@ -699,10 +757,11 @@ function listAllEntries(root) { return out; } -/** Validate --target: must exist, be a directory, be writable, and be on a - * different filesystem device than `home`. */ +/** Validate --target: must be an absolute path, must exist, be a directory, + * be writable, and be on a different filesystem device than `home`. */ export function validateTarget(target, home) { if (!target) return { ok: false, error: '--target is required with --apply' }; + if (!isAbsolute(target)) return { ok: false, error: `--target ${target} must be an absolute path (a relative target makes the resulting symlink text ambiguous)` }; if (!existsSync(target)) return { ok: false, error: `--target ${target} does not exist` }; const st = statSync(target); if (!st.isDirectory()) return { ok: false, error: `--target ${target} is not a directory` }; @@ -718,18 +777,68 @@ export function validateTarget(target, home) { return { ok: true }; } +const STAGING_SUFFIX = '.tidy-moving'; + +/** + * Swap one source directory for a symlink to its already-verified copy, + * crash-safely: + * + * rename(src, src + STAGING_SUFFIX) [atomic, same filesystem] + * symlink(dst, src) + * verify src resolves to dst + * remove src + STAGING_SUFFIX + * + * A same-filesystem rename is atomic, so a crash at any point leaves EITHER + * the original directory in place (still at its staging name, never lost) + * OR a working symlink to the verified copy — never neither. If anything + * after the rename fails, this function removes the half-made symlink (if + * any) and renames the staging directory back to `src` before returning, so + * the caller sees `src` exactly as it was on entry. + */ +function swapToSymlink(src, dst, opts = {}) { + const doSymlink = opts.symlinkSync || symlinkSync; + const staging = src + STAGING_SUFFIX; + if (existsSync(staging)) { + throw new Error(`refusing to touch ${src}: a leftover ${staging} from a previous interrupted apply already exists — resolve it manually (verify which of ${src}/${staging} is intact, then remove the other) before retrying`); + } + renameSync(src, staging); + try { + doSymlink(dst, src); + const real = realpathSync(src); + if (real !== realpathSync(dst) || !lstatSync(src).isSymbolicLink()) { + throw new Error(`post-symlink verification failed for ${src}`); + } + rmSync(staging, { recursive: true }); + } catch (e) { + // Roll back: remove any half-made symlink, then restore the original + // from staging so the caller finds `src` exactly as it was. + try { + if (lstatSync(src).isSymbolicLink()) rmSync(src); + } catch { + // src may not exist at all if symlinkSync itself never ran — fine. + } + renameSync(staging, src); + throw e; + } +} + /** * Apply a plan: move every selected unit to `target`, unit by unit. Each - * unit is copied, verified, and only THEN does its source directory get - * replaced with a symlink — so a failure at any step for a unit leaves that - * unit's source completely untouched. One unit's failure does not roll back - * units that already succeeded; the run still exits non-zero overall. + * unit is copied, verified, and only THEN is its source directory swapped + * for a symlink via swapToSymlink()'s rename/symlink/verify/cleanup + * sequence. A failure at ANY step — copy, verify, or swap — leaves that + * unit's source EITHER fully in place (copy/verify failures never touch + * the source) OR replaced by a working symlink to a verified copy (a swap + * failure rolls back to the original). There is no state in between. One + * unit's failure does not roll back units that already succeeded earlier + * in the same run; the process still exits non-zero overall. */ export function applyRun(options) { const { plan, target, home } = options; const copyFn = options.copyFn || copyUnitPureNode; const verifyFn = options.verifyFn || verifyUnit; const validateTargetFn = options.validateTarget || validateTarget; + const symlinkFn = options.symlinkFn; // test-only injection; real default is symlinkSync inside swapToSymlink const validation = validateTargetFn(target, home); if (!validation.ok) { @@ -747,6 +856,18 @@ export function applyRun(options) { for (const [groupId, members] of byGroup) { const sourceAbsPaths = members.map(m => m.path); + + const leftoverStaging = sourceAbsPaths.filter(src => existsSync(src + STAGING_SUFFIX)); + if (leftoverStaging.length > 0) { + errors.push({ + groupId, + paths: sourceAbsPaths, + step: 'pre-check', + error: `refusing: ${leftoverStaging.map(p => `${p}${STAGING_SUFFIX}`).join(', ')} left over from a previous interrupted apply — resolve manually before retrying this unit` + }); + continue; + } + try { copyFn(sourceAbsPaths, home, target); } catch (e) { @@ -767,12 +888,7 @@ export function applyRun(options) { const rel = relative(home, src); const dst = join(target, rel); try { - rmSync(src, { recursive: true }); - symlinkSync(dst, src); - const real = realpathSync(src); - if (real !== realpathSync(dst) || !lstatSync(src).isSymbolicLink()) { - throw new Error(`post-symlink verification failed for ${src}`); - } + swapToSymlink(src, dst, { symlinkSync: symlinkFn }); swapped.push({ source: src, target: dst }); } catch (e) { swapFailed = { path: src, error: e.message }; diff --git a/test/model-tidy.test.mjs b/test/model-tidy.test.mjs index 106d99b..f83de5d 100644 --- a/test/model-tidy.test.mjs +++ b/test/model-tidy.test.mjs @@ -177,15 +177,19 @@ describe('planRun', () => { assert.equal(plan.skipped.length, 3); }); - it('skips a whole hardlink set if either member is not idle', () => { + it('skips a whole hardlink set if either member is in use', () => { const { home, keepFile, hardlinkA, hardlinkB } = buildFixture(); - touch(join(hardlinkB, 'file.bin'), daysAgo(1)); // make B recent - + // Note: touching hardlinkB's file.bin mtime would also move hardlinkA's + // mtime, since they are literally the same inode — that's not a useful + // way to make just one member fail independently. Use a per-path check + // (process-in-use) instead, which is genuinely independent per path. const plan = planRun({ home, keepFile, minIdleDays: 14, - listProcessUsers: noProcessUsers, + listProcessUsers: (path) => (path === hardlinkB + ? { checked: true, users: [{ pid: '9999', via: 'fd', target: join(path, 'file.bin') }] } + : { checked: true, users: [] }), listDockerBindUsers: dockerNotInUse, diskFreeBytes: fixedDiskFree }); @@ -194,11 +198,17 @@ describe('planRun', () => { assert.ok(!selectedPaths.includes(hardlinkA)); assert.ok(!selectedPaths.includes(hardlinkB)); const skippedA = plan.skipped.find(r => r.path === hardlinkA); - assert.match(skippedA.reason, /hardlink set/); + assert.match(skippedA.reason, /hardlinked to .*which is not moving/); + assert.match(skippedA.reason, /in use/); + const skippedB = plan.skipped.find(r => r.path === hardlinkB); + assert.match(skippedB.reason, /in use: pid 9999/); }); - it('treats ~/.cache/huggingface as in-use when docker is unreadable', () => { - const { home, keepFile, idleRoot } = buildFixture(); + it('fails closed for EVERY candidate on the box when docker is unreadable, not just the HF cache', () => { + // Regression test for the fail-open defect: an unreadable docker used + // to only fail-safe ~/.cache/huggingface, leaving ~/models/* and ad-hoc + // dirs selectable even though a container could bind-mount anything. + const { home, keepFile, idleRoot, recentRoot, keepRoot, hardlinkA } = buildFixture(); const plan = planRun({ home, keepFile, @@ -207,9 +217,71 @@ describe('planRun', () => { listDockerBindUsers: () => ({ available: false, inUse: false, reason: 'permission denied' }), diskFreeBytes: fixedDiskFree }); + + assert.equal(plan.selected.length, 0, 'nothing should be selected while docker is unreadable'); + const skippedIdle = plan.skipped.find(r => r.path === idleRoot); - assert.ok(skippedIdle, 'HF cache dir should be fail-safe skipped when docker is unreadable'); - assert.match(skippedIdle.reason, /docker not readable/); + assert.ok(skippedIdle); + assert.match(skippedIdle.reason, /in-use status unverified: docker not readable/); + + // The bug: hardlinkA lives under ~/models, NOT under ~/.cache/huggingface. + // It must ALSO be unverified now, box-wide. + const skippedHardlinkA = plan.skipped.find(r => r.path === hardlinkA); + assert.ok(skippedHardlinkA); + assert.match(skippedHardlinkA.reason, /unverified/); + + // Rule priority is unchanged: KEEP list still wins before the docker + // check is ever reached. + const skippedKeep = plan.skipped.find(r => r.path === keepRoot); + assert.match(skippedKeep.reason, /on KEEP list/); + + assert.match(plan.summaryLine, /\d+ unverified/); + assert.ok(plan.unverifiedCount >= 3, `expected several unverified candidates, got ${plan.unverifiedCount}`); + }); + + it('positive control: with a readable process list and docker, the idle dir is still selected', () => { + const { home, keepFile, idleRoot } = buildFixture(); + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: () => ({ checked: true, users: [] }), + listDockerBindUsers: () => ({ available: true, inUse: false, containers: [] }), + diskFreeBytes: fixedDiskFree + }); + const selectedIdle = plan.selected.find(r => r.path === idleRoot); + assert.ok(selectedIdle, 'idle dir should be selected when everything is verifiably readable and idle'); + assert.equal(plan.unverifiedCount, 0); + }); + + it('fails closed when the process check cannot be fully completed (e.g. EACCES on one pid)', () => { + const home = tempDir('model-tidy-unverified-'); + const idleA = join(home, 'models', 'idle-a'); + const idleB = join(home, 'models', 'idle-b'); + writeFile(join(idleA, 'weights.gguf'), 'a'.repeat(2048)); + writeFile(join(idleB, 'weights.gguf'), 'b'.repeat(2048)); + touch(join(idleA, 'weights.gguf'), daysAgo(30)); + touch(join(idleB, 'weights.gguf'), daysAgo(30)); + + const plan = planRun({ + home, + minIdleDays: 14, + listProcessUsers: () => ({ + checked: false, + users: [], + note: 'could not read /proc for pid(s) 4242 (permission denied) — cannot rule out those processes using this path' + }), + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + + assert.equal(plan.selected.length, 0, 'nothing should be selected when process status cannot be verified'); + assert.equal(plan.skipped.length, 2); + for (const r of plan.skipped) { + assert.match(r.reason, /in-use status unverified: could not read \/proc/); + } + assert.match(plan.summaryLine, /2 unverified/); + assert.equal(plan.unverifiedCount, 2); }); it('skips a dir with an open file handle (simulated process check)', () => { @@ -261,6 +333,38 @@ describe('planRun', () => { const deferred = plan.skipped.filter(r => /max-gb.*cap/.test(r.reason)); assert.ok(deferred.length > 0); }); + + it('skips an idle dir hardlinked to a KEEP-listed dir, even though both are otherwise idle-eligible', () => { + // Regression test for hardlink-scan scope: the KEEP-listed copy lives + // outside the plain "idle" candidate set (it's filtered out by the + // KEEP rule), so the hardlink group it belongs to must still be + // detected and the whole group skipped — never just the KEEP-listed + // half, which would leave the idle-looking half free to move and + // silently double disk usage. + const { home, keepFile, idleRoot, keepRoot } = buildFixture(); + // Make idleRoot's blob and keepRoot's file share an inode. + const idleBlobDir = join(idleRoot, 'blobs'); + const idleBlobName = readdirSync(idleBlobDir)[0]; + rmSync(join(keepRoot, 'weights.gguf')); + linkSync(join(idleBlobDir, idleBlobName), join(keepRoot, 'weights.gguf')); + + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + + const selectedPaths = plan.selected.map(r => r.path); + assert.ok(!selectedPaths.includes(idleRoot), 'idle dir must not move while hardlinked to a KEEP-listed copy'); + + const skippedIdle = plan.skipped.find(r => r.path === idleRoot); + assert.ok(skippedIdle); + assert.match(skippedIdle.reason, /hardlinked to .*which is not moving/); + assert.match(skippedIdle.reason, /on KEEP list/); + }); }); // tmpdir()-based fixtures and their "target" dir are normally on the SAME @@ -378,4 +482,93 @@ describe('applyRun', () => { const content = readFileSync(join(blobDir, blobName), 'utf8'); assert.equal(content, 'idle-model-bytes'.repeat(100)); }); + + it('crash-window negative control: if symlink creation fails after the source was staged, the source is restored byte-identical', () => { + const { home, keepFile, idleRoot } = buildFixture(); + const target = tempDir('model-tidy-target-'); + + const originalBlobName = readdirSync(join(idleRoot, 'blobs'))[0]; + const originalContent = readFileSync(join(idleRoot, 'blobs', originalBlobName), 'utf8'); + + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + plan.selected = plan.selected.filter(r => r.path === idleRoot); + + // Inject a failing symlink function (the same seam applyRun exposes + // for copyFn/verifyFn/validateTarget) so swapToSymlink's real call + // fails right after the real rename has already happened — the exact + // crash window the fix closes. + let callCount = 0; + const failingSymlinkSync = () => { + callCount++; + throw new Error('simulated symlink failure (disk full / EIO / etc.)'); + }; + + const result = applyRun({ plan, target, home, validateTarget: bypassCrossFsCheck, symlinkFn: failingSymlinkSync }); + assert.equal(result.ok, false); + assert.equal(result.moved.length, 0); + assert.equal(result.errors.length, 1); + assert.equal(result.errors[0].step, 'swap'); + assert.ok(callCount > 0, 'the injected failing symlinkSync should have been invoked'); + + // The source must be back at its original path, not a symlink, and + // byte-identical — never left staged, never left half-swapped. + assert.equal(existsSync(idleRoot), true); + assert.equal(lstatSync(idleRoot).isSymbolicLink(), false); + assert.equal(existsSync(idleRoot + '.tidy-moving'), false); + const restoredContent = readFileSync(join(idleRoot, 'blobs', originalBlobName), 'utf8'); + assert.equal(restoredContent, originalContent); + }); + + it('refuses a unit with a leftover *.tidy-moving from a previous crash, without touching it', () => { + const { home, keepFile, idleRoot } = buildFixture(); + const target = tempDir('model-tidy-target-'); + const staging = `${idleRoot}.tidy-moving`; + + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + plan.selected = plan.selected.filter(r => r.path === idleRoot); + + // Simulate a previous crash: idleRoot was already renamed to staging, + // and (for this test) never got its symlink, so both idleRoot and the + // staging dir happen to coexist here only in the sense that we recreate + // idleRoot fresh below to prove apply won't touch EITHER. + mkdirSync(staging, { recursive: true }); + writeFileSync(join(staging, 'marker'), 'leftover-from-a-previous-crash'); + tempPaths.push(staging); + + let copyCalled = false; + const result = applyRun({ + plan, + target, + home, + validateTarget: bypassCrossFsCheck, + copyFn: () => { copyCalled = true; } + }); + + assert.equal(result.ok, false); + assert.equal(result.moved.length, 0); + assert.equal(result.errors.length, 1); + assert.equal(result.errors[0].step, 'pre-check'); + assert.match(result.errors[0].error, /tidy-moving/); + assert.equal(copyCalled, false, 'apply must refuse before even attempting to copy this unit'); + + // Neither the original nor the leftover staging dir were touched. + assert.ok(existsSync(idleRoot)); + assert.equal(lstatSync(idleRoot).isSymbolicLink(), false); + assert.ok(existsSync(staging)); + assert.equal(readFileSync(join(staging, 'marker'), 'utf8'), 'leftover-from-a-previous-crash'); + }); }); From e45642e5fee1b7dc9d425f0c030c13b773181ce2 Mon Sep 17 00:00:00 2001 From: Petrus Pennanen Date: Mon, 21 Sep 2026 09:29:19 +0200 Subject: [PATCH 3/9] Fix 2 more gaps from codexmb's re-review: hardlink completeness, journaled crash recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses codexmb's re-review of 0f6b682, with exact reproductions (probe.mjs), plus a follow-up amendment from the lead overriding three points of the original ask. GAP 1: hardlink to a file outside every discovered candidate root. computeHardlinkGroups only ever saw candidates it had itself discovered, so a second link sitting outside every discovered root (e.g. directly under $HOME, above ~/models) was invisible no matter how wide the scan went. Now every file's st_nlink is compared against how many links this scan actually observed under the scanned roots; a shortfall skips the whole candidate with "hardlinked N times, only M links found under scanned roots; refusing incomplete set" — checked before every other rule, since it's a filesystem fact, not a policy choice. GAP 2: a hard crash (SIGKILL/OOM/power loss, not a thrown exception) at various points during the swap-to-symlink step. The old rename-then- symlink sequence's in-process try/catch rollback cannot run at all when the process is hard-killed instead of throwing, which the review's probe demonstrated with a real process.exit() in a child. Per the lead's amendment, the fix is journal-based, not a bigger try/catch: - plan is READ-ONLY. It never recovers anything, including interrupted state — it only detects it (via the journal) and reports it per unit as "interrupted move found: ...", refusing to select that unit. A new `recover` subcommand is the only place (besides apply's own start) that mutates. - Before touching a unit at all, apply writes a journal record (JSON, one file per unit under /.cache/ide-agent-kit/model-tidy-journal/, written with an explicit fsync) holding the source/staged/temp-link/ target paths and the per-file size+SHA-256 manifest already computed. The record is rewritten (and re-fsynced) after every subsequent step: pending -> linked -> staged -> swapped -> removed on cleanup. - Recovery is journal-driven, not naming-driven: a directory literally named *.tidy-moving or *.tidy-link with no matching journal record is reported and left alone, never touched. A journaled unit is only acted on after its manifest is re-verified against whatever currently exists on disk; a mismatch is also left alone and reported. - The exact safety claim, as specified: "the original data is preserved and recoverable at every step; the original path is unavailable for the instant between the two renames, and until recovery runs if a crash lands there. It is not continuously available." Not claiming more than that. New tests (11, all pass standalone and in the full suite): GAP 1 reproduces codexmb's probe verbatim (link outside home/models) plus a positive control (all links inside the tree); GAP 2 crashes a real child process at each step (before either rename via the symlink hook, right after the link is verified, between the two renames, and after the second rename before cleanup — including codexmb's exact probe kept verbatim), asserts the tree is whole after one recovery pass, and covers the amendment's required cases: plan makes zero filesystem writes (before/after full tree snapshot, sizes+mtimes) while still reporting the interrupted unit; recovery never touches an unjournaled *.tidy-moving directory; recovery refuses a journaled unit whose on-disk content no longer matches its manifest. Test results: 11/11 new tests in isolation, 29/29 full model-tidy suite, 688/688 full repo suite. Not done: the two previously-flagged Codex findings (streaming checksums for large files, no lsof fallback) remain out of scope and are noted in docs/model-tidy.md, which now also notes computeManifest/verifyManifest share the same whole-file-readFileSync property. The journal-based recovery itself has only been exercised against synthetic fixtures on this dev machine — never against asus1/asus2, a real multi-GiB model, or a genuinely full disk. Co-Authored-By: Claude Fable 5.1 --- bin/model-tidy.mjs | 42 +++- docs/model-tidy.md | 169 ++++++++++--- src/model-tidy.mjs | 512 +++++++++++++++++++++++++++++++++++---- test/model-tidy.test.mjs | 296 +++++++++++++++++++++- 4 files changed, 930 insertions(+), 89 deletions(-) diff --git a/bin/model-tidy.mjs b/bin/model-tidy.mjs index 8b14142..68cef64 100755 --- a/bin/model-tidy.mjs +++ b/bin/model-tidy.mjs @@ -7,8 +7,22 @@ * model-tidy plan [--home ] [--keep-file ] [--min-idle-days N] * [--max-gb N] [--json] [--log-dir ] * [--report-to-room] [--room ] [--config ] + * Read-only. Never mutates anything. If a previous `apply` was + * interrupted (crash, kill, power loss), plan DETECTS and reports it + * per unit ("interrupted move found: ...") and refuses to select that + * unit — it does not attempt to fix it. Run `recover` for that. * * model-tidy apply --apply --target [same options as plan] + * Runs recovery once at the very start (only reached because --apply + * was given), then copies/verifies/swaps every selected unit. + * + * model-tidy recover [--home ] + * Explicit, mutating recovery pass for interrupted `apply` swaps. Acts + * ONLY on units with a journal record whose on-disk state matches the + * journaled manifest; anything else (including a directory merely + * *named* like a leftover, with no journal) is reported and left + * alone. Safe to run at any time, including on a healthy tree (a + * no-op). * * model-tidy --dry-run-remote [--remote-home ] * ssh's to and runs `plan` there, read-only. Never copies, @@ -23,7 +37,7 @@ import { existsSync, readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { dirname, join } from 'node:path'; import { homedir } from 'node:os'; -import { planRun, applyRun, writeRunLog } from '../src/model-tidy.mjs'; +import { planRun, applyRun, recoverInterruptedMoves, writeRunLog } from '../src/model-tidy.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = join(__dirname, '..'); @@ -75,6 +89,13 @@ function printPlan(plan, opts) { const gib = (r.sizeBytes / 2 ** 30).toFixed(2); console.log(` [skip] ${r.path} (${gib} GiB, ${r.kind}) — ${r.reason}`); } + if (plan.interrupted && plan.interrupted.length > 0) { + console.log(''); + console.log(`Interrupted-move findings (${plan.interrupted.length}) — plan never mutates these, run 'model-tidy recover':`); + for (const f of plan.interrupted) { + console.log(` [${f.status}] ${f.path || f.journalFile} — ${f.note}`); + } + } } function reportToRoom(summaryLine, args) { @@ -170,6 +191,23 @@ function main() { } const logDir = args['log-dir'] || DEFAULT_LOG_DIR; + if (mode === 'recover') { + const recovered = recoverInterruptedMoves(home); + writeRunLog(logDir, { mode: 'recover', home, recovered, argv }); + if (args.json) { + console.log(JSON.stringify(recovered, null, 2)); + } else if (recovered.length === 0) { + console.log('recover: nothing to do (no journaled interrupted moves, no stray leftovers found).'); + } else { + console.log(`recover: ${recovered.length} finding(s):`); + for (const r of recovered) { + console.log(` [${r.action}] ${r.path || r.journalFile} — ${r.note}`); + } + } + process.exit(recovered.some(r => r.action === 'error') ? 1 : 0); + return; + } + const plan = planRun({ home, keepFile, minIdleDays, maxGb }); const logFile = writeRunLog(logDir, { @@ -210,7 +248,7 @@ function main() { process.exit(result.ok ? 0 : 1); } - console.error(`unknown mode: ${mode} (expected "plan" or "apply")`); + console.error(`unknown mode: ${mode} (expected "plan", "apply", or "recover")`); process.exit(2); } diff --git a/docs/model-tidy.md b/docs/model-tidy.md index a0cec76..3624db6 100644 --- a/docs/model-tidy.md +++ b/docs/model-tidy.md @@ -61,6 +61,24 @@ Prints, for every discovered candidate: "free before -> ~free after" estimate, and writes a JSON record to the log dir (`--log-dir`, default `~/.cache/ide-agent-kit/model-tidy-logs/`). +**`plan` makes zero filesystem writes, full stop** — including when it +finds a unit left mid-swap by an interrupted `apply`. It reports that as +an "interrupted move found" skip (see "Crash safety" below) and moves on; +resolving it is `recover`'s job, not `plan`'s. + +### `recover` — explicit, mutating, journal-validated + +``` +node bin/model-tidy.mjs recover [--home ] [--json] +``` + +Heals units left mid-swap by an interrupted `apply` (a crash, a kill, a +power loss). Acts **only** on units that have a matching journal record +whose manifest still matches what's on disk; a directory merely *named* +like a leftover, with no journal record, is reported and left alone. Safe +to run any time, including against a fully healthy tree (a no-op). See +"Crash safety" below for exactly what it checks and does. + ### `apply` — only with `--apply --target ` ``` @@ -74,36 +92,96 @@ Refuses immediately, before touching anything, unless `--target`: `stat().dev`, not by path string — a bind mount of the same device would still be refused) +`plan` is **read-only** — it never mutates anything, including when it +finds a unit left mid-swap by a previous interrupted `apply`. It only +*detects* that state (via the journal — see "Crash safety" below) and +*reports* it per unit as skipped, with reason `"interrupted move found: +..."`; it never touches the filesystem to fix it. Only two things ever run +recovery: the explicit `model-tidy recover` subcommand, and `apply` itself +(once, at its own start — reached only because `--apply` was actually +given). + For each selected unit (a single directory, or a whole hardlink set moved -together): -0. Refuse if a `.tidy-moving` directory already exists (see step 3) - — that means a previous `apply` was interrupted mid-swap for this unit, - and this run will not guess which of the two paths is authoritative. -1. Copy every file into `--target`, preserving the relative-to-home path, +together), `apply`: +0. Refuses the whole run if a unit's journal record shows it's already + mid-swap in a way this run's own recovery pass (see below) couldn't + validate and resolve — that means an ambiguous leftover, and this run + will not guess which state is authoritative. +1. Copies every file into `--target`, preserving the relative-to-home path, symlinks, and hardlink relationships. -2. Verify every file: same relative paths, same symlink targets, same byte - size AND SHA-256 for every regular file. -3. Only if verification passed for every file in the unit, swap the source - for a symlink via a crash-safe staged sequence: - `rename(source, source + '.tidy-moving')` (atomic on the same - filesystem) → `symlink(target-copy, source)` → verify the symlink - resolves (`realpathSync`) to the target copy → remove - `source + '.tidy-moving'`. -4. If anything in step 3 fails after the rename, the symlink (if partially - created) is removed and `source + '.tidy-moving'` is renamed straight - back to `source`, so the unit ends up exactly as it started. - -**The safety claim is: a failure at any step leaves a unit's source EITHER -with its original completely untouched (any failure in steps 0-2, or a -step-3 failure that rolled back) OR with a working symlink to a verified -copy (a step-3 swap that completed) — never neither, and never deleted -without a verified copy existing both at the target and reachable through -the symlink.** A failure for one unit does not roll back units that already -succeeded earlier in the same run, but the process still exits non-zero and -prints exactly which unit failed, at which step, and why. +2. Verifies every file: same relative paths, same symlink targets, same + byte size AND SHA-256 for every regular file. +3. Only if verification passed for every file in the unit, swaps the + source for a symlink via the journaled, crash-safe sequence below. + +### Crash safety + +A plain "delete, then create the symlink" has a window where a **hard** +crash — SIGKILL, an OOM kill, a power loss; anything a JS `try/catch` +cannot intercept, unlike a thrown exception — leaves neither the original +nor a symlink at the source path. Every path that referenced the model +breaks, and nothing in the same process ever gets a chance to roll back. + +**Before touching a unit at all**, `apply` writes a journal record — JSON, +one file per unit (named by a hash of the source path), under +`/.cache/ide-agent-kit/model-tidy-journal/`, written with an +explicit `fsync` so it survives a crash immediately after the write +returns. The record holds the source/staged/temp-link/target paths, the +already-computed per-file size+SHA-256 manifest, and the step reached. It +is rewritten (and re-fsynced) after every subsequent step: + +| Step written | What happens next | +| --- | --- | +| `pending` | build `.tidy-link -> target-copy`; verify it resolves and matches the manifest. **The source itself is not touched by anything up to and including this step** — a crash here leaves the source exactly as it was. | +| `linked` | `renameSync(source, source + '.tidy-moving')` — atomic. | +| `staged` | `renameSync(source + '.tidy-link', source)` — atomic, run immediately after the previous rename. | +| `swapped` | re-verify the symlink at `source` resolves to the target copy, then remove `source + '.tidy-moving'` and the journal record. | + +**The safety claim, exactly:** the original data is preserved and +recoverable at every step; the original path is unavailable for the +instant between the two renames, and until recovery runs if a crash lands +there. It is not continuously available — that instant is real and the +tests below cover it, not just the steps either side of it. + +Interrupted moves are **not** recovered automatically by `plan` or by the +passage of time. They are recovered by `model-tidy recover` (or by the +next `apply`, which runs the same recovery at its own start). Recovery is +**journal-driven, not naming-driven**: it reads every journal record, +checks that whichever of {source, staged copy} currently exists on disk +still matches that record's manifest, and only then acts: + +| Journaled state found | Recovery action | +| --- | --- | +| staged exists, source missing, temp link exists | rename the link into place, verify, then remove staged and the journal record — the link was already verified before it was ever created, so completing it is safe | +| staged exists, source missing, no temp link | rename staged back to source, remove the journal record — no verified pending swap existed to trust instead | +| staged exists, source is a symlink | the swap itself already completed; re-verify the symlink target, then remove staged and the journal record | +| temp link exists, source is a real directory, no staged | crashed before source was ever touched; remove the stray link and the journal record | +| source is a symlink, no staged | the move had already fully completed; just remove the stale journal record | +| source is a real directory, nothing else exists | never touched at all; remove the stale journal record | +| manifest mismatch against whatever currently exists | **left alone**, reported, regardless of step | + +A `*.tidy-moving` or `*.tidy-link` directory with **no matching journal +record at all** is reported but never touched, no matter how it's named — +recovery does not treat a name as ownership. A failure for one unit does +not roll back units that already succeeded earlier in the same `apply` +run, but the process still exits non-zero and prints exactly which unit +failed, at which step, and why. ## Selection rules (in order, each with an explicit reason string) +0. **Interrupted move found** — highest priority, checked before anything + else. If the journal (see "Crash safety" above) shows this unit is + mid-swap from a previous interrupted `apply`, it is skipped with + `interrupted move found:
` and never selected, regardless of + any other rule. `plan` only reports this; run `recover` to resolve it. +0.5. **Hardlink completeness** — also checked before KEEP/process/docker/ + etc. If any file's `st_nlink` exceeds the number of links this scan + actually found under every discovered candidate root, the candidate is + skipped with `hardlinked N times, only M links found under scanned + roots; refusing incomplete set`. A hardlink partner can sit entirely + outside every discovered root (e.g. a manual backup copy directly under + `$HOME`, above `~/models`) — no amount of scanning wider closes this in + general, only comparing against `st_nlink` does. 1. **KEEP list** — anything matching `--keep-file` (path or glob, `~` expanded, `#` comments) is always skipped. See `config/model-tidy.keep.example`. @@ -210,7 +288,9 @@ it never attempts to install anything itself. ## Safety invariants -- `plan` mode makes zero filesystem writes anywhere. It's the default. +- `plan` makes zero filesystem writes, ever — including for interrupted + moves, which it only detects and reports. Only `recover` and `apply` + (once, at its own start) mutate anything. - `apply` requires both `--apply` AND `--target ` — neither alone is enough. - `--target` must be an absolute, existing, writable directory on a @@ -219,18 +299,25 @@ it never attempts to install anything itself. - A source directory's original copy is only ever removed after (a) its copy has been verified byte-for-byte (size + SHA-256) at the target, and (b) the replacement symlink at the source has itself been created and - verified to resolve to that copy. There is no code path that removes the - original before both of those have happened — see the staged - rename/symlink/verify/cleanup sequence above. A crash between the rename - and the final cleanup leaves a `.tidy-moving` directory, which a - later `apply` run detects and refuses to touch until it's resolved by - hand. + verified to resolve to that copy. **Exact claim: the original data is + preserved and recoverable at every step; the original path is + unavailable for the instant between the two renames, and until recovery + runs if a crash lands there. It is not continuously available.** +- Recovery is journal-driven, not naming-driven: it only acts on a unit + whose on-disk state (source and/or staged copy, whichever exist) matches + the manifest recorded in that unit's own journal file. A directory named + like a leftover with no matching journal record — or a journaled unit + whose content has since changed — is reported and left alone, never + guessed at. - A hardlink set moves as a unit or not at all — never partially — and the hardlink scan covers every discovered model root, not just the ones that - already passed every other rule. + already passed every other rule. It also refuses a candidate whose + `st_nlink` exceeds the number of links actually found under the scanned + roots — a hardlink partner can sit entirely outside every discovered + candidate, where no amount of scanning wider would find it. - Process- and docker-in-use checks are fail-closed: anything that could not be fully verified is treated as in-use, never as idle. -- KEEP-listed paths are never touched by either mode. +- KEEP-listed paths are never touched by any mode. - `--report-to-room` and any nightly `apply` are both opt-in and off by default. @@ -280,4 +367,16 @@ it never attempts to install anything itself. exhaust memory or exceed Node's Buffer limits, making `apply` fail at the verify step for large real models even though the copy itself succeeded; (2) there is no `lsof`-based fallback for the process-in-use - check, only `/proc`. + check, only `/proc`. `computeManifest`/`verifyManifest` (the journal's + own integrity check) have the exact same whole-file-`readFileSync` + property, so the same real-world risk applies there too. +- The journal-based crash recovery (`recoverInterruptedMoves`, the + `recover` subcommand) has only ever been exercised against synthetic + fixtures with tiny files and a hard-killed child process on this dev + machine (macOS). It has never been exercised against a real interrupted + `apply` on asus1/asus2, against a multi-GiB real model, or against a + genuinely full disk (an `ENOSPC` mid-copy, mid-rename, or mid-journal- + write is untested). The journal directory + (`/.cache/ide-agent-kit/model-tidy-journal/`) itself is assumed to + be on the same filesystem as `--home` and writable; that assumption is + untested on either box. diff --git a/src/model-tidy.mjs b/src/model-tidy.mjs index 24d1e8a..54033a8 100644 --- a/src/model-tidy.mjs +++ b/src/model-tidy.mjs @@ -50,7 +50,8 @@ import { spawnSync } from 'node:child_process'; import { readdirSync, lstatSync, existsSync, readFileSync, readlinkSync, realpathSync, symlinkSync, rmSync, mkdirSync, copyFileSync, linkSync, - statSync, appendFileSync, constants as FS_CONSTANTS, accessSync, renameSync + statSync, appendFileSync, constants as FS_CONSTANTS, accessSync, renameSync, + openSync, writeSync, fsyncSync, closeSync } from 'node:fs'; import { join, relative, sep, isAbsolute } from 'node:path'; import { createHash } from 'node:crypto'; @@ -99,7 +100,7 @@ export function walkFiles(root) { continue; } if (st.isFile()) { - results.push({ path: full, size: st.size, mtimeMs: st.mtimeMs, dev: st.dev, ino: st.ino }); + results.push({ path: full, size: st.size, mtimeMs: st.mtimeMs, dev: st.dev, ino: st.ino, nlink: st.nlink }); } } } @@ -177,6 +178,7 @@ export function discoverCandidates(home) { const hfHub = join(home, '.cache', 'huggingface', 'hub'); if (existsSync(hfHub)) { for (const entry of safeReaddir(hfHub)) { + if (isJournalSuffixed(entry)) continue; if (entry.startsWith('models--')) { add(join(hfHub, entry), 'hf-cache'); } @@ -186,12 +188,14 @@ export function discoverCandidates(home) { const modelsDir = join(home, 'models'); if (existsSync(modelsDir)) { for (const entry of safeReaddir(modelsDir)) { + if (isJournalSuffixed(entry)) continue; add(join(modelsDir, entry), 'models-dir'); } } for (const entry of safeReaddir(home)) { if (entry.startsWith('.')) continue; + if (isJournalSuffixed(entry)) continue; if (AD_HOC_EXCLUDE.has(entry)) continue; const full = join(home, entry); let st; @@ -207,6 +211,7 @@ export function discoverCandidates(home) { } // one level deep: e.g. ~/GLM-5.3-Flash-EXL3-2x-DGX-Sparks/model for (const child of safeReaddir(full)) { + if (isJournalSuffixed(child)) continue; const childFull = join(full, child); let cst; try { @@ -410,6 +415,8 @@ export function findDockerBindUsers(path, opts = {}) { export function computeHardlinkGroups(candidates) { const filesByCandidate = new Map(); const inodeMap = new Map(); // "dev:ino" -> Set(candidateId) + const inodeObservedCount = new Map(); // "dev:ino" -> how many file entries we actually found + const inodeNlink = new Map(); // "dev:ino" -> st_nlink reported by the filesystem for (const c of candidates) { const files = walkFiles(c.path); @@ -418,6 +425,35 @@ export function computeHardlinkGroups(candidates) { const key = `${f.dev}:${f.ino}`; if (!inodeMap.has(key)) inodeMap.set(key, new Set()); inodeMap.get(key).add(c.id); + inodeObservedCount.set(key, (inodeObservedCount.get(key) || 0) + 1); + inodeNlink.set(key, f.nlink); + } + } + + // Scanning wider cannot fully close this gap: st_nlink tells us how many + // directory entries point at this inode SYSTEM-WIDE, including ones + // completely outside any discovered candidate root (e.g. a manual backup + // hardlink sitting directly under $HOME, above ~/models). If the number + // of entries we actually found while walking the discovered candidates + // is less than st_nlink, there is at least one more link we cannot see + // and therefore cannot move safely — moving what we CAN see would still + // break the invisible link and double disk usage. Every candidate that + // owns such a file is flagged incomplete here; planRun turns this into a + // skip reason with top priority, before any other rule. + const incompleteReasonByCandidate = new Map(); + for (const c of candidates) { + if (incompleteReasonByCandidate.has(c.id)) continue; + for (const f of filesByCandidate.get(c.id)) { + const key = `${f.dev}:${f.ino}`; + const observed = inodeObservedCount.get(key); + const nlink = inodeNlink.get(key); + if (nlink > observed) { + incompleteReasonByCandidate.set( + c.id, + `hardlinked ${nlink} times, only ${observed} links found under scanned roots; refusing incomplete set` + ); + break; + } } } @@ -463,7 +499,7 @@ export function computeHardlinkGroups(candidates) { groupSizeBytes.set(g, total); } - return { groupOf: id => find(id), groupMembers, groupSizeBytes, filesByCandidate }; + return { groupOf: id => find(id), groupMembers, groupSizeBytes, filesByCandidate, incompleteReasonByCandidate }; } // --------------------------------------------------------------------------- @@ -497,15 +533,29 @@ export function planRun(options = {}) { const listDockerBindUsers = options.listDockerBindUsers || findDockerBindUsers; const getDiskFreeBytes = options.diskFreeBytes || diskFreeBytes; const nowMs = options.now ? options.now.getTime() : Date.now(); + const journalDir = options.journalDir || defaultJournalDir(home); + + // plan is READ-ONLY: it only DETECTS an interrupted apply from a previous + // crash (via the journal) and reports it per unit below — it never + // mutates anything. Only the explicit `recover` subcommand, or `apply` + // itself, actually heals interrupted state. + const detectFn = options.detect || detectInterruptedMoves; + const interrupted = detectFn(home, { journalDir }); + const interruptedByPath = new Map(interrupted.filter(f => f.path).map(f => [f.path, f])); const rawCandidates = options.candidates || discoverCandidates(home); - const { groupOf, groupMembers, groupSizeBytes, filesByCandidate } = computeHardlinkGroups(rawCandidates); + const { groupOf, groupMembers, groupSizeBytes, filesByCandidate, incompleteReasonByCandidate } = computeHardlinkGroups(rawCandidates); const perCandidate = new Map(); for (const c of rawCandidates) { let skipReason = null; - if (matchesKeepList(c.path, keepEntries)) { + // Rule 0, highest priority: a hardlink whose partner is invisible to + // this scan (outside every discovered candidate root) is a filesystem + // fact, not a policy choice — check it before KEEP/process/docker/etc. + if (incompleteReasonByCandidate.has(c.id)) { + skipReason = incompleteReasonByCandidate.get(c.id); + } else if (matchesKeepList(c.path, keepEntries)) { skipReason = 'on KEEP list'; } else if (isSymlink(c.path)) { skipReason = 'already a symlink (tidied)'; @@ -607,6 +657,32 @@ export function planRun(options = {}) { } } + // An interrupted move takes absolute priority over every other rule: it + // is a filesystem fact discovered from the journal, not a policy choice, + // and this unit must never be selected until 'recover' resolves it. + // Override any existing result for this path, or synthesize one if the + // unit's real directory is currently missing (mid-swap) so discovery + // never saw it at all. + for (const finding of interruptedByPath.values()) { + const existingIdx = results.findIndex(r => r.path === finding.path); + const reason = `interrupted move found: ${finding.note}`; + if (existingIdx >= 0) { + results[existingIdx].selected = false; + results[existingIdx].reason = reason; + } else { + results.push({ + id: finding.path, + path: finding.path, + kind: 'interrupted', + sizeBytes: 0, + groupId: finding.path, + groupSizeBytes: 0, + selected: false, + reason + }); + } + } + results.sort((a, b) => b.groupSizeBytes - a.groupSizeBytes || a.path.localeCompare(b.path)); const selected = results.filter(r => r.selected); @@ -622,14 +698,18 @@ export function planRun(options = {}) { ? 'free before/after: unknown (df unavailable)' : `free before/after: ${(freeBeforeBytes / 2 ** 30).toFixed(1)} GiB -> ~${(freeAfterEstimateBytes / 2 ** 30).toFixed(1)} GiB`; + const interruptedLine = interrupted.length > 0 + ? ` ${interrupted.length} interrupted-move finding(s) detected (never mutated by plan — run 'recover' to resolve).` + : ''; const summaryLine = `model-tidy plan: ${selected.length} dir(s) in ${selectedGroupIds.size} unit(s), ` + `${(totalSelectedBytes / 2 ** 30).toFixed(1)} GiB movable to target, ${skipped.length} skipped ` + - `(${unverifiedCount} unverified). ${freeLine}`; + `(${unverifiedCount} unverified). ${freeLine}${interruptedLine}`; return { home, minIdleDays, maxGb, + interrupted, selected, skipped, totalSelectedBytes, @@ -778,60 +858,390 @@ export function validateTarget(target, home) { } const STAGING_SUFFIX = '.tidy-moving'; +const LINK_SUFFIX = '.tidy-link'; +const JOURNAL_SUBDIR = 'model-tidy-journal'; + +function isJournalSuffixed(name) { + return name.endsWith(STAGING_SUFFIX) || name.endsWith(LINK_SUFFIX); +} + +function defaultJournalDir(home) { + return join(home, '.cache', 'ide-agent-kit', JOURNAL_SUBDIR); +} + +function journalKey(sourcePath) { + return createHash('sha256').update(sourcePath).digest('hex'); +} + +function journalFilePath(journalDir, sourcePath) { + return join(journalDir, `${journalKey(sourcePath)}.json`); +} /** - * Swap one source directory for a symlink to its already-verified copy, - * crash-safely: + * Write (or overwrite) the journal record for one unit, fsynced so it + * survives a crash immediately after this call returns. Called BEFORE the + * first filesystem mutation for a unit, and again after every subsequent + * step, so the journal always reflects the furthest step actually reached. + */ +function writeJournalRecordSync(journalDir, record) { + mkdirSync(journalDir, { recursive: true }); + const file = journalFilePath(journalDir, record.sourcePath); + const data = JSON.stringify({ ...record, updatedAt: new Date().toISOString() }, null, 2); + const fd = openSync(file, 'w'); + try { + writeSync(fd, data); + fsyncSync(fd); + } finally { + closeSync(fd); + } + return file; +} + +function removeJournalRecord(journalDir, sourcePath) { + const file = journalFilePath(journalDir, sourcePath); + try { + rmSync(file); + } catch { + // already gone — fine + } +} + +/** All journal records currently on disk. A record that fails to parse is + * still returned (with `record: null, corrupt: true`) so callers can + * report it rather than silently skip it. */ +function listJournalRecords(journalDir) { + if (!existsSync(journalDir)) return []; + const out = []; + for (const name of safeReaddir(journalDir)) { + if (!name.endsWith('.json')) continue; + const file = join(journalDir, name); + try { + out.push({ file, record: JSON.parse(readFileSync(file, 'utf8')) }); + } catch { + out.push({ file, record: null, corrupt: true }); + } + } + return out; +} + +/** size + sha256 (files) / link target (symlinks) for every entry under + * `rootDir`, relative to it. Directories are implied by their children's + * relPaths and not recorded individually. */ +function computeManifest(rootDir) { + const manifest = []; + for (const full of listAllEntries(rootDir)) { + let lst; + try { + lst = lstatSync(full); + } catch { + continue; + } + const relPath = relative(rootDir, full); + if (lst.isSymbolicLink()) { + manifest.push({ relPath, type: 'symlink', linkTarget: readlinkSync(full) }); + } else if (lst.isFile()) { + manifest.push({ relPath, type: 'file', size: lst.size, sha256: sha256File(full) }); + } + } + return manifest; +} + +/** True only if every manifest entry exists under `rootDir` with matching + * size+sha256 (files) or link target (symlinks). Used by recovery to + * refuse to act on a journal record whose paths don't actually match what + * it claims — see recoverInterruptedMoves(). */ +function verifyManifest(rootDir, manifest) { + for (const entry of manifest) { + const full = join(rootDir, entry.relPath); + let lst; + try { + lst = lstatSync(full); + } catch { + return false; + } + if (entry.type === 'symlink') { + if (!lst.isSymbolicLink() || readlinkSync(full) !== entry.linkTarget) return false; + } else { + if (!lst.isFile() || lst.size !== entry.size || sha256File(full) !== entry.sha256) return false; + } + } + return true; +} + +/** + * Swap one source directory for a symlink to its already-verified copy. + * + * A plain "delete then symlink" (or even "rename then symlink") has a + * window where a hard crash (SIGKILL, power loss, OOM kill) — which a + * try/catch cannot intercept, unlike a thrown JS exception — leaves NEITHER + * the original NOR a symlink at `src`. To close that, the symlink is built + * and verified at a side path BEFORE `src` is touched at all, and every + * step (including this one) is journaled to `opts.journalDir` — fsynced — + * BEFORE the step's filesystem mutation happens, so recovery always knows + * the furthest point actually reached: * - * rename(src, src + STAGING_SUFFIX) [atomic, same filesystem] - * symlink(dst, src) - * verify src resolves to dst - * remove src + STAGING_SUFFIX + * 'pending' written -> symlink(dst, src+LINK_SUFFIX); verify it resolves + * [src untouched so far — a crash here leaves src exactly as it was] + * 'linked' written -> rename(src, src+STAGING_SUFFIX) [atomic] + * [a crash here leaves the original at STAGING_SUFFIX, recoverable] + * 'staged' written -> rename(src+LINK_SUFFIX, src) [atomic] + * [THE GAP: for this instant, neither a real directory nor a symlink + * exists at src — see the module docstring. A crash here needs + * recovery to finish this exact rename from the journal.] + * 'swapped' written -> verify src resolves to dst again, remove staging + * [a crash here leaves a working symlink at src already — the only + * thing left undone is freeing the disk space, which recovery does] + * journal record removed once cleanup succeeds. * - * A same-filesystem rename is atomic, so a crash at any point leaves EITHER - * the original directory in place (still at its staging name, never lost) - * OR a working symlink to the verified copy — never neither. If anything - * after the rename fails, this function removes the half-made symlink (if - * any) and renames the staging directory back to `src` before returning, so - * the caller sees `src` exactly as it was on entry. + * This function itself does NOT roll back on a thrown (catchable) failure + * — recovery is deliberately a separate, explicit, journal-validated step + * (recoverInterruptedMoves, run via the `recover` subcommand or at the + * start of `apply`), not an automatic side effect of error handling, so a + * hard kill and a thrown exception are recovered the exact same way. */ function swapToSymlink(src, dst, opts = {}) { const doSymlink = opts.symlinkSync || symlinkSync; const staging = src + STAGING_SUFFIX; - if (existsSync(staging)) { - throw new Error(`refusing to touch ${src}: a leftover ${staging} from a previous interrupted apply already exists — resolve it manually (verify which of ${src}/${staging} is intact, then remove the other) before retrying`); + const link = src + LINK_SUFFIX; + const journalDir = opts.journalDir; + + if (existsSync(staging) || existsSync(link)) { + throw new Error(`refusing to touch ${src}: a leftover ${existsSync(staging) ? staging : link} already exists from a previous run — run the 'recover' subcommand first`); + } + + const manifest = computeManifest(src); + const base = { sourcePath: src, stagedPath: staging, linkPath: link, targetPath: dst, manifest }; + + writeJournalRecordSync(journalDir, { ...base, step: 'pending' }); + if (opts.beforeLink) opts.beforeLink(); // test-only hook: simulate a crash here + + doSymlink(dst, link); + const realLink = realpathSync(link); + if (realLink !== realpathSync(dst) || !lstatSync(link).isSymbolicLink()) { + rmSync(link, { recursive: true }); + removeJournalRecord(journalDir, src); + throw new Error(`pre-swap symlink verification failed for ${link}`); } + writeJournalRecordSync(journalDir, { ...base, step: 'linked' }); + if (opts.afterLink) opts.afterLink(); // test-only hook: simulate a crash here + renameSync(src, staging); - try { - doSymlink(dst, src); - const real = realpathSync(src); - if (real !== realpathSync(dst) || !lstatSync(src).isSymbolicLink()) { - throw new Error(`post-symlink verification failed for ${src}`); + writeJournalRecordSync(journalDir, { ...base, step: 'staged' }); + if (opts.afterStage) opts.afterStage(); // test-only hook: simulate a crash IN THE GAP between the two renames + + renameSync(link, src); + writeJournalRecordSync(journalDir, { ...base, step: 'swapped' }); + if (opts.afterSwap) opts.afterSwap(); // test-only hook: simulate a crash here + + const real = realpathSync(src); + if (real !== realpathSync(dst) || !lstatSync(src).isSymbolicLink()) { + throw new Error(`post-swap verification failed for ${src}`); + } + rmSync(staging, { recursive: true }); + removeJournalRecord(journalDir, src); +} + +/** Every location discoverCandidates() looks at, reused so detection and + * recovery scan for stray *.tidy-moving/*.tidy-link siblings in exactly + * the same places. */ +function candidateParentDirs(home) { + const dirs = new Set(); + const hfHub = join(home, '.cache', 'huggingface', 'hub'); + if (existsSync(hfHub)) dirs.add(hfHub); + const modelsDir = join(home, 'models'); + if (existsSync(modelsDir)) dirs.add(modelsDir); + dirs.add(home); + for (const entry of safeReaddir(home)) { + if (entry.startsWith('.')) continue; + const full = join(home, entry); + try { + if (lstatSync(full).isDirectory()) dirs.add(full); + } catch { + // vanished between readdir and lstat — ignore } - rmSync(staging, { recursive: true }); - } catch (e) { - // Roll back: remove any half-made symlink, then restore the original - // from staging so the caller finds `src` exactly as it was. + } + return [...dirs]; +} + +function strayJournalSuffixedPaths(home, excludePaths) { + const strays = []; + for (const dir of candidateParentDirs(home)) { + for (const name of safeReaddir(dir)) { + if (!isJournalSuffixed(name)) continue; + const base = name.endsWith(STAGING_SUFFIX) ? name.slice(0, -STAGING_SUFFIX.length) : name.slice(0, -LINK_SUFFIX.length); + const src = join(dir, base); + if (excludePaths.has(src)) continue; + strays.push({ path: src, entryPath: join(dir, name) }); + } + } + return strays; +} + +/** + * READ-ONLY detection of interrupted `apply` swaps, for `plan`. Never + * mutates anything. For every journal record found under + * `home/.cache/ide-agent-kit/model-tidy-journal`, reports the step reached + * and a human-readable description of the on-disk state, WITHOUT touching + * it. Also reports (but never touches) any `*.tidy-moving` / `*.tidy-link` + * sibling that has no matching journal record at all — those are not this + * tool's to interpret; only recoverInterruptedMoves() (a separate, explicit, + * mutating step) acts on journaled units, and only after validating them. + */ +export function detectInterruptedMoves(home, options = {}) { + const journalDir = options.journalDir || defaultJournalDir(home); + const findings = []; + const journaledPaths = new Set(); + + for (const { file, record, corrupt } of listJournalRecords(journalDir)) { + if (corrupt || !record) { + findings.push({ path: null, journalFile: file, status: 'corrupt-journal', note: `journal file ${file} is corrupt or unreadable` }); + continue; + } + journaledPaths.add(record.sourcePath); + const srcExists = existsSync(record.sourcePath); + let srcIsSymlink = false; try { - if (lstatSync(src).isSymbolicLink()) rmSync(src); + srcIsSymlink = lstatSync(record.sourcePath).isSymbolicLink(); } catch { - // src may not exist at all if symlinkSync itself never ran — fine. + // does not exist — srcIsSymlink stays false } - renameSync(staging, src); - throw e; + const stagedExists = existsSync(record.stagedPath); + const linkExists = existsSync(record.linkPath); + findings.push({ + path: record.sourcePath, + journalFile: file, + step: record.step, + status: 'interrupted', + note: `journal step '${record.step}' — source ${srcExists ? (srcIsSymlink ? 'exists as a symlink' : 'exists as a real directory') : 'missing'}, ` + + `staged copy ${stagedExists ? 'present' : 'absent'}, pending link ${linkExists ? 'present' : 'absent'}. Run the 'recover' subcommand to resolve.` + }); + } + + for (const stray of strayJournalSuffixedPaths(home, journaledPaths)) { + findings.push({ + path: stray.path, + journalFile: null, + status: 'orphan-no-journal', + note: `found ${stray.entryPath} with no matching journal record — not this tool's to interpret, left alone` + }); } + + return findings; +} + +/** + * MUTATING recovery for interrupted `apply` swaps. Only runs from the + * explicit `recover` subcommand, or at the very start of `apply` (i.e. + * only when `--apply` was actually given — `plan` never calls this). + * + * Acts ONLY on units that have a journal record — a directory merely + * *named* `*.tidy-moving` or `*.tidy-link` with no journal entry is never + * touched, regardless of how it looks. For a journaled unit, validates + * that whatever currently exists on disk (the source as a real directory, + * and/or the staged copy) matches the journal's recorded manifest before + * trusting the record enough to act; a mismatch is reported and left + * alone rather than guessed at. + */ +export function recoverInterruptedMoves(home, options = {}) { + const journalDir = options.journalDir || defaultJournalDir(home); + const recovered = []; + const journaledPaths = new Set(); + + for (const { file, record, corrupt } of listJournalRecords(journalDir)) { + if (corrupt || !record) { + recovered.push({ path: null, journalFile: file, action: 'left-alone', note: `journal file ${file} is corrupt or unreadable; left alone` }); + continue; + } + const { sourcePath, stagedPath, linkPath, manifest, step } = record; + journaledPaths.add(sourcePath); + + try { + const srcExists = existsSync(sourcePath); + let srcIsSymlink = false; + try { + srcIsSymlink = lstatSync(sourcePath).isSymbolicLink(); + } catch { + // missing — srcIsSymlink stays false + } + const stagedExists = existsSync(stagedPath); + const linkExists = existsSync(linkPath); + + const stagedOk = !stagedExists || verifyManifest(stagedPath, manifest); + const srcOk = !srcExists || srcIsSymlink || verifyManifest(sourcePath, manifest); + if (!stagedOk || !srcOk) { + recovered.push({ path: sourcePath, journalFile: file, action: 'left-alone', note: `on-disk content (step recorded as '${step}') does not match the journaled manifest; left alone for manual inspection` }); + continue; + } + + if (srcExists && srcIsSymlink && !stagedExists) { + removeJournalRecord(journalDir, sourcePath); + recovered.push({ path: sourcePath, journalFile: file, action: 'removed-stale-journal', note: 'the move had already fully completed; removed the stale journal record' }); + } else if (srcExists && srcIsSymlink && stagedExists) { + const real = realpathSync(sourcePath); + if (existsSync(real)) { + rmSync(stagedPath, { recursive: true }); + removeJournalRecord(journalDir, sourcePath); + recovered.push({ path: sourcePath, journalFile: file, action: 'finished-cleanup', note: 'symlink was already in place; finished removing the staged original' }); + } else { + recovered.push({ path: sourcePath, journalFile: file, action: 'left-alone', note: `symlink target ${real} does not exist — leaving ${stagedPath} for manual inspection` }); + } + } else if (!srcExists && stagedExists && linkExists) { + renameSync(linkPath, sourcePath); + const real = realpathSync(sourcePath); + if (existsSync(real)) { + rmSync(stagedPath, { recursive: true }); + removeJournalRecord(journalDir, sourcePath); + recovered.push({ path: sourcePath, journalFile: file, action: 'completed-swap-and-cleaned', note: 'completed the pending symlink swap (the crash landed in the gap between the two renames) and finished cleanup' }); + } else { + recovered.push({ path: sourcePath, journalFile: file, action: 'completed-swap', note: 'completed the pending symlink swap; leaving the staged original for manual inspection (symlink target unexpectedly missing)' }); + } + } else if (!srcExists && stagedExists && !linkExists) { + renameSync(stagedPath, sourcePath); + removeJournalRecord(journalDir, sourcePath); + recovered.push({ path: sourcePath, journalFile: file, action: 'restored-original', note: 'restored the original directory (no verified pending symlink existed to trust instead)' }); + } else if (srcExists && !srcIsSymlink && linkExists && !stagedExists) { + rmSync(linkPath, { recursive: true }); + removeJournalRecord(journalDir, sourcePath); + recovered.push({ path: sourcePath, journalFile: file, action: 'removed-stray-link', note: 'the original was never touched; removed the unused pending symlink' }); + } else if (srcExists && !srcIsSymlink && !linkExists && !stagedExists) { + removeJournalRecord(journalDir, sourcePath); + recovered.push({ path: sourcePath, journalFile: file, action: 'removed-stale-journal', note: 'the original was never touched; cleared the journal record' }); + } else { + recovered.push({ path: sourcePath, journalFile: file, action: 'left-alone', note: `unrecognized on-disk state for a journaled unit (step='${step}'); left alone for manual inspection` }); + } + } catch (e) { + recovered.push({ path: sourcePath, journalFile: file, action: 'error', note: `recovery failed: ${e.message}` }); + } + } + + for (const stray of strayJournalSuffixedPaths(home, journaledPaths)) { + recovered.push({ + path: stray.path, + journalFile: null, + action: 'left-alone', + note: `found ${stray.entryPath} with no matching journal record — not this tool's to touch` + }); + } + + return recovered; } /** * Apply a plan: move every selected unit to `target`, unit by unit. Each * unit is copied, verified, and only THEN is its source directory swapped - * for a symlink via swapToSymlink()'s rename/symlink/verify/cleanup - * sequence. A failure at ANY step — copy, verify, or swap — leaves that - * unit's source EITHER fully in place (copy/verify failures never touch - * the source) OR replaced by a working symlink to a verified copy (a swap - * failure rolls back to the original). There is no state in between. One + * for a symlink via swapToSymlink()'s journaled sequence (see its + * docstring). A failure at copy or verify leaves the source fully in + * place, untouched. Once the swap itself starts, the original data is + * preserved and recoverable at every step; the original path is + * unavailable for the instant between the two renames, and until recovery + * runs if a crash lands there — it is not continuously available. One * unit's failure does not roll back units that already succeeded earlier * in the same run; the process still exits non-zero overall. + * + * Recovery (recoverInterruptedMoves) runs once, here, at the very start — + * i.e. only when `--apply` was actually given. `planRun` never calls it; + * `plan` only detects and reports interrupted state (detectInterruptedMoves) + * without mutating anything. */ export function applyRun(options) { const { plan, target, home } = options; @@ -839,10 +1249,17 @@ export function applyRun(options) { const verifyFn = options.verifyFn || verifyUnit; const validateTargetFn = options.validateTarget || validateTarget; const symlinkFn = options.symlinkFn; // test-only injection; real default is symlinkSync inside swapToSymlink + const recoverFn = options.recover || recoverInterruptedMoves; + const journalDir = options.journalDir || defaultJournalDir(home); + + // Self-heal any interrupted swap from a previous crash before this run's + // own pre-checks and copy/verify/swap loop — see recoverInterruptedMoves. + // Only reached here, in apply, never from planRun. + const recovered = recoverFn(home, { journalDir }); const validation = validateTargetFn(target, home); if (!validation.ok) { - return { ok: false, error: validation.error, moved: [], errors: [{ error: validation.error }] }; + return { ok: false, error: validation.error, moved: [], errors: [{ error: validation.error }], recovered }; } const byGroup = new Map(); @@ -857,13 +1274,13 @@ export function applyRun(options) { for (const [groupId, members] of byGroup) { const sourceAbsPaths = members.map(m => m.path); - const leftoverStaging = sourceAbsPaths.filter(src => existsSync(src + STAGING_SUFFIX)); - if (leftoverStaging.length > 0) { + const leftover = sourceAbsPaths.filter(src => existsSync(src + STAGING_SUFFIX) || existsSync(src + LINK_SUFFIX)); + if (leftover.length > 0) { errors.push({ groupId, paths: sourceAbsPaths, step: 'pre-check', - error: `refusing: ${leftoverStaging.map(p => `${p}${STAGING_SUFFIX}`).join(', ')} left over from a previous interrupted apply — resolve manually before retrying this unit` + error: `refusing: a leftover from a previous run still exists for ${leftover.join(', ')} — run the 'recover' subcommand first` }); continue; } @@ -888,7 +1305,14 @@ export function applyRun(options) { const rel = relative(home, src); const dst = join(target, rel); try { - swapToSymlink(src, dst, { symlinkSync: symlinkFn }); + swapToSymlink(src, dst, { + symlinkSync: symlinkFn, + journalDir, + beforeLink: options.beforeLink, + afterLink: options.afterLink, + afterStage: options.afterStage, + afterSwap: options.afterSwap + }); swapped.push({ source: src, target: dst }); } catch (e) { swapFailed = { path: src, error: e.message }; @@ -902,7 +1326,7 @@ export function applyRun(options) { } } - return { ok: errors.length === 0, moved, errors }; + return { ok: errors.length === 0, moved, errors, recovered }; } // --------------------------------------------------------------------------- diff --git a/test/model-tidy.test.mjs b/test/model-tidy.test.mjs index f83de5d..3e7e667 100644 --- a/test/model-tidy.test.mjs +++ b/test/model-tidy.test.mjs @@ -4,15 +4,33 @@ import { describe, it, afterEach } from 'node:test'; import { strict as assert } from 'node:assert'; import { mkdtempSync, mkdirSync, writeFileSync, symlinkSync, linkSync, rmSync, - existsSync, readFileSync, utimesSync, lstatSync, readdirSync + existsSync, readFileSync, utimesSync, lstatSync, readdirSync, readlinkSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; +import { spawnSync } from 'node:child_process'; import { planRun, applyRun, discoverCandidates, loadKeepList, computeHardlinkGroups, - validateTarget, copyUnitPureNode + validateTarget, copyUnitPureNode, recoverInterruptedMoves } from '../src/model-tidy.mjs'; +const MODEL_TIDY_MODULE_URL = new URL('../src/model-tidy.mjs', import.meta.url).href; + +/** + * Run applyRun in a CHILD process with one option forced to hard-exit + * (process.exit, not a throw — a try/catch in the parent's process cannot + * intercept this, which is exactly the real-world crash swapToSymlink's + * design has to survive: SIGKILL, OOM kill, power loss). Mirrors codexmb's + * review probe technique verbatim: serialize the plan, embed an absolute + * import URL, embed a literal crashing arrow function as one applyRun + * option keyed by `crashOptionKey` ('symlinkFn', 'afterStage', or + * 'afterSwap'). + */ +function runApplyInChildWithCrash(plan, home, target, crashOptionKey) { + const code = `import {applyRun} from ${JSON.stringify(MODEL_TIDY_MODULE_URL)}; applyRun({plan:${JSON.stringify(plan)},home:${JSON.stringify(home)},target:${JSON.stringify(target)},validateTarget:()=>({ok:true}),${crashOptionKey}:()=>process.exit(77)});`; + return spawnSync(process.execPath, ['--input-type=module', '-e', code], { encoding: 'utf8' }); +} + const tempPaths = []; afterEach(() => { @@ -365,6 +383,48 @@ describe('planRun', () => { assert.match(skippedIdle.reason, /hardlinked to .*which is not moving/); assert.match(skippedIdle.reason, /on KEEP list/); }); + + it('GAP 1 regression: refuses a candidate hardlinked to a file OUTSIDE every discovered root (codexmb probe, reproduced exactly)', () => { + // Exact reproduction of the review's probe.mjs: link placed directly + // under the fixture root, ABOVE home/models, so no discovered + // candidate root ever contains that second path — scanning wider + // cannot see it. Only comparing st_nlink against what we actually + // observed can catch this. + const root = tempDir('model-tidy-gap1-'); + const home = join(root, 'home'); + const src = join(home, 'models', 'idle'); + mkdirSync(src, { recursive: true }); + writeFileSync(join(src, 'weights.gguf'), 'fixture-only'); + utimesSync(join(src, 'weights.gguf'), new Date(0), new Date(0)); + linkSync(join(src, 'weights.gguf'), join(root, 'outside-discovery.gguf')); + + const plan = planRun({ + home, + listProcessUsers: () => ({ checked: true, users: [] }), + listDockerBindUsers: () => ({ available: true, inUse: false, containers: [] }), + diskFreeBytes: () => 0 + }); + + assert.ok(!plan.selected.some(r => r.path === src), 'must not select a candidate with an invisible extra hardlink'); + const skipped = plan.skipped.find(r => r.path === src); + assert.ok(skipped); + assert.match(skipped.reason, /hardlinked 2 times, only 1 links found under scanned roots; refusing incomplete set/); + }); + + it('GAP 1 positive control: an idle dir whose hardlinks are ALL inside the scanned tree is still selected as a unit', () => { + const { home, keepFile, hardlinkA, hardlinkB } = buildFixture(); + const plan = planRun({ + home, + keepFile, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + const selectedPaths = plan.selected.map(r => r.path); + assert.ok(selectedPaths.includes(hardlinkA)); + assert.ok(selectedPaths.includes(hardlinkB)); + }); }); // tmpdir()-based fixtures and their "target" dir are normally on the SAME @@ -526,7 +586,7 @@ describe('applyRun', () => { assert.equal(restoredContent, originalContent); }); - it('refuses a unit with a leftover *.tidy-moving from a previous crash, without touching it', () => { + it("refuses a unit with an UNJOURNALED leftover *.tidy-moving, without touching it (apply's own recovery pass only heals journaled units)", () => { const { home, keepFile, idleRoot } = buildFixture(); const target = tempDir('model-tidy-target-'); const staging = `${idleRoot}.tidy-moving`; @@ -541,10 +601,11 @@ describe('applyRun', () => { }); plan.selected = plan.selected.filter(r => r.path === idleRoot); - // Simulate a previous crash: idleRoot was already renamed to staging, - // and (for this test) never got its symlink, so both idleRoot and the - // staging dir happen to coexist here only in the sense that we recreate - // idleRoot fresh below to prove apply won't touch EITHER. + // Simulate a leftover with NO journal record at all (e.g. a directory + // that just happens to be named like one of ours, or a journal file + // that was separately lost). Recovery must never guess about this — + // it should be reported and left alone, and apply's own pre-check + // should then refuse the unit outright. mkdirSync(staging, { recursive: true }); writeFileSync(join(staging, 'marker'), 'leftover-from-a-previous-crash'); tempPaths.push(staging); @@ -562,7 +623,8 @@ describe('applyRun', () => { assert.equal(result.moved.length, 0); assert.equal(result.errors.length, 1); assert.equal(result.errors[0].step, 'pre-check'); - assert.match(result.errors[0].error, /tidy-moving/); + assert.match(result.errors[0].error, /leftover from a previous run/); + assert.match(result.errors[0].error, /recover/); assert.equal(copyCalled, false, 'apply must refuse before even attempting to copy this unit'); // Neither the original nor the leftover staging dir were touched. @@ -572,3 +634,221 @@ describe('applyRun', () => { assert.equal(readFileSync(join(staging, 'marker'), 'utf8'), 'leftover-from-a-previous-crash'); }); }); + +describe('GAP 2: crash-safety across a hard process kill (not just a thrown exception)', () => { + function buildSingleIdleFixture() { + const home = tempDir('model-tidy-gap2-home-'); + const src = join(home, 'models', 'idle'); + mkdirSync(src, { recursive: true }); + writeFileSync(join(src, 'weights.gguf'), 'gap2-fixture-bytes'.repeat(50)); + utimesSync(join(src, 'weights.gguf'), daysAgo(30), daysAgo(30)); + return { home, src }; + } + + function planFor(home) { + return planRun({ + home, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + } + + it("codexmb probe, kept verbatim: a hard crash during the symlink step leaves the original intact, and the invariant holds after one recovery pass", () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + assert.ok(plan.selected.some(r => r.path === src)); + + const child = runApplyInChildWithCrash(plan, home, target, 'symlinkFn'); + assert.equal(child.status, 77); + + recoverInterruptedMoves(home); + + // codexmb's probe assertion, kept verbatim in spirit: after the crash + // and one recovery pass, either the original path exists, or a + // working symlink exists at src. Never neither. + let srcIsWorkingSymlink = false; + try { + srcIsWorkingSymlink = lstatSync(src).isSymbolicLink() && existsSync(src); + } catch { + srcIsWorkingSymlink = false; + } + const originalPathExists = existsSync(src) && !srcIsWorkingSymlink; + assert.ok(originalPathExists || srcIsWorkingSymlink, 'neither the original nor a working symlink exists at src'); + + // This specific crash point happens before any rename, so nothing + // should have moved at all: the original itself, untouched. + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), false); + assert.equal(existsSync(src + '.tidy-moving'), false); + assert.equal(existsSync(src + '.tidy-link'), false); + }); + + it('a hard crash between the two renames is healed by completing the pending symlink swap', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterStage'); + assert.equal(child.status, 77); + // Mid-crash snapshot: src renamed away, symlink not yet swapped in. + assert.equal(existsSync(src), false); + assert.ok(existsSync(`${src}.tidy-moving`)); + + const recovered = recoverInterruptedMoves(home); + assert.ok(recovered.some(r => r.path === src && r.action === 'completed-swap-and-cleaned'), JSON.stringify(recovered)); + + assert.equal(existsSync(src), true, 'src must exist after recovery'); + assert.equal(lstatSync(src).isSymbolicLink(), true, 'src must be a working symlink after recovery'); + assert.equal(existsSync(`${src}.tidy-moving`), false, 'staged original should be fully cleaned up'); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'gap2-fixture-bytes'.repeat(50)); + }); + + it('a hard crash after the second rename (before cleanup) is healed by finishing the cleanup', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterSwap'); + assert.equal(child.status, 77); + // Mid-crash snapshot: the swap already completed structurally — src is + // already a working symlink — only the old original's cleanup is + // pending. + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), true); + assert.ok(existsSync(`${src}.tidy-moving`)); + + const recovered = recoverInterruptedMoves(home); + assert.ok(recovered.some(r => r.path === src && r.action === 'finished-cleanup'), JSON.stringify(recovered)); + + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), true); + assert.equal(existsSync(`${src}.tidy-moving`), false, 'staged original should be cleaned up after recovery'); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'gap2-fixture-bytes'.repeat(50)); + }); + + it('recovery is idempotent and a no-op on a healthy tree', () => { + const { home } = buildSingleIdleFixture(); + const first = recoverInterruptedMoves(home); + const second = recoverInterruptedMoves(home); + assert.deepEqual(first, []); + assert.deepEqual(second, []); + }); + + it('a hard crash right after the link is verified, before either rename, is healed with the original untouched', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterLink'); + assert.equal(child.status, 77); + assert.equal(existsSync(src), true, 'mid-crash: src untouched, no rename has happened yet'); + assert.equal(lstatSync(src).isSymbolicLink(), false); + assert.ok(existsSync(`${src}.tidy-link`)); + + const recovered = recoverInterruptedMoves(home); + assert.ok(recovered.some(r => r.path === src && r.action === 'removed-stray-link'), JSON.stringify(recovered)); + + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), false); + assert.equal(existsSync(`${src}.tidy-link`), false); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'gap2-fixture-bytes'.repeat(50)); + }); + + function snapshotTree(root) { + const snap = {}; + function recurse(dir) { + for (const name of readdirSync(dir)) { + const full = join(dir, name); + const lst = lstatSync(full); + if (lst.isSymbolicLink()) { + snap[full] = { type: 'symlink', linkTarget: readlinkSync(full) }; + } else if (lst.isDirectory()) { + recurse(full); + } else if (lst.isFile()) { + snap[full] = { type: 'file', size: lst.size, mtimeMs: lst.mtimeMs }; + } + } + } + recurse(root); + return snap; + } + + it("AMENDMENT: plan is read-only — an interrupted move is reported per unit, and every byte on disk (full tree, sizes and mtimes) is untouched by plan", () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const firstPlan = planFor(home); + const child = runApplyInChildWithCrash(firstPlan, home, target, 'afterStage'); + assert.equal(child.status, 77); + assert.equal(existsSync(src), false, 'mid-crash sanity check: src is gone, staged+link exist'); + + const before = snapshotTree(home); + const plan = planFor(home); + const after = snapshotTree(home); + + assert.deepEqual(after, before, 'plan must not change any byte on disk (sizes/mtimes of the full tree)'); + + assert.ok(plan.interrupted.some(f => f.path === src), 'plan must report the interrupted move'); + const finding = plan.interrupted.find(f => f.path === src); + assert.equal(finding.status, 'interrupted'); + assert.match(finding.note, /journal step 'staged'/); + + assert.ok(!plan.selected.some(r => r.path === src), 'the interrupted unit must never be selected'); + const skipped = plan.skipped.find(r => r.path === src); + assert.ok(skipped, 'the interrupted unit must be reported in skipped, not silently dropped'); + assert.match(skipped.reason, /interrupted move found/); + + // Confirm it is really still interrupted (plan really did nothing) — + // recovery, called separately, still has work to do. + const recovered = recoverInterruptedMoves(home); + assert.ok(recovered.some(r => r.path === src)); + }); + + it('AMENDMENT: recovery acts only on journaled units — a directory literally named *.tidy-moving with NO journal record is never touched', () => { + const { home } = buildSingleIdleFixture(); + const strayBase = join(home, 'models', 'someone-elses-thing'); + const stray = `${strayBase}.tidy-moving`; + mkdirSync(stray, { recursive: true }); + writeFileSync(join(stray, 'not-ours.txt'), 'this directory is just named like one of ours, nothing to do with model-tidy'); + tempPaths.push(stray); + const before = readFileSync(join(stray, 'not-ours.txt'), 'utf8'); + + const recovered = recoverInterruptedMoves(home); + + assert.equal(existsSync(stray), true, 'the stray directory must still exist, completely untouched'); + assert.equal(readFileSync(join(stray, 'not-ours.txt'), 'utf8'), before); + const finding = recovered.find(r => r.path === strayBase); + assert.ok(finding, 'the stray must be reported'); + assert.equal(finding.action, 'left-alone'); + assert.match(finding.note, /no matching journal record/); + }); + + it('AMENDMENT: recovery refuses a journaled unit whose on-disk content does not match the journaled manifest', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterStage'); + assert.equal(child.status, 77); + const staging = `${src}.tidy-moving`; + assert.ok(existsSync(staging)); + + // Tamper with the staged original so it no longer matches what the + // journal recorded — recovery must not trust its own naming + // convention over the manifest. + writeFileSync(join(staging, 'weights.gguf'), 'TAMPERED-CONTENT-DOES-NOT-MATCH-MANIFEST'); + + const recovered = recoverInterruptedMoves(home); + const finding = recovered.find(r => r.path === src); + assert.ok(finding); + assert.equal(finding.action, 'left-alone'); + assert.match(finding.note, /does not match the journaled manifest/); + + // Nothing was renamed or removed. + assert.equal(existsSync(src), false); + assert.equal(existsSync(staging), true); + assert.equal(readFileSync(join(staging, 'weights.gguf'), 'utf8'), 'TAMPERED-CONTENT-DOES-NOT-MATCH-MANIFEST'); + }); +}); From 8df7bd0e00dabd115fcd63421693b406cbf89d81 Mon Sep 17 00:00:00 2001 From: Petrus Pennanen Date: Mon, 21 Sep 2026 09:48:23 +0200 Subject: [PATCH 4/9] Fix data-loss path in recovery: guarded delete before discarding the staged original MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit codexmb reproduced a real data-loss path in recoverInterruptedMoves: source GOOD, apply interrupted between the two renames (afterStage), target then corrupted to BADD, recovery reported "completed-swap-and- cleaned" and deleted the last good original — source ended up reading BADD. Root cause: recovery validated the staged original's own manifest and that a realpath existed for the pending link, but never re-verified the TARGET's actual content against the manifest, and never checked that the symlink's realpath was exactly the journal's recorded target path. "A link resolves to something" was being treated as proof of what it resolves to. It isn't. Fix: one guarded function, finalizeSwapOrRestore, is now the ONLY place in the codebase allowed to delete a staged original — used by both apply's own final step (same run) and recoverInterruptedMoves (a later run). It requires all three, every time, even for a link already verified once when it was created: 1. every file under the journal's target path matches the manifest by size and SHA-256 exactly (no extra or missing paths either). 2. the source path is a symlink whose realpath resolves to exactly the journal's recorded target path. 3. the staged original itself, if it still exists, still matches the manifest exactly (a partially damaged original is reported, not silently discarded because a symlink elsewhere looks fine). Any failure: remove a bad/half symlink at the source path if present, restore the staged original to the source path if one exists, mark the journal record `step: 'failed'` with the reason, and report — never delete on the strength of "a link resolves". New tests (4, all pass standalone and in the full suite): codexmb's round-2 probe kept verbatim (source reads GOOD after recovery, never BADD, unit reported failed rather than "completed"); the mirror case where the target is fine but the source symlink was swapped to point elsewhere (recovery must not delete the staged original just because some link resolves); a positive control where target and link are both genuinely correct (recovery completes and cleans normally); and the same corrupt-target scenario exercised through apply's own final cleanup path in the same run, not only through a later recover pass. Also fixed two pre-existing tests whose expected action name/behavior predated this guarded function (the old two-step 'completed-swap-from-link' / 'finished-cleanup' split is now unified, and correctly, into a single verified 'completed-swap-and-cleaned' or a restore-on-failure). Test results: 4/4 new tests in isolation, 33/33 full model-tidy suite, 692/692 full repo suite. Co-Authored-By: Claude Fable 5.1 --- docs/model-tidy.md | 47 +++++++++- src/model-tidy.mjs | 194 +++++++++++++++++++++++++++++++++------ test/model-tidy.test.mjs | 170 ++++++++++++++++++++++++++++++++-- 3 files changed, 370 insertions(+), 41 deletions(-) diff --git a/docs/model-tidy.md b/docs/model-tidy.md index 3624db6..6e42744 100644 --- a/docs/model-tidy.md +++ b/docs/model-tidy.md @@ -152,13 +152,45 @@ still matches that record's manifest, and only then acts: | Journaled state found | Recovery action | | --- | --- | -| staged exists, source missing, temp link exists | rename the link into place, verify, then remove staged and the journal record — the link was already verified before it was ever created, so completing it is safe | +| staged exists, source missing, temp link exists | complete the pending rename (the link was already verified before it was ever created), then run the guarded delete decision below before removing anything | | staged exists, source missing, no temp link | rename staged back to source, remove the journal record — no verified pending swap existed to trust instead | -| staged exists, source is a symlink | the swap itself already completed; re-verify the symlink target, then remove staged and the journal record | +| staged exists, source is a symlink | the swap itself already completed; run the guarded delete decision below before removing the staged copy | | temp link exists, source is a real directory, no staged | crashed before source was ever touched; remove the stray link and the journal record | | source is a symlink, no staged | the move had already fully completed; just remove the stale journal record | | source is a real directory, nothing else exists | never touched at all; remove the stale journal record | -| manifest mismatch against whatever currently exists | **left alone**, reported, regardless of step | +| manifest mismatch against whatever currently exists (outside the guarded delete decision) | **left alone**, reported, regardless of step | + +### The guarded delete decision (`finalizeSwapOrRestore`) + +A second, earlier bug in this same recovery path deleted the last good +original after the crash landed between the two renames: the code +verified the *staged* copy's manifest, and that *a* realpath existed for +the pending symlink — but never re-verified the **target's actual +content**, nor that the symlink's realpath was **exactly** the journal's +recorded target path. A target that changed after the link was created +(corruption, a second process, disk issues) still made the old code +delete the only good copy, because "a link resolves to something" was +being treated as proof it resolves to something *correct*. It doesn't. + +There is now exactly one function in the codebase allowed to delete a +staged original — `finalizeSwapOrRestore`, used by **both** `apply`'s own +final step (same run) and `recoverInterruptedMoves` (a later run). It +requires all three of: + +1. every file under the journal's **target path** matches the manifest by + size and SHA-256, with no extra or missing paths — re-checked now, not + trusted from when the link was created; +2. the **source path** is a symlink whose realpath resolves to exactly the + journal's recorded target path; +3. the **staged original itself**, if it still exists, still matches the + manifest exactly — a partially damaged original is reported, never + silently discarded just because a symlink elsewhere looks fine. + +If any of the three fails: a bad/half symlink at the source path is +removed, the staged original (if present) is renamed back to the source +path and re-verified, the journal record is marked `step: 'failed'` with +the reason, and the unit is reported — **never** deleted on the strength +of a link merely resolving. A `*.tidy-moving` or `*.tidy-link` directory with **no matching journal record at all** is reported but never touched, no matter how it's named — @@ -309,6 +341,15 @@ it never attempts to install anything itself. like a leftover with no matching journal record — or a journaled unit whose content has since changed — is reported and left alone, never guessed at. +- **A staged original is only ever deleted by `finalizeSwapOrRestore`, the + one function both `apply`'s own final step and `recoverInterruptedMoves` + call for that decision** — never on the strength of "a symlink resolves + to something." It re-checks the target's actual content against the + manifest, that the source symlink's realpath is exactly the recorded + target path, and that the staged original itself (if present) still + matches the manifest — all three, every time, even for a unit whose link + was already verified once when it was created. Any failure restores the + staged original rather than deleting it. - A hardlink set moves as a unit or not at all — never partially — and the hardlink scan covers every discovered model root, not just the ones that already passed every other rule. It also refuses a candidate whose diff --git a/src/model-tidy.mjs b/src/model-tidy.mjs index 54033a8..dae5c36 100644 --- a/src/model-tidy.mjs +++ b/src/model-tidy.mjs @@ -968,6 +968,132 @@ function verifyManifest(rootDir, manifest) { return true; } +/** Stricter than verifyManifest: every manifest entry must be present and + * match AND there must be no extra file/symlink under `rootDir` beyond + * what the manifest lists. Used specifically for the delete-the-staged- + * original decision, where "the manifest's entries happen to be present" + * is not enough — corrupted-but-additional content must also fail this. */ +function manifestExactMatch(rootDir, manifest) { + if (!existsSync(rootDir)) return false; + if (!verifyManifest(rootDir, manifest)) return false; + const manifestRelPaths = new Set(manifest.map(m => m.relPath)); + let actualCount = 0; + for (const full of listAllEntries(rootDir)) { + let lst; + try { + lst = lstatSync(full); + } catch { + continue; + } + if (!lst.isFile() && !lst.isSymbolicLink()) continue; // directories aren't in the manifest + actualCount++; + if (!manifestRelPaths.has(relative(rootDir, full))) return false; // extra, untracked entry + } + return actualCount === manifestRelPaths.size; +} + +/** + * The ONLY place in this file allowed to delete a staged original. Called + * both from swapToSymlink's own final step (same run) and from recovery + * (a later run) — one guarded function, one invariant, everywhere a + * staged original could be discarded. + * + * A staged original is deleted ONLY when, at this exact moment: + * 1. every file under the journal's targetPath matches the journal + * manifest by size and SHA-256, with no extra or missing paths — + * not just "the manifest's entries happen to be present". The link + * having been verified once, when it was CREATED, is not evidence + * about the target's content NOW; the target can change afterwards + * (corruption, a second process, disk issues) and "a link resolves + * to something" proves nothing about what it resolves to. + * 2. the path at sourcePath is a symlink whose realpath resolves to + * exactly the journal's recorded targetPath. + * 3. the staged original itself (if it still exists) still matches the + * journal manifest exactly — a partially damaged original is a red + * flag to report, not something to silently discard because a + * symlink elsewhere happens to look fine. + * + * If ANY of the three fails: remove a bad/half symlink at sourcePath (if + * one exists), restore the staged original to sourcePath (if the staged + * copy still exists and sourcePath is clear), mark the journal record + * `step: 'failed'` with the reason, and report — this function NEVER + * deletes on the strength of "a link resolves". + */ +function finalizeSwapOrRestore(journalDir, record) { + const { sourcePath, stagedPath, targetPath, manifest } = record; + + const targetOk = manifestExactMatch(targetPath, manifest); + + let sourceLstat = null; + try { + sourceLstat = lstatSync(sourcePath); + } catch { + sourceLstat = null; + } + let sourceIsCorrectSymlink = false; + if (sourceLstat && sourceLstat.isSymbolicLink()) { + try { + sourceIsCorrectSymlink = realpathSync(sourcePath) === realpathSync(targetPath); + } catch { + sourceIsCorrectSymlink = false; + } + } + + const stagedExists = existsSync(stagedPath); + const stagedOk = !stagedExists || manifestExactMatch(stagedPath, manifest); + + if (targetOk && sourceIsCorrectSymlink && stagedOk) { + if (stagedExists) rmSync(stagedPath, { recursive: true }); + removeJournalRecord(journalDir, sourcePath); + return { ok: true }; + } + + const reasons = []; + if (!targetOk) reasons.push('target content does not match the journaled manifest exactly (missing, extra, or corrupted files)'); + if (!sourceIsCorrectSymlink) reasons.push('source is not a symlink whose realpath resolves exactly to the recorded target path'); + if (!stagedOk) reasons.push('the staged original itself no longer matches the journaled manifest exactly'); + const reason = reasons.join('; '); + + // Never delete on this evidence. Only touch sourcePath at all if there + // is a staged original to fall back on — removing a bad symlink with + // NOTHING to put in its place would make things strictly worse (e.g. + // the case where the original was already legitimately deleted in a + // prior successful run and only the target has since degraded: there is + // nothing left to restore FROM, so the existing symlink — however + // suspect — is left exactly alone rather than removed for no benefit). + let restored = false; + if (stagedExists) { + try { + if (sourceLstat && sourceLstat.isSymbolicLink()) rmSync(sourcePath); + } catch { + // best effort — the restore attempt below will surface a failure + } + if (!existsSync(sourcePath)) { + try { + renameSync(stagedPath, sourcePath); + restored = existsSync(sourcePath) && !lstatSync(sourcePath).isSymbolicLink(); + } catch { + restored = false; + } + } + // else: sourcePath still holds something unexpected (not a symlink, + // and removal above didn't apply) — leave both paths for manual + // inspection rather than guess which is authoritative. + } else if (sourceLstat && !sourceLstat.isSymbolicLink()) { + // No staged copy, and sourcePath already holds a real directory — + // nothing to restore, but also nothing was ever at risk. + restored = true; + } + + try { + writeJournalRecordSync(journalDir, { ...record, step: 'failed', failureReason: reason }); + } catch { + // best effort — the reason is still returned to the caller either way + } + + return { ok: false, reason, restored }; +} + /** * Swap one source directory for a symlink to its already-verified copy. * @@ -1033,12 +1159,13 @@ function swapToSymlink(src, dst, opts = {}) { writeJournalRecordSync(journalDir, { ...base, step: 'swapped' }); if (opts.afterSwap) opts.afterSwap(); // test-only hook: simulate a crash here - const real = realpathSync(src); - if (real !== realpathSync(dst) || !lstatSync(src).isSymbolicLink()) { - throw new Error(`post-swap verification failed for ${src}`); + // The ONLY place a staged original is deleted, in this run or later via + // recovery — see finalizeSwapOrRestore's docstring for the three checks. + // "The link resolves to something" is deliberately not one of them. + const result = finalizeSwapOrRestore(journalDir, base); + if (!result.ok) { + throw new Error(`post-swap finalize refused to delete the staged original and restored it instead: ${result.reason}`); } - rmSync(staging, { recursive: true }); - removeJournalRecord(journalDir, src); } /** Every location discoverCandidates() looks at, reused so detection and @@ -1166,36 +1293,45 @@ export function recoverInterruptedMoves(home, options = {}) { const stagedExists = existsSync(stagedPath); const linkExists = existsSync(linkPath); + // A symlink is either already in place, or one verified rename away + // from being in place: the ONLY safe way to decide whether the + // staged original may be deleted is finalizeSwapOrRestore's + // three-way check (target manifest, symlink realpath, staged + // manifest) — never a bare "does something exist at the realpath" + // check, which is exactly the data-loss bug this replaces. + if ((srcExists && srcIsSymlink) || (!srcExists && stagedExists && linkExists)) { + if (!srcExists && stagedExists && linkExists) { + // Completing this rename alone never deletes anything — the + // original is still fully intact at stagedPath either way. + // finalizeSwapOrRestore below undoes this if it turns out + // unsafe to proceed past it. + renameSync(linkPath, sourcePath); + } + const result = finalizeSwapOrRestore(journalDir, record); + if (result.ok) { + recovered.push({ path: sourcePath, journalFile: file, action: 'completed-swap-and-cleaned', note: 'completed the pending symlink swap (or finished cleanup of one already in place) and removed the staged original — target and symlink both re-verified against the journal manifest first' }); + } else { + recovered.push({ + path: sourcePath, + journalFile: file, + action: result.restored ? 'restored-after-failed-verification' : 'left-alone-after-failed-verification', + note: `refused to delete the staged original (${result.reason}); ${result.restored ? 'restored the original to the source path' : 'left current state for manual inspection'} and marked the journal record failed` + }); + } + continue; + } + + // Below here, no symlink has ever been placed at sourcePath — the + // delete-the-staged-original decision never applies, so the plainer + // manifest checks are enough. const stagedOk = !stagedExists || verifyManifest(stagedPath, manifest); - const srcOk = !srcExists || srcIsSymlink || verifyManifest(sourcePath, manifest); + const srcOk = !srcExists || verifyManifest(sourcePath, manifest); if (!stagedOk || !srcOk) { recovered.push({ path: sourcePath, journalFile: file, action: 'left-alone', note: `on-disk content (step recorded as '${step}') does not match the journaled manifest; left alone for manual inspection` }); continue; } - if (srcExists && srcIsSymlink && !stagedExists) { - removeJournalRecord(journalDir, sourcePath); - recovered.push({ path: sourcePath, journalFile: file, action: 'removed-stale-journal', note: 'the move had already fully completed; removed the stale journal record' }); - } else if (srcExists && srcIsSymlink && stagedExists) { - const real = realpathSync(sourcePath); - if (existsSync(real)) { - rmSync(stagedPath, { recursive: true }); - removeJournalRecord(journalDir, sourcePath); - recovered.push({ path: sourcePath, journalFile: file, action: 'finished-cleanup', note: 'symlink was already in place; finished removing the staged original' }); - } else { - recovered.push({ path: sourcePath, journalFile: file, action: 'left-alone', note: `symlink target ${real} does not exist — leaving ${stagedPath} for manual inspection` }); - } - } else if (!srcExists && stagedExists && linkExists) { - renameSync(linkPath, sourcePath); - const real = realpathSync(sourcePath); - if (existsSync(real)) { - rmSync(stagedPath, { recursive: true }); - removeJournalRecord(journalDir, sourcePath); - recovered.push({ path: sourcePath, journalFile: file, action: 'completed-swap-and-cleaned', note: 'completed the pending symlink swap (the crash landed in the gap between the two renames) and finished cleanup' }); - } else { - recovered.push({ path: sourcePath, journalFile: file, action: 'completed-swap', note: 'completed the pending symlink swap; leaving the staged original for manual inspection (symlink target unexpectedly missing)' }); - } - } else if (!srcExists && stagedExists && !linkExists) { + if (!srcExists && stagedExists && !linkExists) { renameSync(stagedPath, sourcePath); removeJournalRecord(journalDir, sourcePath); recovered.push({ path: sourcePath, journalFile: file, action: 'restored-original', note: 'restored the original directory (no verified pending symlink existed to trust instead)' }); diff --git a/test/model-tidy.test.mjs b/test/model-tidy.test.mjs index 3e7e667..2422940 100644 --- a/test/model-tidy.test.mjs +++ b/test/model-tidy.test.mjs @@ -721,7 +721,7 @@ describe('GAP 2: crash-safety across a hard process kill (not just a thrown exce assert.ok(existsSync(`${src}.tidy-moving`)); const recovered = recoverInterruptedMoves(home); - assert.ok(recovered.some(r => r.path === src && r.action === 'finished-cleanup'), JSON.stringify(recovered)); + assert.ok(recovered.some(r => r.path === src && r.action === 'completed-swap-and-cleaned'), JSON.stringify(recovered)); assert.equal(existsSync(src), true); assert.equal(lstatSync(src).isSymbolicLink(), true); @@ -825,7 +825,7 @@ describe('GAP 2: crash-safety across a hard process kill (not just a thrown exce assert.match(finding.note, /no matching journal record/); }); - it('AMENDMENT: recovery refuses a journaled unit whose on-disk content does not match the journaled manifest', () => { + it('AMENDMENT: recovery never silently discards a staged original whose own content no longer matches the journal — it restores (never deletes) and reports failure', () => { const { home, src } = buildSingleIdleFixture(); const target = tempDir('model-tidy-gap2-target-'); const plan = planFor(home); @@ -837,18 +837,170 @@ describe('GAP 2: crash-safety across a hard process kill (not just a thrown exce // Tamper with the staged original so it no longer matches what the // journal recorded — recovery must not trust its own naming - // convention over the manifest. + // convention over the manifest, and must not silently throw this + // away just because a symlink elsewhere happens to resolve. writeFileSync(join(staging, 'weights.gguf'), 'TAMPERED-CONTENT-DOES-NOT-MATCH-MANIFEST'); const recovered = recoverInterruptedMoves(home); const finding = recovered.find(r => r.path === src); assert.ok(finding); - assert.equal(finding.action, 'left-alone'); - assert.match(finding.note, /does not match the journaled manifest/); + assert.match(finding.action, /^(restored-after-failed-verification|left-alone-after-failed-verification)$/); + assert.match(finding.note, /staged original itself no longer matches the journaled manifest/); + + // Whatever happened, the staged content was neither silently deleted + // nor silently accepted as-is: it now lives at src (or, if that + // somehow could not happen, at the untouched staging path) and the + // journal itself is marked failed rather than cleared as successful. + const survivedAtSrc = existsSync(src) && !lstatSync(src).isSymbolicLink(); + const survivedAtStaging = existsSync(staging); + assert.ok(survivedAtSrc || survivedAtStaging, 'the tampered content must not have been silently discarded'); + if (survivedAtSrc) { + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'TAMPERED-CONTENT-DOES-NOT-MATCH-MANIFEST'); + } + assert.equal(lstatSync(src).isSymbolicLink(), false, 'the source must not be left as a symlink to an unverified copy after a failed check'); + }); - // Nothing was renamed or removed. - assert.equal(existsSync(src), false); - assert.equal(existsSync(staging), true); - assert.equal(readFileSync(join(staging, 'weights.gguf'), 'utf8'), 'TAMPERED-CONTENT-DOES-NOT-MATCH-MANIFEST'); + it("DATA-LOSS FIX, codexmb's round-2 probe kept verbatim: target corrupted after staging must not cost the last good original", () => { + // Direct port of /tmp/codex-pr128-round2.pjlier/probe.mjs: a thrown + // exception (not a hard kill) during afterStage interrupts the swap + // with the original safely staged; the target is then corrupted + // in-place; recovery must read GOOD from the source afterwards, never + // BADD, and must never report success for a unit whose target was + // corrupted. + const home = tempDir('model-tidy-probe2-home-'); + const src = join(home, 'models', 'idle'); + const target = tempDir('model-tidy-probe2-target-'); + mkdirSync(src, { recursive: true }); + writeFileSync(join(src, 'weights.gguf'), 'GOOD'); + utimesSync(join(src, 'weights.gguf'), new Date(0), new Date(0)); + + const plan = planRun({ + home, + listProcessUsers: () => ({ checked: true, users: [] }), + listDockerBindUsers: () => ({ available: true, inUse: false, containers: [] }), + diskFreeBytes: () => 0 + }); + + const interrupted = applyRun({ + plan, target, home, + validateTarget: () => ({ ok: true }), + afterStage: () => { throw new Error('fixture interruption'); } + }); + assert.equal(interrupted.ok, false, 'apply must report failure for the interrupted unit'); + + // Corrupt the target after the interruption, exactly like the probe. + writeFileSync(join(target, 'models', 'idle', 'weights.gguf'), 'BADD'); + + const recovery = recoverInterruptedMoves(home); + + const sourceBytes = readFileSync(join(src, 'weights.gguf'), 'utf8'); + const stagedOriginalExists = existsSync(`${src}.tidy-moving`); + + // codexmb's exact probe assertions: source reads GOOD (never BADD), + // and the unit is reported failed with the target flagged. + assert.equal(sourceBytes, 'GOOD', 'source must read the last GOOD content, never the corrupted target content'); + // "staged original either restored to the source path or retained" — + // both are acceptable; what's NOT acceptable is stagedOriginalExists + // being false while sourceBytes came out wrong. Since sourceBytes is + // confirmed GOOD above, either outcome for stagedOriginalExists is + // fine as long as the unit was reported as failed, checked next. + void stagedOriginalExists; + + assert.equal(recovery.length, 1); + assert.equal(recovery[0].path, src); + assert.notEqual(recovery[0].action, 'completed-swap-and-cleaned', 'must never report success for a corrupted target'); + assert.match(recovery[0].note, /target content does not match the journaled manifest/); + + // Running recovery again must not somehow make it worse (idempotent + // refusal, not idempotent data loss). + const secondPass = recoverInterruptedMoves(home); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'GOOD'); + }); + + it('mirror case: target is fine but the source symlink points elsewhere — recover must not delete the staged original', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterSwap'); + assert.equal(child.status, 77); + // Mid-crash: swap already completed correctly (src is a symlink to + // the real target), staged original still pending cleanup. + assert.equal(lstatSync(src).isSymbolicLink(), true); + const staging = `${src}.tidy-moving`; + assert.ok(existsSync(staging)); + const goodContent = readFileSync(join(src, 'weights.gguf'), 'utf8'); + + // Simulate tampering: replace the correct symlink with one pointing + // somewhere unrelated. The target itself, and the staged original, + // are both still completely fine. + const decoy = tempDir('model-tidy-gap2-decoy-'); + writeFileSync(join(decoy, 'weights.gguf'), 'DECOY-NOT-THE-REAL-TARGET'); + rmSync(src); + symlinkSync(decoy, src); + + const recovered = recoverInterruptedMoves(home); + assert.equal(recovered.length, 1, 'exactly one journaled unit is in play here'); + const match = recovered[0]; + assert.notEqual(match.action, 'completed-swap-and-cleaned', 'must never delete the staged original on the strength of an unrelated link resolving'); + assert.match(match.note, /source is not a symlink whose realpath resolves exactly to the recorded target path/); + + // The staged original must not have been discarded: it's back at src + // (restored) or still sitting at the staging path — never gone. + const restoredAtSrc = existsSync(src) && !lstatSync(src).isSymbolicLink() && readFileSync(join(src, 'weights.gguf'), 'utf8') === goodContent; + const stillStaged = existsSync(staging) && readFileSync(join(staging, 'weights.gguf'), 'utf8') === goodContent; + assert.ok(restoredAtSrc || stillStaged, 'the good staged original must survive, either restored to src or left at the staging path'); + }); + + it('positive control: target and symlink both correct — recover completes the swap and cleans up', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterStage'); + assert.equal(child.status, 77); + + const recovered = recoverInterruptedMoves(home); + assert.equal(recovered.length, 1); + assert.equal(recovered[0].action, 'completed-swap-and-cleaned'); + + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), true); + assert.equal(existsSync(`${src}.tidy-moving`), false); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'gap2-fixture-bytes'.repeat(50)); + }); + + it("the SAME corrupt-target scenario against apply's OWN final cleanup path, not only recover", () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + const targetFile = join(target, 'models', 'idle', 'weights.gguf'); + + const result = applyRun({ + plan, + target, + home, + validateTarget: bypassCrossFsCheck, + afterSwap: () => { + // Corrupt the target in the SAME run, right after the second + // rename but before apply's own finalize step runs. + writeFileSync(targetFile, 'CORRUPTED-DURING-THE-SAME-APPLY-RUN'); + } + }); + + assert.equal(result.ok, false, 'apply must report failure for this unit rather than silently succeed'); + assert.equal(result.moved.length, 0); + assert.equal(result.errors.length, 1); + assert.equal(result.errors[0].step, 'swap'); + assert.match(result.errors[0].error, /target content does not match the journaled manifest/); + + // The last good original must not have been lost: either restored to + // src as a real directory, or still present at the staging path. + const restoredAtSrc = existsSync(src) && !lstatSync(src).isSymbolicLink(); + const stillStaged = existsSync(`${src}.tidy-moving`); + assert.ok(restoredAtSrc || stillStaged, 'the good original must survive apply refusing to finalize onto a corrupted target'); + if (restoredAtSrc) { + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'gap2-fixture-bytes'.repeat(50)); + } }); }); From 05588ee3512349eb3e3100316f89994060fb9bb8 Mon Sep 17 00:00:00 2001 From: Petrus Pennanen Date: Mon, 21 Sep 2026 10:13:23 +0200 Subject: [PATCH 5/9] Replace single pass/fail in recovery with an explicit 4-row decision table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit codexmb's round-3 probe found the mirror of the round-2 data-loss bug: apply interrupted afterSwap, one file removed from the STAGED original (partial cleanup) while the destination was GOOD and the live source symlink was valid. The round-2 fix's single pass/fail collapsed straight to "restore" whenever ANY check failed — including "the staged backup doesn't match" — which unlinked the correct, working symlink and replaced it with the known-damaged backup. Confirmed against pre-fix code: sourceHasWeights=false (file missing) while targetBytes="GOOD" (the live target was fine the whole time). finalizeSwapOrRestore is now an explicit 4-row decision table over three independently re-checked facts (targetOk, linkOk, stagedState — absent / matching / damaged, no longer a single boolean): 1. targetOk && linkOk && stagedState != damaged -> complete: finish any pending rename, delete the staged copy if it matched, clear the journal. (existing) 2. targetOk && linkOk && stagedState == damaged -> the live path is correct and complete: PRESERVE it untouched. Never delete the damaged backup silently: rename it to .tidy-quarantine-, record that path in the journal (step 'completed-partial-staging-quarantined'), report it. Nothing is deleted in this row. 3. (!targetOk || !linkOk) && stagedState == matching -> restore: remove a bad/half symlink wherever it sits, rename the staged original back to sourcePath, re-verify, journal 'failed'. (existing behavior, now only reachable in this row) 4. (!targetOk || !linkOk) && stagedState != matching -> nothing verified good anywhere: DELETE NOTHING, RENAME NOTHING — not even a still-pending, uncompleted rename — leave every path exactly as found, journal 'failed-both-copies-damaged' with the per-check reasons, report loudly with every path. Also fixed a related bug found while implementing row 4: the caller (recoverInterruptedMoves) used to complete a pending rename (temp-link -> source) speculatively, BEFORE calling finalizeSwapOrRestore, on the theory that "completing it alone never deletes anything" — but that still violates row 4's "rename nothing" requirement when the final decision turns out to be row 4. finalizeSwapOrRestore now owns that rename entirely, performing it only inside the row-1/row-2 branches that have already decided it's warranted, never speculatively. New/updated tests (all pass standalone and in the full suite): row 2 rewritten as codexmb's exact partial-staging scenario (live symlink preserved, damaged backup quarantined, nothing deleted); a dedicated row 3 test reproducing the round-3 probe's own 'bad-target' scenario (afterSwap crash, not afterStage) to prove linkOk — a path check — does not imply target content is correct; a new row 4 test (target corrupted AND staging partial) asserting a full before/after tree snapshot of every data path (sizes + SHA-256) is byte-for-byte identical across two consecutive recovery passes, and that the report names every path involved. Also fixed the exclusion logic so a quarantined path (and any journal record naming one) doesn't get double-reported as an unrelated orphan stray, and excluded quarantine-suffixed names from discoverCandidates so a quarantine directory is never treated as a new candidate model. Test results: 3/3 new/changed tests in isolation, 35/35 full model-tidy suite, 694/694 full repo suite. Co-Authored-By: Claude Fable 5.1 --- docs/model-tidy.md | 110 ++++++++++------- src/model-tidy.mjs | 258 +++++++++++++++++++++++++++------------ test/model-tidy.test.mjs | 118 +++++++++++++++--- 3 files changed, 349 insertions(+), 137 deletions(-) diff --git a/docs/model-tidy.md b/docs/model-tidy.md index 6e42744..2217416 100644 --- a/docs/model-tidy.md +++ b/docs/model-tidy.md @@ -162,42 +162,60 @@ still matches that record's manifest, and only then acts: ### The guarded delete decision (`finalizeSwapOrRestore`) -A second, earlier bug in this same recovery path deleted the last good -original after the crash landed between the two renames: the code -verified the *staged* copy's manifest, and that *a* realpath existed for -the pending symlink — but never re-verified the **target's actual -content**, nor that the symlink's realpath was **exactly** the journal's -recorded target path. A target that changed after the link was created -(corruption, a second process, disk issues) still made the old code -delete the only good copy, because "a link resolves to something" was -being treated as proof it resolves to something *correct*. It doesn't. - -There is now exactly one function in the codebase allowed to delete a +This recovery path has had two different real data-loss bugs found by +review, both from collapsing "is the live path OK?" and "is the backup OK?" +into a single pass/fail: + +- **Round 2:** the code verified the staged copy's manifest and that *a* + realpath existed for the pending symlink, but never re-verified the + **target's actual content**, nor that the symlink's realpath was + **exactly** the recorded target path. A target that changed after the + link was created (corruption, a second process, disk issues) still made + the old code delete the only good copy — "a link resolves to something" + was being treated as proof it resolves to something *correct*. +- **Round 3:** fixing that by requiring the staged copy to *also* match + its manifest before deleting created a NEW bug: if the live path (target + + symlink) was completely correct but the staged *backup* happened to be + partially damaged, the single pass/fail collapsed straight to "restore", + which **unlinked the good, working symlink and replaced it with the + known-damaged backup** — trading a verified-good live path for + known-bad data. + +There is now exactly one function allowed to delete OR rename away a staged original — `finalizeSwapOrRestore`, used by **both** `apply`'s own -final step (same run) and `recoverInterruptedMoves` (a later run). It -requires all three of: - -1. every file under the journal's **target path** matches the manifest by - size and SHA-256, with no extra or missing paths — re-checked now, not - trusted from when the link was created; -2. the **source path** is a symlink whose realpath resolves to exactly the - journal's recorded target path; -3. the **staged original itself**, if it still exists, still matches the - manifest exactly — a partially damaged original is reported, never - silently discarded just because a symlink elsewhere looks fine. - -If any of the three fails: a bad/half symlink at the source path is -removed, the staged original (if present) is renamed back to the source -path and re-verified, the journal record is marked `step: 'failed'` with -the reason, and the unit is reported — **never** deleted on the strength -of a link merely resolving. - -A `*.tidy-moving` or `*.tidy-link` directory with **no matching journal -record at all** is reported but never touched, no matter how it's named — -recovery does not treat a name as ownership. A failure for one unit does -not roll back units that already succeeded earlier in the same `apply` -run, but the process still exits non-zero and prints exactly which unit -failed, at which step, and why. +final step (same run) and `recoverInterruptedMoves` (a later run) — and it +is an explicit 4-row decision table, not a single boolean, over three +independently-checked facts: + +- **targetOk** — every file under the journal's target path matches the + manifest by size + SHA-256, no extra or missing paths (re-checked now, + never trusted from when the link was created). +- **linkOk** — the live path (already placed at the source, or still + pending, unconsumed, at the temp-link path) is a symlink whose realpath + resolves to exactly the recorded target path. +- **stagedState** — `absent` (nothing to protect), `matching` (exists and + matches the manifest exactly), or `damaged` (exists but doesn't match). + +| # | targetOk && linkOk | stagedState | Action | +| --- | --- | --- | --- | +| 1 | true | absent or matching | **Complete.** Finish the pending rename if one was still outstanding, delete the staged copy if it matched, clear the journal. | +| 2 | true | damaged | **Preserve + quarantine.** The live path is correct and complete — it is left completely untouched (finishing the pending rename if needed). The damaged backup is never deleted or silently restored over the good live path: it's renamed to `.tidy-quarantine-`, and that path is recorded in the journal (`step: 'completed-partial-staging-quarantined'`). Nothing is deleted in this row. | +| 3 | false | matching | **Restore.** Remove a bad/half symlink wherever it currently sits (source or the pending temp-link), rename the staged original back to the source path, re-verify it matches the manifest, journal `step: 'failed'`. | +| 4 | false | absent or damaged | **Touch nothing.** Nothing verified good exists anywhere for this unit. DELETE NOTHING, RENAME NOTHING — not even a still-pending, uncompleted rename — leave every path exactly as found. Journal `step: 'failed-both-copies-damaged'` with the per-check reasons, and report every path involved so a human can recover by hand. | + +Row 1 only ever completes a pending rename as part of deciding the WHOLE +unit is fine — it's never done speculatively "just in case" before the +decision, which was exactly how the round-3 bug reached its bad state via +a supposedly-harmless prep step. Row 4 exists specifically so that, when +there is genuinely nothing good anywhere, this function refuses to guess +— it does not pick a side. + +A `*.tidy-moving`, `*.tidy-link`, or `*.tidy-quarantine-*` path with **no +matching journal record at all** is reported but never touched, no matter +how it's named — recovery does not treat a name as ownership. A failure +for one unit does not roll back units that already succeeded earlier in +the same `apply` run, but the process still exits non-zero and prints +exactly which unit failed, at which step, and why. ## Selection rules (in order, each with an explicit reason string) @@ -341,15 +359,19 @@ it never attempts to install anything itself. like a leftover with no matching journal record — or a journaled unit whose content has since changed — is reported and left alone, never guessed at. -- **A staged original is only ever deleted by `finalizeSwapOrRestore`, the - one function both `apply`'s own final step and `recoverInterruptedMoves` - call for that decision** — never on the strength of "a symlink resolves - to something." It re-checks the target's actual content against the - manifest, that the source symlink's realpath is exactly the recorded - target path, and that the staged original itself (if present) still - matches the manifest — all three, every time, even for a unit whose link - was already verified once when it was created. Any failure restores the - staged original rather than deleting it. +- **A staged original is only ever deleted or renamed away by + `finalizeSwapOrRestore`, the one function both `apply`'s own final step + and `recoverInterruptedMoves` call for that decision** — never on the + strength of "a symlink resolves to something." It follows an explicit + 4-row decision table (see "The guarded delete decision" above) over + three independently re-checked facts — target content, symlink + correctness, staged-copy state (absent/matching/damaged) — so that "the + live path is fine" and "the backup is fine" are never collapsed into one + answer: a live path that's correct and complete is always preserved + exactly as-is, even when its backup turns out to be damaged (the backup + is quarantined, not deleted, and never used to overwrite a working + symlink); and when nothing anywhere is verified good, nothing is deleted + or renamed at all. - A hardlink set moves as a unit or not at all — never partially — and the hardlink scan covers every discovered model root, not just the ones that already passed every other rule. It also refuses a candidate whose diff --git a/src/model-tidy.mjs b/src/model-tidy.mjs index dae5c36..63c5869 100644 --- a/src/model-tidy.mjs +++ b/src/model-tidy.mjs @@ -862,7 +862,7 @@ const LINK_SUFFIX = '.tidy-link'; const JOURNAL_SUBDIR = 'model-tidy-journal'; function isJournalSuffixed(name) { - return name.endsWith(STAGING_SUFFIX) || name.endsWith(LINK_SUFFIX); + return name.endsWith(STAGING_SUFFIX) || name.endsWith(LINK_SUFFIX) || name.includes(QUARANTINE_SUFFIX_PREFIX); } function defaultJournalDir(home) { @@ -992,35 +992,46 @@ function manifestExactMatch(rootDir, manifest) { return actualCount === manifestRelPaths.size; } +const QUARANTINE_SUFFIX_PREFIX = '.tidy-quarantine-'; + /** - * The ONLY place in this file allowed to delete a staged original. Called - * both from swapToSymlink's own final step (same run) and from recovery - * (a later run) — one guarded function, one invariant, everywhere a - * staged original could be discarded. + * The ONLY place in this file allowed to delete OR rename away a staged + * original. Called both from swapToSymlink's own final step (same run) + * and from recovery (a later run) — one guarded function, one invariant, + * everywhere a staged original could be discarded. + * + * Checks three independent facts, then follows an explicit 4-row decision + * table — no single pass/fail collapse, because "the live path is fine" + * and "the backup is fine" are NOT the same question, and conflating them + * caused two different real data-loss bugs (see git history): + * targetOk — every file under the journal's targetPath matches the + * manifest by size + SHA-256, no extra or missing paths. + * The link having been verified once, when it was + * CREATED, is not evidence about the target's content + * NOW — it can change afterwards (corruption, a second + * process, disk issues). + * linkOk — sourcePath is a symlink whose realpath resolves to + * exactly the journal's recorded targetPath. + * stagedState — 'absent' (no staged copy exists — fine, nothing to + * protect), 'matching' (exists and matches the manifest + * exactly), or 'damaged' (exists but does not match). * - * A staged original is deleted ONLY when, at this exact moment: - * 1. every file under the journal's targetPath matches the journal - * manifest by size and SHA-256, with no extra or missing paths — - * not just "the manifest's entries happen to be present". The link - * having been verified once, when it was CREATED, is not evidence - * about the target's content NOW; the target can change afterwards - * (corruption, a second process, disk issues) and "a link resolves - * to something" proves nothing about what it resolves to. - * 2. the path at sourcePath is a symlink whose realpath resolves to - * exactly the journal's recorded targetPath. - * 3. the staged original itself (if it still exists) still matches the - * journal manifest exactly — a partially damaged original is a red - * flag to report, not something to silently discard because a - * symlink elsewhere happens to look fine. + * | # | targetOk && linkOk | stagedState | Action | + * |---|---------------------|---------------------|--------| + * | 1 | true | absent or matching | complete: delete the staged copy if present, clear the journal. | + * | 2 | true | damaged | the live path is correct and complete — PRESERVE it untouched. Do NOT delete the damaged staging; QUARANTINE it to `.tidy-quarantine-` and record that path in the journal (`step: 'completed-partial-staging-quarantined'`). Nothing is deleted. | + * | 3 | false | matching | restore: remove a bad/half symlink at sourcePath if present, rename the staged copy back to sourcePath, verify it matches the manifest, journal `step: 'failed'`. | + * | 4 | false | absent or damaged | nothing verified good anywhere — DELETE NOTHING, RENAME NOTHING, leave every path exactly as found, journal `step: 'failed-both-copies-damaged'` with the per-check reasons. | * - * If ANY of the three fails: remove a bad/half symlink at sourcePath (if - * one exists), restore the staged original to sourcePath (if the staged - * copy still exists and sourcePath is clear), mark the journal record - * `step: 'failed'` with the reason, and report — this function NEVER - * deletes on the strength of "a link resolves". + * Row 1 and row 3 never delete/restore a staged copy that's merely + * "absent" — there's nothing there to act on. Row 2 exists specifically + * so a live, correct, already-in-use path is never sacrificed just + * because its backup has a problem. Row 4 exists specifically so that, + * when there is genuinely nothing good anywhere, this function refuses to + * guess — it touches nothing rather than pick a side. */ function finalizeSwapOrRestore(journalDir, record) { - const { sourcePath, stagedPath, targetPath, manifest } = record; + const { sourcePath, stagedPath, linkPath, targetPath, manifest } = record; const targetOk = manifestExactMatch(targetPath, manifest); @@ -1030,68 +1041,122 @@ function finalizeSwapOrRestore(journalDir, record) { } catch { sourceLstat = null; } - let sourceIsCorrectSymlink = false; - if (sourceLstat && sourceLstat.isSymbolicLink()) { + const srcExists = !!sourceLstat; + const srcIsSymlink = !!(sourceLstat && sourceLstat.isSymbolicLink()); + const linkExists = existsSync(linkPath); + + // Is there a verifiably-correct symlink representing this unit's live + // path — already placed at sourcePath, or still pending (unconsumed) at + // linkPath? Either counts as "the link is fine"; WHICH one it is + // determines whether completing the pending rename is even on the + // table below (it only ever happens as part of executing row 1 or row + // 2 — never speculatively, and never for row 3 or row 4). + let linkOk = false; + let linkLocation = null; // 'source' | 'pending' | null + if (srcIsSymlink) { + try { + if (realpathSync(sourcePath) === realpathSync(targetPath)) { + linkOk = true; + linkLocation = 'source'; + } + } catch { + linkOk = false; + } + } else if (!srcExists && linkExists) { try { - sourceIsCorrectSymlink = realpathSync(sourcePath) === realpathSync(targetPath); + if (lstatSync(linkPath).isSymbolicLink() && realpathSync(linkPath) === realpathSync(targetPath)) { + linkOk = true; + linkLocation = 'pending'; + } } catch { - sourceIsCorrectSymlink = false; + linkOk = false; } } - const stagedExists = existsSync(stagedPath); - const stagedOk = !stagedExists || manifestExactMatch(stagedPath, manifest); + const stagedState = !existsSync(stagedPath) + ? 'absent' + : (manifestExactMatch(stagedPath, manifest) ? 'matching' : 'damaged'); - if (targetOk && sourceIsCorrectSymlink && stagedOk) { - if (stagedExists) rmSync(stagedPath, { recursive: true }); + // Row 1: live path correct, and there's either nothing staged to worry + // about or it matches too — complete normally. Only HERE does a + // pending rename actually get completed. + if (targetOk && linkOk && stagedState !== 'damaged') { + if (linkLocation === 'pending') renameSync(linkPath, sourcePath); + if (stagedState === 'matching') rmSync(stagedPath, { recursive: true }); removeJournalRecord(journalDir, sourcePath); - return { ok: true }; + return { ok: true, row: 1 }; + } + + // Row 2: live path correct and complete, but the backup is damaged. + // Preserve the live path exactly as-is (completing the pending rename + // if needed, since the target+link half of this unit is fully + // verified); never delete the damaged backup silently — quarantine it. + if (targetOk && linkOk && stagedState === 'damaged') { + if (linkLocation === 'pending') renameSync(linkPath, sourcePath); + const quarantinePath = `${sourcePath}${QUARANTINE_SUFFIX_PREFIX}${journalKey(sourcePath)}`; + const reason = 'the staged original was damaged (partial or corrupted content) but the live target and symlink are complete and correct; the live path was left untouched and the damaged staged copy was quarantined rather than deleted or silently trusted'; + renameSync(stagedPath, quarantinePath); + writeJournalRecordSync(journalDir, { ...record, step: 'completed-partial-staging-quarantined', quarantinePath }); + return { ok: true, row: 2, quarantined: true, quarantinePath, reason }; } const reasons = []; if (!targetOk) reasons.push('target content does not match the journaled manifest exactly (missing, extra, or corrupted files)'); - if (!sourceIsCorrectSymlink) reasons.push('source is not a symlink whose realpath resolves exactly to the recorded target path'); - if (!stagedOk) reasons.push('the staged original itself no longer matches the journaled manifest exactly'); - const reason = reasons.join('; '); + if (!linkOk) reasons.push('source is not a symlink whose realpath resolves exactly to the recorded target path'); - // Never delete on this evidence. Only touch sourcePath at all if there - // is a staged original to fall back on — removing a bad symlink with - // NOTHING to put in its place would make things strictly worse (e.g. - // the case where the original was already legitimately deleted in a - // prior successful run and only the target has since degraded: there is - // nothing left to restore FROM, so the existing symlink — however - // suspect — is left exactly alone rather than removed for no benefit). - let restored = false; - if (stagedExists) { + // Row 3: live path has a problem, but the staged original is intact — + // restore from it. Clears away a bad symlink wherever it currently + // sits (already at sourcePath, or still pending at linkPath). + if (stagedState === 'matching') { + const reason = reasons.join('; '); try { - if (sourceLstat && sourceLstat.isSymbolicLink()) rmSync(sourcePath); + if (srcIsSymlink) rmSync(sourcePath); + else if (!srcExists && linkExists) rmSync(linkPath, { recursive: true }); } catch { // best effort — the restore attempt below will surface a failure } + let restored = false; if (!existsSync(sourcePath)) { try { renameSync(stagedPath, sourcePath); - restored = existsSync(sourcePath) && !lstatSync(sourcePath).isSymbolicLink(); + restored = manifestExactMatch(sourcePath, manifest); } catch { restored = false; } } - // else: sourcePath still holds something unexpected (not a symlink, - // and removal above didn't apply) — leave both paths for manual - // inspection rather than guess which is authoritative. - } else if (sourceLstat && !sourceLstat.isSymbolicLink()) { - // No staged copy, and sourcePath already holds a real directory — - // nothing to restore, but also nothing was ever at risk. - restored = true; + // else: sourcePath still holds something unexpected (removal above + // didn't apply) — leave both paths for manual inspection rather than + // guess which is authoritative. + try { + writeJournalRecordSync(journalDir, { ...record, step: 'failed', failureReason: reason }); + } catch { + // best effort — the reason is still returned to the caller either way + } + return { ok: false, row: 3, reason, restored }; } + // Row 4: nothing verified good anywhere (live path has a problem AND + // the backup is absent or also damaged). DELETE NOTHING, RENAME + // NOTHING — sourcePath, stagedPath, and linkPath are all left exactly + // as found, including a still-pending, uncompleted rename. Report + // loudly with every path so a human can recover by hand. + reasons.push(stagedState === 'absent' + ? 'no staged original exists to fall back on' + : 'the staged original itself no longer matches the journaled manifest exactly'); + const reason = reasons.join('; '); try { - writeJournalRecordSync(journalDir, { ...record, step: 'failed', failureReason: reason }); + writeJournalRecordSync(journalDir, { ...record, step: 'failed-both-copies-damaged', failureReason: reason }); } catch { - // best effort — the reason is still returned to the caller either way + // best effort } - - return { ok: false, reason, restored }; + return { + ok: false, + row: 4, + reason, + restored: false, + untouched: true, + paths: { sourcePath, stagedPath, linkPath, targetPath } + }; } /** @@ -1164,8 +1229,12 @@ function swapToSymlink(src, dst, opts = {}) { // "The link resolves to something" is deliberately not one of them. const result = finalizeSwapOrRestore(journalDir, base); if (!result.ok) { - throw new Error(`post-swap finalize refused to delete the staged original and restored it instead: ${result.reason}`); + const outcome = result.row === 4 + ? 'left every path exactly as found (nothing verified good anywhere)' + : `restored the staged original to ${src}`; + throw new Error(`post-swap finalize refused to delete the staged original and ${outcome}: ${result.reason}`); } + return result; // row 1: plain success. row 2: success, but carries {quarantined:true, quarantinePath, reason}. } /** Every location discoverCandidates() looks at, reused so detection and @@ -1195,8 +1264,16 @@ function strayJournalSuffixedPaths(home, excludePaths) { for (const dir of candidateParentDirs(home)) { for (const name of safeReaddir(dir)) { if (!isJournalSuffixed(name)) continue; - const base = name.endsWith(STAGING_SUFFIX) ? name.slice(0, -STAGING_SUFFIX.length) : name.slice(0, -LINK_SUFFIX.length); - const src = join(dir, base); + let src; + if (name.includes(QUARANTINE_SUFFIX_PREFIX)) { + // A quarantine dir represents itself (not some other "base" name + // with a suffix stripped) — it's excluded by its own full path, + // which is what a journal record's `quarantinePath` field holds. + src = join(dir, name); + } else { + const base = name.endsWith(STAGING_SUFFIX) ? name.slice(0, -STAGING_SUFFIX.length) : name.slice(0, -LINK_SUFFIX.length); + src = join(dir, base); + } if (excludePaths.has(src)) continue; strays.push({ path: src, entryPath: join(dir, name) }); } @@ -1225,6 +1302,7 @@ export function detectInterruptedMoves(home, options = {}) { continue; } journaledPaths.add(record.sourcePath); + if (record.quarantinePath) journaledPaths.add(record.quarantinePath); const srcExists = existsSync(record.sourcePath); let srcIsSymlink = false; try { @@ -1281,6 +1359,7 @@ export function recoverInterruptedMoves(home, options = {}) { } const { sourcePath, stagedPath, linkPath, manifest, step } = record; journaledPaths.add(sourcePath); + if (record.quarantinePath) journaledPaths.add(record.quarantinePath); try { const srcExists = existsSync(sourcePath); @@ -1294,23 +1373,42 @@ export function recoverInterruptedMoves(home, options = {}) { const linkExists = existsSync(linkPath); // A symlink is either already in place, or one verified rename away - // from being in place: the ONLY safe way to decide whether the - // staged original may be deleted is finalizeSwapOrRestore's - // three-way check (target manifest, symlink realpath, staged - // manifest) — never a bare "does something exist at the realpath" - // check, which is exactly the data-loss bug this replaces. + // from being in place: the ONLY safe way to decide anything here — + // including whether it's even safe to COMPLETE that pending rename + // — is finalizeSwapOrRestore's own decision table (target manifest, + // symlink realpath, staged manifest, all re-checked now). It is + // NOT safe to complete the rename speculatively before that + // decision: row 4 (nothing verified good anywhere) must rename + // NOTHING, so finalizeSwapOrRestore performs the pending rename + // itself, only inside the row-1/row-2 branches that decide it's + // warranted. if ((srcExists && srcIsSymlink) || (!srcExists && stagedExists && linkExists)) { - if (!srcExists && stagedExists && linkExists) { - // Completing this rename alone never deletes anything — the - // original is still fully intact at stagedPath either way. - // finalizeSwapOrRestore below undoes this if it turns out - // unsafe to proceed past it. - renameSync(linkPath, sourcePath); - } const result = finalizeSwapOrRestore(journalDir, record); - if (result.ok) { + if (result.ok && result.quarantined) { + // Row 2: live path is correct and complete — preserved, + // untouched. The damaged backup was quarantined, not deleted. + // The journal record was just rewritten with this quarantine + // path, so exclude it from the stray scan below too. + journaledPaths.add(result.quarantinePath); + recovered.push({ + path: sourcePath, + journalFile: file, + action: 'completed-partial-staging-quarantined', + note: `${result.reason} (quarantined at ${result.quarantinePath})` + }); + } else if (result.ok) { + // Row 1. recovered.push({ path: sourcePath, journalFile: file, action: 'completed-swap-and-cleaned', note: 'completed the pending symlink swap (or finished cleanup of one already in place) and removed the staged original — target and symlink both re-verified against the journal manifest first' }); + } else if (result.row === 4) { + // Row 4: nothing verified good anywhere — touched nothing. + recovered.push({ + path: sourcePath, + journalFile: file, + action: 'left-alone-nothing-verified-good', + note: `${result.reason} — every path left exactly as found (source: ${result.paths.sourcePath}, staged: ${result.paths.stagedPath}, target: ${result.paths.targetPath}); recover by hand` + }); } else { + // Row 3. recovered.push({ path: sourcePath, journalFile: file, @@ -1441,7 +1539,7 @@ export function applyRun(options) { const rel = relative(home, src); const dst = join(target, rel); try { - swapToSymlink(src, dst, { + const finalizeResult = swapToSymlink(src, dst, { symlinkSync: symlinkFn, journalDir, beforeLink: options.beforeLink, @@ -1449,7 +1547,13 @@ export function applyRun(options) { afterStage: options.afterStage, afterSwap: options.afterSwap }); - swapped.push({ source: src, target: dst }); + const entry = { source: src, target: dst }; + if (finalizeResult && finalizeResult.quarantined) { + entry.quarantined = true; + entry.quarantinePath = finalizeResult.quarantinePath; + entry.note = finalizeResult.reason; + } + swapped.push(entry); } catch (e) { swapFailed = { path: src, error: e.message }; break; diff --git a/test/model-tidy.test.mjs b/test/model-tidy.test.mjs index 2422940..5ecef12 100644 --- a/test/model-tidy.test.mjs +++ b/test/model-tidy.test.mjs @@ -9,6 +9,7 @@ import { import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; import { planRun, applyRun, discoverCandidates, loadKeepList, computeHardlinkGroups, validateTarget, copyUnitPureNode, recoverInterruptedMoves @@ -768,13 +769,16 @@ describe('GAP 2: crash-safety across a hard process kill (not just a thrown exce } else if (lst.isDirectory()) { recurse(full); } else if (lst.isFile()) { - snap[full] = { type: 'file', size: lst.size, mtimeMs: lst.mtimeMs }; + snap[full] = { type: 'file', size: lst.size, mtimeMs: lst.mtimeMs, sha256: sha256Sync(full) }; } } } recurse(root); return snap; } + function sha256Sync(path) { + return createHash('sha256').update(readFileSync(path)).digest('hex'); + } it("AMENDMENT: plan is read-only — an interrupted move is reported per unit, and every byte on disk (full tree, sizes and mtimes) is untouched by plan", () => { const { home, src } = buildSingleIdleFixture(); @@ -825,7 +829,15 @@ describe('GAP 2: crash-safety across a hard process kill (not just a thrown exce assert.match(finding.note, /no matching journal record/); }); - it('AMENDMENT: recovery never silently discards a staged original whose own content no longer matches the journal — it restores (never deletes) and reports failure', () => { + it('AMENDMENT / decision-table row 2: recovery never silently discards a staged original whose own content no longer matches the journal — with the live path (target + link) still correct, it quarantines rather than deletes or restores over the good symlink', () => { + // NOTE: this scenario crashes 'afterStage' (between the two renames), + // but since the pending link was already verified BEFORE it was ever + // created, recovery completes that rename first — so by the time + // finalizeSwapOrRestore runs, target and link are both fine. Only the + // staged backup is damaged. That is row 2 of the decision table, not + // row 3: an earlier version of this fix used to restore the tampered + // staged copy over the good live symlink here, which was itself a + // (milder) data-loss bug — see the round-3 fix. const { home, src } = buildSingleIdleFixture(); const target = tempDir('model-tidy-gap2-target-'); const plan = planFor(home); @@ -834,6 +846,7 @@ describe('GAP 2: crash-safety across a hard process kill (not just a thrown exce assert.equal(child.status, 77); const staging = `${src}.tidy-moving`; assert.ok(existsSync(staging)); + const goodContent = 'gap2-fixture-bytes'.repeat(50); // Tamper with the staged original so it no longer matches what the // journal recorded — recovery must not trust its own naming @@ -844,20 +857,22 @@ describe('GAP 2: crash-safety across a hard process kill (not just a thrown exce const recovered = recoverInterruptedMoves(home); const finding = recovered.find(r => r.path === src); assert.ok(finding); - assert.match(finding.action, /^(restored-after-failed-verification|left-alone-after-failed-verification)$/); - assert.match(finding.note, /staged original itself no longer matches the journaled manifest/); - - // Whatever happened, the staged content was neither silently deleted - // nor silently accepted as-is: it now lives at src (or, if that - // somehow could not happen, at the untouched staging path) and the - // journal itself is marked failed rather than cleared as successful. - const survivedAtSrc = existsSync(src) && !lstatSync(src).isSymbolicLink(); - const survivedAtStaging = existsSync(staging); - assert.ok(survivedAtSrc || survivedAtStaging, 'the tampered content must not have been silently discarded'); - if (survivedAtSrc) { - assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'TAMPERED-CONTENT-DOES-NOT-MATCH-MANIFEST'); - } - assert.equal(lstatSync(src).isSymbolicLink(), false, 'the source must not be left as a symlink to an unverified copy after a failed check'); + assert.equal(finding.action, 'completed-partial-staging-quarantined'); + assert.match(finding.note, /staged original was damaged/); + + // The live path must be preserved exactly as it was: a working + // symlink to the good target. + assert.equal(lstatSync(src).isSymbolicLink(), true, 'the live symlink must be preserved, not replaced with the tampered staged copy'); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), goodContent); + + // The damaged staging must not have been silently deleted: it must + // now live at a quarantine path, still holding the tampered content + // for a human to inspect — never lost. + assert.equal(existsSync(staging), false, 'the staging path itself is gone (renamed to quarantine)'); + const quarantineDirs = readdirSync(join(home, 'models')).filter(n => n.includes('.tidy-quarantine-')); + assert.equal(quarantineDirs.length, 1); + const quarantinePath = join(home, 'models', quarantineDirs[0]); + assert.equal(readFileSync(join(quarantinePath, 'weights.gguf'), 'utf8'), 'TAMPERED-CONTENT-DOES-NOT-MATCH-MANIFEST'); }); it("DATA-LOSS FIX, codexmb's round-2 probe kept verbatim: target corrupted after staging must not cost the last good original", () => { @@ -1003,4 +1018,75 @@ describe('GAP 2: crash-safety across a hard process kill (not just a thrown exce assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'gap2-fixture-bytes'.repeat(50)); } }); + + it("decision-table row 3, reproducing codexmb's round-3 probe's own 'bad-target' scenario (afterSwap crash, not afterStage): a corrupted target still triggers restore-from-staged, never quarantine", () => { + // Exact mechanics of round-3's probe: crash at afterSwap (src is + // ALREADY a symlink, staged still pending cleanup), then corrupt the + // target's content. linkOk only checks that the symlink's realpath + // equals the recorded target PATH — it does not, and must not, imply + // the target's CONTENT is still correct. targetOk must independently + // fail here, landing on row 3 (restore), not row 1 or row 2. + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterSwap'); + assert.equal(child.status, 77); + assert.equal(lstatSync(src).isSymbolicLink(), true, 'mid-crash: the swap already completed structurally'); + const staging = `${src}.tidy-moving`; + assert.ok(existsSync(staging)); + + writeFileSync(join(target, 'models', 'idle', 'weights.gguf'), 'BADD-TARGET-CONTENT'); + + const recovered = recoverInterruptedMoves(home); + const finding = recovered.find(r => r.path === src); + assert.ok(finding); + assert.equal(finding.action, 'restored-after-failed-verification'); + assert.match(finding.note, /target content does not match the journaled manifest/); + + assert.equal(lstatSync(src).isSymbolicLink(), false, 'row 3 restores the real directory, it does not quarantine or preserve a symlink to a corrupted target'); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'gap2-fixture-bytes'.repeat(50)); + assert.equal(existsSync(staging), false); + }); + + it('decision-table row 4: nothing verified good anywhere (target corrupted AND staging partial) — recovery deletes nothing, renames nothing, and leaves every path exactly as found', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterStage'); + assert.equal(child.status, 77); + const staging = `${src}.tidy-moving`; + assert.ok(existsSync(staging)); + + // Damage BOTH copies: the target (live) and the staged backup. + writeFileSync(join(target, 'models', 'idle', 'weights.gguf'), 'BADD-TARGET'); + rmSync(join(staging, 'weights.gguf')); + + // Snapshot only the DATA paths (source tree + target tree), not + // model-tidy's own journal bookkeeping under home/.cache — the + // journal is EXPECTED to record this failure (step + // 'failed-both-copies-damaged'); what must stay byte-for-byte + // unchanged is the actual model data. + const snapshotData = () => ({ ...snapshotTree(join(home, 'models')), ...snapshotTree(target) }); + + const before = snapshotData(); + const recovered = recoverInterruptedMoves(home); + const after = snapshotData(); + + assert.deepEqual(after, before, 'row 4 must delete nothing and rename nothing — every data path (sizes and content hashes) must be byte-for-byte unchanged'); + + const finding = recovered.find(r => r.path === src); + assert.ok(finding); + assert.equal(finding.action, 'left-alone-nothing-verified-good'); + assert.match(finding.note, /target content does not match the journaled manifest/); + assert.match(finding.note, /no staged original exists to fall back on|staged original itself no longer matches/); + assert.match(finding.note, new RegExp(src.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')), 'the report must name the affected paths so a human can recover by hand'); + + // A second recovery pass must be equally inert — refusing is stable, + // not just a one-time fluke. + const secondPass = recoverInterruptedMoves(home); + assert.deepEqual(snapshotData(), before); + assert.equal(secondPass.find(r => r.path === src).action, 'left-alone-nothing-verified-good'); + }); }); From 18e9144e6c7922bd05deb2b99fa666b5a9f41533 Mon Sep 17 00:00:00 2001 From: Petrus Pennanen Date: Mon, 21 Sep 2026 10:34:13 +0200 Subject: [PATCH 6/9] Fix two round-4 blockers: never recursive-delete an assumed symlink; durable journal writes/reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit codexmb's round-4 review flagged two structural issues in the recovery machinery added over the previous three rounds, both confirmed in src at 05588ee before fixing. BLOCKER 1: three call sites removed a path they believed was a symlink using rmSync(path, {recursive:true}) without verifying that belief first — finalizeSwapOrRestore row 3 (two sites: the sourcePath obstacle and the pending-link obstacle) and recoverInterruptedMoves's pre-swap stray-link branch. If a real directory ever sat at that path instead (or the symlink pointed somewhere unexpected), a recursive remove could destroy an entire directory tree that was never model-tidy's to touch. Replaced all three with one helper, unlinkOwnedSymlink(path, expectedTargetPath): lstatSync first — not a symlink at all, or missing, refuses and touches nothing; is a symlink, but its realpath doesn't resolve to exactly the journal's recorded target, refuses and touches nothing; only then unlinkSync (never rmSync, never recursive). Every caller now treats a refusal as "left alone, reported, journal step 'failed'", never a reason to fall back to something more aggressive. BLOCKER 2: writeJournalRecordSync did open(file,'w') + write + fsync — a crash between the truncating open and the write leaves an empty or partial record at the path readers expect, which would make a real, in-flight unit look like there's nothing to recover. Changed to write-temp-fsync-rename: write the full record to .tmp-- in the same directory, fsync and close that temp file, renameSync it atomically over the real path, then fsync the journal directory itself (best-effort — some platforms can't fsync a directory fd; that failure is swallowed since the rename is still atomic there). Readers (listJournalRecords, used by both detectInterruptedMoves and recoverInterruptedMoves) now treat a record that is missing, empty, truncated, unparseable as JSON, or doesn't match the expected schema as 'journal-unreadable' / 'left alone' — that unit is reported and never acted on by plan, apply, or recover. A stray .json.tmp-* file left over from an interrupted journal WRITE is recognized by name and reported the same way; it is never parsed as a record. New tests (8, all pass standalone and in the full suite): for each of the three unlinkOwnedSymlink call sites, a real directory with a sentinel file is substituted for the expected symlink and the sentinel is asserted to survive, the unit reported refused, and a full before/ after tree snapshot (sizes + SHA-256) proves nothing else changed; plus a dedicated test that a symlink pointing to the wrong target is refused, not unlinked. For journal durability: a record truncated to half its bytes and to zero bytes is reported unreadable by both plan and recover with every path unchanged (tree snapshot); a stray .tmp-* file is reported and never parsed as a record; a positive control confirms a complete, valid record still drives recovery normally. Test results: 8/8 new tests in isolation, 43/43 full model-tidy suite, 702/702 full repo suite. Co-Authored-By: Claude Fable 5.1 --- docs/model-tidy.md | 63 ++++++++++ src/model-tidy.mjs | 213 +++++++++++++++++++++++++++---- test/model-tidy.test.mjs | 263 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 512 insertions(+), 27 deletions(-) diff --git a/docs/model-tidy.md b/docs/model-tidy.md index 2217416..9d02b9b 100644 --- a/docs/model-tidy.md +++ b/docs/model-tidy.md @@ -217,6 +217,58 @@ for one unit does not roll back units that already succeeded earlier in the same `apply` run, but the process still exits non-zero and prints exactly which unit failed, at which step, and why. +### Never recursive-delete a path assumed to be a symlink (`unlinkOwnedSymlink`) + +A fourth review round found that several places removed a path they +*believed* was a symlink using `rmSync(path, {recursive: true})`. If that +belief were ever wrong — a real directory happens to sit at that path, or +the symlink itself points somewhere other than expected — a recursive +remove can destroy an entire directory tree that was never model-tidy's to +touch. + +`unlinkOwnedSymlink(path, expectedTargetPath)` is now the only way +anything in this file removes a path it believes is one of its own +symlinks. It never recurses: + +1. `lstatSync(path)` — if the path doesn't exist, or isn't a symlink at + all, **refuse**. Touch nothing. +2. its realpath must resolve to exactly `expectedTargetPath`'s own + realpath — a symlink pointing somewhere else is **refused**, not + unlinked. +3. only then: `unlinkSync(path)` — never `rmSync`, never recursive. A + plain unlink can only ever remove the one symlink entry itself. + +Every caller treats a refusal as "left alone, reported, journal marked +`failed`" — never a reason to fall back to something more aggressive. + +### Journal durability + +A naive `open(file, 'w')` + write + `fsync` has its own crash window: a +kill landing between the truncating open and the write leaves an EMPTY or +PARTIAL record at the path readers expect — making a real, in-flight unit +look like there's nothing to recover, or worse, giving a reader a document +that parses as JSON but describes something incoherent. + +Every journal write now follows write-temp-fsync-rename: the full record +is written to a throwaway `.tmp--` in the same +directory, that temp file is `fsync`ed and closed, then `renameSync`d +atomically over the real path (same-directory renames are atomic — a +reader sees either the old complete record or the new complete one, never +a partial one), then the journal directory itself is `fsync`ed so the +rename survives a crash immediately after (on a platform where a +directory can't be opened for `fsync`, that failure is swallowed — the +rename is still atomic there, just not immediately durable). + +On the read side, a journal record that is **missing, empty, truncated, +fails to parse, or doesn't match the expected schema** is never treated as +"nothing to see here" — it's reported as `journal-unreadable` +(`plan`/`detectInterruptedMoves`) or `left-alone` +(`recover`/`recoverInterruptedMoves`), and the unit it might describe is +never acted on, never deleted, by any of `plan`, `apply`, or `recover`. A +stray `.json.tmp-*` file — the leftover of a write that itself got +interrupted — is recognized by name and reported the same way; it is never +parsed as a record. + ## Selection rules (in order, each with an explicit reason string) 0. **Interrupted move found** — highest priority, checked before anything @@ -338,6 +390,17 @@ it never attempts to install anything itself. ## Safety invariants +- **Nothing removes a path it believes is a symlink without verifying it + first, and never recursively.** `unlinkOwnedSymlink` is the only code + path allowed to do this: it refuses (touches nothing) if the path isn't + actually a symlink, or resolves to somewhere other than the journal's + recorded target — a plain `unlinkSync`, never `rmSync`, only after both + checks pass. +- **Every journal write is crash-safe: write-temp-fsync-rename, then fsync + the directory.** A journal record that's missing, empty, truncated, + unparseable, or the wrong shape is treated as `journal-unreadable` by + every reader (`plan`, `apply`, `recover`) — that unit is reported and + left alone, never acted on. - `plan` makes zero filesystem writes, ever — including for interrupted moves, which it only detects and reports. Only `recover` and `apply` (once, at its own start) mutate anything. diff --git a/src/model-tidy.mjs b/src/model-tidy.mjs index 63c5869..7e13a67 100644 --- a/src/model-tidy.mjs +++ b/src/model-tidy.mjs @@ -51,10 +51,10 @@ import { readdirSync, lstatSync, existsSync, readFileSync, readlinkSync, realpathSync, symlinkSync, rmSync, mkdirSync, copyFileSync, linkSync, statSync, appendFileSync, constants as FS_CONSTANTS, accessSync, renameSync, - openSync, writeSync, fsyncSync, closeSync + openSync, writeSync, fsyncSync, closeSync, unlinkSync } from 'node:fs'; import { join, relative, sep, isAbsolute } from 'node:path'; -import { createHash } from 'node:crypto'; +import { createHash, randomBytes } from 'node:crypto'; import { homedir } from 'node:os'; export const SERVING_PROCESS_NAMES = [ @@ -877,23 +877,69 @@ function journalFilePath(journalDir, sourcePath) { return join(journalDir, `${journalKey(sourcePath)}.json`); } +/** Minimal structural check for a parsed journal record. Anything that + * doesn't match — including a record that parsed as valid JSON but isn't + * actually one of ours (wrong shape) — is treated as unreadable, exactly + * like a truncated or corrupt file. */ +function isValidJournalRecordShape(record) { + return !!record + && typeof record === 'object' + && typeof record.sourcePath === 'string' && record.sourcePath.length > 0 + && typeof record.stagedPath === 'string' && record.stagedPath.length > 0 + && typeof record.linkPath === 'string' && record.linkPath.length > 0 + && typeof record.targetPath === 'string' && record.targetPath.length > 0 + && Array.isArray(record.manifest) + && typeof record.step === 'string' && record.step.length > 0; +} + /** - * Write (or overwrite) the journal record for one unit, fsynced so it - * survives a crash immediately after this call returns. Called BEFORE the - * first filesystem mutation for a unit, and again after every subsequent - * step, so the journal always reflects the furthest step actually reached. + * Write (or overwrite) the journal record for one unit, durably. Called + * BEFORE the first filesystem mutation for a unit, and again after every + * subsequent step, so the journal always reflects the furthest point + * actually reached — including across a hard crash. + * + * A naive `open(file, 'w')` + write + fsync has its own crash window: a + * kill between the truncating open and the write leaves an EMPTY or + * PARTIAL record at the well-known path readers expect — which would + * make a real, in-flight unit look like there's simply nothing to + * recover. Instead: write the full content to a throwaway temp file in + * the same directory, fsync THAT file, close it, atomically rename it + * over the real path (a same-directory rename is atomic — readers either + * see the old complete record or the new complete record, never a + * partial one), then fsync the directory itself so the rename survives a + * crash immediately after. On platforms where a directory can't be + * opened for fsync, that failure is swallowed (the rename itself is still + * safe there) — noted here in case it needs to change for a supported + * platform where journal loss would matter. */ function writeJournalRecordSync(journalDir, record) { mkdirSync(journalDir, { recursive: true }); const file = journalFilePath(journalDir, record.sourcePath); + const tmpFile = `${file}.tmp-${process.pid}-${randomBytes(6).toString('hex')}`; const data = JSON.stringify({ ...record, updatedAt: new Date().toISOString() }, null, 2); - const fd = openSync(file, 'w'); + + const fd = openSync(tmpFile, 'w'); try { writeSync(fd, data); fsyncSync(fd); } finally { closeSync(fd); } + renameSync(tmpFile, file); + + try { + const dirFd = openSync(journalDir, 'r'); + try { + fsyncSync(dirFd); + } finally { + closeSync(dirFd); + } + } catch { + // Some platforms can't fsync a directory fd — the rename above is + // still atomic there, just not guaranteed durable against a crash in + // the same instant. Nothing further to do about it here. + } + return file; } @@ -906,20 +952,50 @@ function removeJournalRecord(journalDir, sourcePath) { } } -/** All journal records currently on disk. A record that fails to parse is - * still returned (with `record: null, corrupt: true`) so callers can - * report it rather than silently skip it. */ +/** + * All journal records currently on disk. A record that is missing, + * empty, truncated, fails to parse, or doesn't match the expected schema + * is still returned — as `{record: null, corrupt: true, reason}` — so + * callers report it rather than silently skip it or, worse, act on + * whatever partial data it happens to contain. A leftover `.tmp-*` file + * from an interrupted journal WRITE (crashed between creating the temp + * file and the rename) is recognized by name and reported the same way; + * it is never parsed as a record. + */ function listJournalRecords(journalDir) { if (!existsSync(journalDir)) return []; const out = []; for (const name of safeReaddir(journalDir)) { - if (!name.endsWith('.json')) continue; const file = join(journalDir, name); + if (name.includes('.tmp-')) { + out.push({ file, record: null, corrupt: true, reason: 'stray incomplete journal write (a .tmp file left over from an interrupted write); never parsed as a record' }); + continue; + } + if (!name.endsWith('.json')) continue; + + let raw; try { - out.push({ file, record: JSON.parse(readFileSync(file, 'utf8')) }); - } catch { - out.push({ file, record: null, corrupt: true }); + raw = readFileSync(file, 'utf8'); + } catch (e) { + out.push({ file, record: null, corrupt: true, reason: `journal file unreadable (${e.code || e.message})` }); + continue; + } + if (raw.length === 0) { + out.push({ file, record: null, corrupt: true, reason: 'journal file is empty (zero bytes)' }); + continue; + } + let parsed; + try { + parsed = JSON.parse(raw); + } catch (e) { + out.push({ file, record: null, corrupt: true, reason: `journal file is not valid JSON, likely truncated (${e.message})` }); + continue; + } + if (!isValidJournalRecordShape(parsed)) { + out.push({ file, record: null, corrupt: true, reason: 'journal file does not match the expected record schema' }); + continue; } + out.push({ file, record: parsed }); } return out; } @@ -1030,6 +1106,64 @@ const QUARANTINE_SUFFIX_PREFIX = '.tidy-quarantine-'; * when there is genuinely nothing good anywhere, this function refuses to * guess — it touches nothing rather than pick a side. */ + +/** + * The ONLY way anything in this file removes a path it believes is a + * symlink it created. `rmSync(..., {recursive:true})` on an assumed + * symlink is dangerous: if that assumption is wrong — the path is + * actually a real directory someone else put there, or a symlink + * pointing somewhere unexpected — a recursive remove can delete an + * entire directory tree that was never ours to touch. This helper + * verifies before it ever unlinks anything, and never recurses: + * + * 1. lstatSync(path) — if it doesn't exist, or isn't a symlink at all + * (a real file or directory sitting where we expected a symlink), + * REFUSE. Touch nothing. + * 2. its realpath must resolve to exactly `expectedTargetPath`'s own + * realpath — a symlink pointing somewhere else (tampered, or a + * coincidentally-named path from something unrelated) is REFUSED, + * not unlinked. + * 3. only then: `unlinkSync(path)` — never `rmSync`, never recursive. + * A single unlink can only ever remove the one symlink entry itself, + * never anything reachable through it. + * + * Callers treat a refusal as "left alone, reported" — never a reason to + * fall back to a more aggressive removal. + */ +function unlinkOwnedSymlink(path, expectedTargetPath) { + let lst; + try { + lst = lstatSync(path); + } catch (e) { + return { ok: false, reason: `expected an owned symlink at ${path}, but it does not exist (${e.code || e.message})` }; + } + if (!lst.isSymbolicLink()) { + const kind = lst.isDirectory() ? 'a directory' : lst.isFile() ? 'a regular file' : 'a non-symlink entry'; + return { ok: false, reason: `expected an owned symlink at ${path}, found ${kind} instead — refusing to touch it` }; + } + let real; + try { + real = realpathSync(path); + } catch (e) { + return { ok: false, reason: `symlink at ${path} could not be resolved (${e.code || e.message}) — refusing to touch it` }; + } + let expectedReal; + try { + expectedReal = realpathSync(expectedTargetPath); + } catch (e) { + return { ok: false, reason: `recorded target ${expectedTargetPath} could not be resolved (${e.code || e.message}) — refusing to touch ${path}` }; + } + if (real !== expectedReal) { + return { ok: false, reason: `symlink at ${path} resolves to ${real}, not the recorded target ${expectedTargetPath} — refusing to touch it` }; + } + try { + unlinkSync(path); + } catch (e) { + return { ok: false, reason: `unlink of verified symlink ${path} failed (${e.code || e.message})` }; + } + return { ok: true }; +} + function finalizeSwapOrRestore(journalDir, record) { const { sourcePath, stagedPath, linkPath, targetPath, manifest } = record; @@ -1106,15 +1240,33 @@ function finalizeSwapOrRestore(journalDir, record) { // Row 3: live path has a problem, but the staged original is intact — // restore from it. Clears away a bad symlink wherever it currently - // sits (already at sourcePath, or still pending at linkPath). + // sits (already at sourcePath, or still pending at linkPath) — but ONLY + // via unlinkOwnedSymlink's verify-then-unlink, never a recursive + // remove on an assumed symlink. If that verification refuses (the path + // isn't actually a symlink, or resolves somewhere unexpected), row 3 + // itself refuses too: touch nothing further, report, journal 'failed'. if (stagedState === 'matching') { const reason = reasons.join('; '); - try { - if (srcIsSymlink) rmSync(sourcePath); - else if (!srcExists && linkExists) rmSync(linkPath, { recursive: true }); - } catch { - // best effort — the restore attempt below will surface a failure + + let obstaclePath = null; + if (srcIsSymlink) obstaclePath = sourcePath; + else if (!srcExists && linkExists) obstaclePath = linkPath; + // else: sourcePath holds something that's neither a verifiable + // symlink nor absent-with-a-pending-link — nothing to clear first. + + if (obstaclePath) { + const unlinkResult = unlinkOwnedSymlink(obstaclePath, targetPath); + if (!unlinkResult.ok) { + const combinedReason = `${reason}; additionally, could not safely clear the way for restore: ${unlinkResult.reason}`; + try { + writeJournalRecordSync(journalDir, { ...record, step: 'failed', failureReason: combinedReason }); + } catch { + // best effort + } + return { ok: false, row: 3, reason: combinedReason, restored: false, refusedUnlink: true }; + } } + let restored = false; if (!existsSync(sourcePath)) { try { @@ -1296,9 +1448,9 @@ export function detectInterruptedMoves(home, options = {}) { const findings = []; const journaledPaths = new Set(); - for (const { file, record, corrupt } of listJournalRecords(journalDir)) { + for (const { file, record, corrupt, reason } of listJournalRecords(journalDir)) { if (corrupt || !record) { - findings.push({ path: null, journalFile: file, status: 'corrupt-journal', note: `journal file ${file} is corrupt or unreadable` }); + findings.push({ path: null, journalFile: file, status: 'journal-unreadable', note: `journal file ${file} is unreadable: ${reason || 'unknown reason'} — the unit it may describe is left alone until this is resolved` }); continue; } journaledPaths.add(record.sourcePath); @@ -1352,12 +1504,12 @@ export function recoverInterruptedMoves(home, options = {}) { const recovered = []; const journaledPaths = new Set(); - for (const { file, record, corrupt } of listJournalRecords(journalDir)) { + for (const { file, record, corrupt, reason: unreadableReason } of listJournalRecords(journalDir)) { if (corrupt || !record) { - recovered.push({ path: null, journalFile: file, action: 'left-alone', note: `journal file ${file} is corrupt or unreadable; left alone` }); + recovered.push({ path: null, journalFile: file, action: 'left-alone', note: `journal file ${file} is unreadable: ${unreadableReason || 'unknown reason'} — left alone, never acted on` }); continue; } - const { sourcePath, stagedPath, linkPath, manifest, step } = record; + const { sourcePath, stagedPath, linkPath, targetPath, manifest, step } = record; journaledPaths.add(sourcePath); if (record.quarantinePath) journaledPaths.add(record.quarantinePath); @@ -1434,7 +1586,16 @@ export function recoverInterruptedMoves(home, options = {}) { removeJournalRecord(journalDir, sourcePath); recovered.push({ path: sourcePath, journalFile: file, action: 'restored-original', note: 'restored the original directory (no verified pending symlink existed to trust instead)' }); } else if (srcExists && !srcIsSymlink && linkExists && !stagedExists) { - rmSync(linkPath, { recursive: true }); + const unlinkResult = unlinkOwnedSymlink(linkPath, targetPath); + if (!unlinkResult.ok) { + try { + writeJournalRecordSync(journalDir, { ...record, step: 'failed', failureReason: unlinkResult.reason }); + } catch { + // best effort + } + recovered.push({ path: sourcePath, journalFile: file, action: 'left-alone', note: `refused to remove the pending link (${unlinkResult.reason}); the original itself was never touched` }); + continue; + } removeJournalRecord(journalDir, sourcePath); recovered.push({ path: sourcePath, journalFile: file, action: 'removed-stray-link', note: 'the original was never touched; removed the unused pending symlink' }); } else if (srcExists && !srcIsSymlink && !linkExists && !stagedExists) { diff --git a/test/model-tidy.test.mjs b/test/model-tidy.test.mjs index 5ecef12..eaa7d44 100644 --- a/test/model-tidy.test.mjs +++ b/test/model-tidy.test.mjs @@ -12,7 +12,7 @@ import { spawnSync } from 'node:child_process'; import { createHash } from 'node:crypto'; import { planRun, applyRun, discoverCandidates, loadKeepList, computeHardlinkGroups, - validateTarget, copyUnitPureNode, recoverInterruptedMoves + validateTarget, copyUnitPureNode, recoverInterruptedMoves, detectInterruptedMoves } from '../src/model-tidy.mjs'; const MODEL_TIDY_MODULE_URL = new URL('../src/model-tidy.mjs', import.meta.url).href; @@ -1089,4 +1089,265 @@ describe('GAP 2: crash-safety across a hard process kill (not just a thrown exce assert.deepEqual(snapshotData(), before); assert.equal(secondPass.find(r => r.path === src).action, 'left-alone-nothing-verified-good'); }); + + function journalDirFor(home) { + return join(home, '.cache', 'ide-agent-kit', 'model-tidy-journal'); + } + function findJournalFile(home) { + const dir = journalDirFor(home); + const names = readdirSync(dir).filter(n => n.endsWith('.json')); + assert.equal(names.length, 1, `expected exactly one journal file, found ${names.length}: ${names.join(', ')}`); + return join(dir, names[0]); + } + function snapshotData(home, target) { + return { ...snapshotTree(join(home, 'models')), ...snapshotTree(target) }; + } + + describe('BLOCKER 1: never recursive-delete a path assumed to be a symlink (unlinkOwnedSymlink)', () => { + it('site 1 (finalizeSwapOrRestore row 3, sourcePath obstacle): a REAL DIRECTORY with a sentinel file sitting where a symlink was expected is never touched', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + // afterSwap: src is already a symlink, staged still pending cleanup. + const child = runApplyInChildWithCrash(plan, home, target, 'afterSwap'); + assert.equal(child.status, 77); + assert.equal(lstatSync(src).isSymbolicLink(), true); + const staging = `${src}.tidy-moving`; + assert.ok(existsSync(staging)); + + // Force row 3/4 by corrupting the target, THEN replace src (the + // symlink) with a real directory holding a sentinel file — as if + // something else had put a real directory exactly where model-tidy + // expected to find (and remove) its own symlink. + writeFileSync(join(target, 'models', 'idle', 'weights.gguf'), 'BADD'); + rmSync(src); + mkdirSync(src, { recursive: true }); + writeFileSync(join(src, 'SENTINEL.txt'), 'do not delete me'); + + const before = snapshotData(home, target); + const recovered = recoverInterruptedMoves(home); + const after = snapshotData(home, target); + + assert.equal(existsSync(join(src, 'SENTINEL.txt')), true, 'the sentinel file must survive'); + assert.equal(readFileSync(join(src, 'SENTINEL.txt'), 'utf8'), 'do not delete me'); + assert.deepEqual(after, before, 'nothing on disk may change when a real directory sits where a symlink was expected'); + + const finding = recovered.find(r => r.path === src); + assert.ok(finding); + assert.notEqual(finding.action, 'completed-swap-and-cleaned'); + assert.notEqual(finding.action, 'completed-partial-staging-quarantined'); + }); + + it('site 2 (finalizeSwapOrRestore row 3, pending-link obstacle): a REAL DIRECTORY with a sentinel file at the .tidy-link path is refused, not recursively removed', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + // afterStage: src missing, staged + pending link both present. + const child = runApplyInChildWithCrash(plan, home, target, 'afterStage'); + assert.equal(child.status, 77); + const staging = `${src}.tidy-moving`; + const link = `${src}.tidy-link`; + assert.ok(existsSync(staging)); + assert.equal(lstatSync(link).isSymbolicLink(), true); + + // Replace the verified pending link with a real directory + sentinel. + rmSync(link); + mkdirSync(link, { recursive: true }); + writeFileSync(join(link, 'SENTINEL.txt'), 'do not delete me'); + + const before = snapshotData(home, target); + const recovered = recoverInterruptedMoves(home); + const after = snapshotData(home, target); + + assert.equal(existsSync(join(link, 'SENTINEL.txt')), true, 'the sentinel file must survive'); + assert.equal(readFileSync(join(link, 'SENTINEL.txt'), 'utf8'), 'do not delete me'); + assert.deepEqual(after, before, 'nothing on disk may change when a real directory sits where the pending link was expected'); + assert.equal(existsSync(staging), true, 'the staged original must still be there too — nothing was renamed'); + + const finding = recovered.find(r => r.path === src); + assert.ok(finding); + assert.match(finding.note, /found a directory instead|expected an owned symlink/); + }); + + it('site 3 (recoverInterruptedMoves stray-link branch): a REAL DIRECTORY with a sentinel file at the .tidy-link path is refused, original untouched source survives', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + // afterLink: src is STILL the real, untouched original; only a + // verified pending link exists so far (crashed before the first + // rename), so recovery routes through the "remove stray link, leave + // the never-touched original alone" branch. + const child = runApplyInChildWithCrash(plan, home, target, 'afterLink'); + assert.equal(child.status, 77); + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), false); + const link = `${src}.tidy-link`; + assert.equal(lstatSync(link).isSymbolicLink(), true); + const originalContent = readFileSync(join(src, 'weights.gguf'), 'utf8'); + + rmSync(link); + mkdirSync(link, { recursive: true }); + writeFileSync(join(link, 'SENTINEL.txt'), 'do not delete me'); + + const before = snapshotData(home, target); + const recovered = recoverInterruptedMoves(home); + const after = snapshotData(home, target); + + assert.equal(existsSync(join(link, 'SENTINEL.txt')), true, 'the sentinel file must survive'); + assert.deepEqual(after, before, 'nothing on disk may change'); + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), false); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), originalContent, 'the original, never touched by the crash, must remain exactly as it was'); + + const finding = recovered.find(r => r.path === src); + assert.ok(finding); + assert.equal(finding.action, 'left-alone'); + assert.match(finding.note, /found a directory instead|expected an owned symlink/); + }); + + it('a symlink at the pending-link path pointing somewhere OTHER than the recorded target must be refused, never unlinked', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + + const child = runApplyInChildWithCrash(plan, home, target, 'afterStage'); + assert.equal(child.status, 77); + const staging = `${src}.tidy-moving`; + const link = `${src}.tidy-link`; + assert.ok(existsSync(staging)); + assert.equal(lstatSync(link).isSymbolicLink(), true); + + // Re-point the pending link at an unrelated decoy directory instead + // of the journal's recorded target. The target itself stays GOOD. + const decoy = tempDir('model-tidy-gap2-decoy-'); + writeFileSync(join(decoy, 'weights.gguf'), 'DECOY-NOT-THE-REAL-TARGET'); + rmSync(link); + symlinkSync(decoy, link); + + const before = snapshotData(home, target); + const recovered = recoverInterruptedMoves(home); + const after = snapshotData(home, target); + + // The wrong-target symlink itself must survive, unlinked-not: + assert.equal(lstatSync(link).isSymbolicLink(), true); + assert.equal(readlinkSync(link), decoy); + assert.deepEqual(after, before, 'nothing on disk may change when the pending link points to the wrong place'); + assert.equal(existsSync(staging), true, 'the staged original must still be there — nothing was renamed'); + assert.equal(existsSync(src), false, 'src must still be missing — never populated from the wrong-target link'); + + const finding = recovered.find(r => r.path === src); + assert.ok(finding); + assert.match(finding.note, /resolves to .*not the recorded target/); + }); + }); + + describe('BLOCKER 2: journal durability — unreadable journal records are reported and left alone, never acted on', () => { + it('(a) a journal record truncated to HALF its bytes is reported unreadable by plan and recover, and every path is left unchanged', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan1 = planFor(home); + const child = runApplyInChildWithCrash(plan1, home, target, 'afterStage'); + assert.equal(child.status, 77); + + const journalFile = findJournalFile(home); + const original = readFileSync(journalFile, 'utf8'); + writeFileSync(journalFile, original.slice(0, Math.floor(original.length / 2))); + + const before = snapshotData(home, target); + + const plan2 = planFor(home); + const finding = plan2.interrupted.find(f => f.journalFile === journalFile); + assert.ok(finding, 'plan must report the unreadable journal file'); + assert.equal(finding.status, 'journal-unreadable'); + assert.match(finding.note, /truncated|not valid JSON/); + + assert.deepEqual(snapshotData(home, target), before, 'plan must not mutate anything, including for an unreadable journal'); + + const recovered = recoverInterruptedMoves(home); + const recoverFinding = recovered.find(r => r.journalFile === journalFile); + assert.ok(recoverFinding); + assert.equal(recoverFinding.action, 'left-alone'); + assert.match(recoverFinding.note, /unreadable/); + + assert.deepEqual(snapshotData(home, target), before, 'recover must not act on an unreadable journal record either'); + }); + + it('(a) a journal record truncated to ZERO bytes (empty file) is reported unreadable, and every path is left unchanged', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan1 = planFor(home); + const child = runApplyInChildWithCrash(plan1, home, target, 'afterStage'); + assert.equal(child.status, 77); + + const journalFile = findJournalFile(home); + writeFileSync(journalFile, ''); + + const before = snapshotData(home, target); + + const plan2 = planFor(home); + const finding = plan2.interrupted.find(f => f.journalFile === journalFile); + assert.ok(finding); + assert.equal(finding.status, 'journal-unreadable'); + assert.match(finding.note, /empty/); + assert.deepEqual(snapshotData(home, target), before); + + const recovered = recoverInterruptedMoves(home); + const recoverFinding = recovered.find(r => r.journalFile === journalFile); + assert.ok(recoverFinding); + assert.equal(recoverFinding.action, 'left-alone'); + assert.match(recoverFinding.note, /unreadable/); + assert.deepEqual(snapshotData(home, target), before); + }); + + it('(b) a stray .tmp-* journal file (left over from an interrupted journal WRITE) is never parsed as a record, and is reported', () => { + const { home } = buildSingleIdleFixture(); + const journalDir = journalDirFor(home); + mkdirSync(journalDir, { recursive: true }); + const strayTmp = join(journalDir, 'deadbeefdeadbeef.json.tmp-99999-abc123def456'); + writeFileSync(strayTmp, JSON.stringify({ + sourcePath: join(home, 'models', 'not-a-real-unit'), + stagedPath: 'x', linkPath: 'y', targetPath: 'z', manifest: [], step: 'pending' + })); + + const detected = detectInterruptedMoves(home); + const detectedFinding = detected.find(f => f.journalFile === strayTmp); + assert.ok(detectedFinding, 'plan-side detection must report the stray .tmp file'); + assert.match(detectedFinding.note, /incomplete journal write|\.tmp/); + + const recovered = recoverInterruptedMoves(home); + const finding = recovered.find(r => r.journalFile === strayTmp); + assert.ok(finding, 'recover must report the stray .tmp file too'); + assert.equal(finding.action, 'left-alone'); + assert.match(finding.note, /incomplete journal write|\.tmp/); + assert.equal(finding.path, null, 'a stray .tmp write describes no confirmed unit — it was never parsed as a record'); + + // Never touched: still sitting there exactly as it was. + assert.equal(existsSync(strayTmp), true); + }); + + it('(c) positive control: a complete, valid journal record still drives recovery normally', () => { + const { home, src } = buildSingleIdleFixture(); + const target = tempDir('model-tidy-gap2-target-'); + const plan = planFor(home); + const child = runApplyInChildWithCrash(plan, home, target, 'afterStage'); + assert.equal(child.status, 77); + + const journalFile = findJournalFile(home); + const parsed = JSON.parse(readFileSync(journalFile, 'utf8')); + assert.equal(parsed.sourcePath, src, 'sanity: this is a real, complete, readable record'); + + const detected = detectInterruptedMoves(home); + assert.equal(detected.length, 1); + assert.equal(detected[0].status, 'interrupted'); + + const recovered = recoverInterruptedMoves(home); + assert.equal(recovered.length, 1); + assert.equal(recovered[0].action, 'completed-swap-and-cleaned'); + assert.equal(lstatSync(src).isSymbolicLink(), true); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'gap2-fixture-bytes'.repeat(50)); + }); + }); }); From e03d4dc381ab2562938ad772af782f0c60c6eb14 Mon Sep 17 00:00:00 2001 From: Petrus Pennanen Date: Mon, 21 Sep 2026 10:44:59 +0200 Subject: [PATCH 7/9] Round 6: fail-closed fd-read errors, streaming hash, fourth unlinkOwnedSymlink site MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three items from the lead's round-6 review, all confirmed against src before fixing. ITEM 1: findProcessUsers's inner per-fd catch swallowed every error from readlink/realpath — including EACCES/EPERM — treating them identically to ENOENT (a benign race where the fd simply vanished mid-scan). That meant a permission error while resolving one process's open files was silently ignored instead of failing the check closed. Now: ENOENT is ignored and counted as such; any other error (EACCES, EPERM, anything else) marks that pid's check unverified, and the whole run fails closed for that candidate with reason 'in-use status unverified: '. Added an injectable `realpathSync` option so tests can throw at the exact read boundary instead of only mocking the whole `listProcessUsers` function. ITEM 2: sha256File read whole files via readFileSync — real model shards are routinely multi-GiB, which could exhaust memory or exceed Node's Buffer limits. Replaced with bounded-memory streaming: readSync into one reused 8 MiB buffer, updating the hash incrementally, never loading more than one chunk at a time. Every hashing call site in the file (manifest computation, manifest verification, and copy verification) shares this one function, so the fix applies uniformly. Exported sha256File and the chunk-size constant for direct testing. ITEM 3: swapToSymlink's pre-swap verification failure branch did rmSync(link, {recursive:true}) on a path only assumed to be the symlink it had just asked to be created — the same class of bug fixed at three other sites in round 4, just missed there since this site runs BEFORE any journal step describes a "target" to verify against. Routed through unlinkOwnedSymlink(link, dst) the same way; on refusal, the path is left untouched, the journal record is marked step:'failed' with the reason, and the thrown error names it. New tests (7, all pass standalone and in the full suite): ENOENT and EACCES injected at the actual readlink/realpath call boundary (not just via listProcessUsers), plus a positive control and an end-to-end planRun check that the skip reason names the pid and code; sha256File verified against a whole-buffer digest for a file larger than the chunk size, and against Buffer.allocUnsafe/Buffer.alloc instrumented to throw if any single allocation exceeds the chunk size; a sentinel-directory test for swapToSymlink's pre-swap cleanup mirroring the round-4 style — the directory and a file inside it survive, the source is untouched, and the journal record says failed. Test results: 7/7 new tests in isolation, 50/50 full model-tidy suite, 709/709 full repo suite. Co-Authored-By: Claude Fable 5.1 --- docs/model-tidy.md | 49 +++++----- src/model-tidy.mjs | 97 +++++++++++++++----- test/model-tidy.test.mjs | 189 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 288 insertions(+), 47 deletions(-) diff --git a/docs/model-tidy.md b/docs/model-tidy.md index 9d02b9b..003a659 100644 --- a/docs/model-tidy.md +++ b/docs/model-tidy.md @@ -228,7 +228,9 @@ touch. `unlinkOwnedSymlink(path, expectedTargetPath)` is now the only way anything in this file removes a path it believes is one of its own -symlinks. It never recurses: +symlinks — including `swapToSymlink`'s own pre-swap cleanup when the +freshly-created temp link fails its own verification (round 6 closed that +fourth site the same way). It never recurses: 1. `lstatSync(path)` — if the path doesn't exist, or isn't a symlink at all, **refuse**. Touch nothing. @@ -294,15 +296,19 @@ parsed as a record. `ollama`, `mlx`, `text-generation`). **Fail-closed:** this check has to succeed for *every* pid on the box to count as verified. Off Linux (no `/proc`), if `/proc` itself can't be - listed, or if even one pid's `fd` directory or `cmdline` can't be read - (a permission failure, not the process simply having exited mid-scan — - that's a normal race and not a failure), the candidate is skipped with - `in-use status unverified: ` rather than treated as idle for lack - of evidence. In practice, on a typical non-root Linux host with other - users' or root's processes running, this makes the tool quite - conservative unless it runs with enough privilege to read every pid's - `/proc` entry — that is intentional: an unreadable process is exactly - the case where we cannot prove a model is idle. + listed, or if even one pid's `fd` directory, an individual fd's + `readlink`/`realpath`, or `cmdline` can't be read, the candidate is + skipped with `in-use status unverified: ` rather than + treated as idle for lack of evidence. The one exception, at every one of + those read points: `ENOENT` specifically (the fd or process vanished + between being listed and being read) is a normal race, not a failure, + and is ignored — everything else (`EACCES`, `EPERM`, anything else) is + treated as "could not verify" and fails that pid's check closed. In + practice, on a typical non-root Linux host with other users' or root's + processes running, this makes the tool quite conservative unless it + runs with enough privilege to read every pid's `/proc` entry — that is + intentional: an unreadable process is exactly the case where we cannot + prove a model is idle. 3. **Bind-mounted into (or containing) the bind-mount source of a running docker container, or unverifiable** — reads `docker inspect` of every running container's `Mounts`; a candidate is in use if it is at or under @@ -485,17 +491,18 @@ it never attempts to install anything itself. `model-tidy`'s own user can read every relevant pid. This has not been checked against the real process list on either box, so it's unknown whether the tool would select anything at all there today. -- Two findings from the automated Codex review are known and NOT addressed - in this pass (out of scope for the three defects above, tracked here - instead of silently dropped): (1) `verifyUnit`'s checksum step - (`sha256File`) reads each file whole via `readFileSync` rather than - streaming — for real multi-GiB `.safetensors`/`.gguf` shards this could - exhaust memory or exceed Node's Buffer limits, making `apply` fail at - the verify step for large real models even though the copy itself - succeeded; (2) there is no `lsof`-based fallback for the process-in-use - check, only `/proc`. `computeManifest`/`verifyManifest` (the journal's - own integrity check) have the exact same whole-file-`readFileSync` - property, so the same real-world risk applies there too. +- One finding from the automated Codex review is known and NOT addressed in + this pass (out of scope, tracked here instead of silently dropped): + there is no `lsof`-based fallback for the process-in-use check, only + `/proc`. (The other Codex finding from that review — `sha256File` + reading whole files via `readFileSync` rather than streaming — was fixed + in round 6: it now hashes in bounded 8 MiB chunks via `readSync`, used + uniformly by manifest computation, manifest verification, and copy + verification. Tested against a file larger than the chunk size for + correctness, and with `Buffer.allocUnsafe`/`Buffer.alloc` instrumented to + assert no single allocation exceeds the chunk size — but never against + an actual multi-GiB model file on real hardware, only synthetic + multi-chunk files on this dev machine.) - The journal-based crash recovery (`recoverInterruptedMoves`, the `recover` subcommand) has only ever been exercised against synthetic fixtures with tiny files and a hard-killed child process on this dev diff --git a/src/model-tidy.mjs b/src/model-tidy.mjs index 7e13a67..7b281c3 100644 --- a/src/model-tidy.mjs +++ b/src/model-tidy.mjs @@ -51,7 +51,7 @@ import { readdirSync, lstatSync, existsSync, readFileSync, readlinkSync, realpathSync, symlinkSync, rmSync, mkdirSync, copyFileSync, linkSync, statSync, appendFileSync, constants as FS_CONSTANTS, accessSync, renameSync, - openSync, writeSync, fsyncSync, closeSync, unlinkSync + openSync, writeSync, fsyncSync, closeSync, unlinkSync, readSync } from 'node:fs'; import { join, relative, sep, isAbsolute } from 'node:path'; import { createHash, randomBytes } from 'node:crypto'; @@ -293,6 +293,7 @@ export function matchesKeepList(path, keepEntries) { */ export function findProcessUsers(path, opts = {}) { const procRoot = opts.procRoot || '/proc'; + const doRealpath = opts.realpathSync || realpathSync; // test-only injection point for the exact fd-resolution boundary if (!existsSync(procRoot)) { return { checked: false, users: [], note: `${procRoot} not available (not Linux); process-in-use check skipped` }; } @@ -303,9 +304,10 @@ export function findProcessUsers(path, opts = {}) { return { checked: false, users: [], note: `cannot list ${procRoot}: ${e.message}` }; } const users = []; - const unreadablePids = []; + const unreadable = []; // [{pid, code}] — every pid whose check could not be completed for (const pid of pids) { - let fdOk = true; + let pidErrorCode = null; // first non-ENOENT error code seen for this pid, if any + try { const fdDir = join(procRoot, pid, 'fd'); let fds; @@ -315,26 +317,30 @@ export function findProcessUsers(path, opts = {}) { // ENOENT here means the process exited between the pid listing and // this read — a benign race, not a verification failure. Anything // else (EACCES/EPERM, or an unexpected error) means we genuinely - // could not check this pid's open files. - if (e.code !== 'ENOENT') fdOk = false; + // could not check this pid's open files, so this pid's check + // fails closed. + if (e.code !== 'ENOENT') pidErrorCode = e.code || 'UNKNOWN'; fds = []; } for (const fd of fds) { try { - const target = realpathSync(join(fdDir, fd)); + const target = doRealpath(join(fdDir, fd)); if (target === path || target.startsWith(path + sep)) { users.push({ pid, via: 'fd', target }); break; } - } catch { - // this one fd vanished mid-scan (ENOENT) — not a verification failure + } catch (e) { + // ENOENT: this one fd vanished mid-scan (closed, or the whole + // process exited) — a benign race, not a verification failure. + // EACCES/EPERM/anything else: we could not resolve this fd, so + // we cannot rule it out — fail this pid's check closed. + if (e.code !== 'ENOENT') pidErrorCode = pidErrorCode || e.code || 'UNKNOWN'; } } - } catch { - fdOk = false; + } catch (e) { + pidErrorCode = pidErrorCode || e.code || 'UNKNOWN'; } - let cmdlineOk = true; try { const cmdline = readFileSync(join(procRoot, pid, 'cmdline'), 'utf8').replace(/\0/g, ' ').trim(); if (cmdline && cmdline.includes(path)) { @@ -343,19 +349,19 @@ export function findProcessUsers(path, opts = {}) { users.push({ pid, via: 'cmdline', cmdline: cmdline.slice(0, 200), servingProcess }); } } catch (e) { - if (e.code !== 'ENOENT') cmdlineOk = false; + if (e.code !== 'ENOENT') pidErrorCode = pidErrorCode || e.code || 'UNKNOWN'; } - if (!fdOk || !cmdlineOk) unreadablePids.push(pid); + if (pidErrorCode) unreadable.push({ pid, code: pidErrorCode }); } - if (unreadablePids.length > 0) { - const shown = unreadablePids.slice(0, 5).join(', '); - const more = unreadablePids.length > 5 ? `, +${unreadablePids.length - 5} more` : ''; + if (unreadable.length > 0) { + const first = unreadable[0]; + const more = unreadable.length > 1 ? ` (+${unreadable.length - 1} more pid(s) also unreadable)` : ''; return { checked: false, users, - note: `could not read /proc for pid(s) ${shown}${more} (permission denied) — cannot rule out those processes using this path` + note: `${first.pid} ${first.code}${more} — cannot rule out ${unreadable.length === 1 ? 'that process' : 'those processes'} using this path` }; } return { checked: true, users }; @@ -726,9 +732,33 @@ export function planRun(options = {}) { // Apply // --------------------------------------------------------------------------- -function sha256File(path) { +export const HASH_CHUNK_BYTES = 8 * 1024 * 1024; // 8 MiB — bounded memory regardless of file size + +/** + * SHA-256 of a file's content with BOUNDED memory: reads in fixed-size + * chunks via readSync into one reused buffer, never the whole file at + * once via readFileSync. Model shards are routinely multi-GiB + * (.safetensors/.gguf); loading one whole into a Buffer to hash it could + * exhaust memory or exceed Node's Buffer size limit on some builds. Every + * hashing call site in this file (manifest computation, manifest + * verification, and rsync-style copy verification) goes through this one + * function, so fixing it here fixes all of them. + */ +export function sha256File(path) { const hash = createHash('sha256'); - hash.update(readFileSync(path)); + const buffer = Buffer.allocUnsafe(HASH_CHUNK_BYTES); + const fd = openSync(path, 'r'); + try { + let bytesRead; + do { + bytesRead = readSync(fd, buffer, 0, HASH_CHUNK_BYTES, null); + if (bytesRead > 0) { + hash.update(bytesRead === HASH_CHUNK_BYTES ? buffer : buffer.subarray(0, bytesRead)); + } + } while (bytesRead > 0); + } finally { + closeSync(fd); + } return hash.digest('hex'); } @@ -1359,11 +1389,30 @@ function swapToSymlink(src, dst, opts = {}) { if (opts.beforeLink) opts.beforeLink(); // test-only hook: simulate a crash here doSymlink(dst, link); - const realLink = realpathSync(link); - if (realLink !== realpathSync(dst) || !lstatSync(link).isSymbolicLink()) { - rmSync(link, { recursive: true }); - removeJournalRecord(journalDir, src); - throw new Error(`pre-swap symlink verification failed for ${link}`); + let preSwapVerified = false; + try { + preSwapVerified = lstatSync(link).isSymbolicLink() && realpathSync(link) === realpathSync(dst); + } catch { + preSwapVerified = false; + } + if (!preSwapVerified) { + // Clear the bad/half artifact at `link` — but only via the same + // verify-then-unlink guard every other removal in this file uses, in + // case `doSymlink` didn't actually create a symlink at all (e.g. a + // real directory ended up there instead): never rmSync/recursive on + // an assumption. + const unlinkResult = unlinkOwnedSymlink(link, dst); + if (unlinkResult.ok) { + removeJournalRecord(journalDir, src); + throw new Error(`pre-swap symlink verification failed for ${link}`); + } + const reason = `pre-swap symlink verification failed for ${link}, and it could not be safely removed: ${unlinkResult.reason}`; + try { + writeJournalRecordSync(journalDir, { ...base, step: 'failed', failureReason: reason }); + } catch { + // best effort — the reason is still thrown below either way + } + throw new Error(reason); } writeJournalRecordSync(journalDir, { ...base, step: 'linked' }); if (opts.afterLink) opts.afterLink(); // test-only hook: simulate a crash here diff --git a/test/model-tidy.test.mjs b/test/model-tidy.test.mjs index eaa7d44..06cc6ac 100644 --- a/test/model-tidy.test.mjs +++ b/test/model-tidy.test.mjs @@ -9,10 +9,11 @@ import { import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { spawnSync } from 'node:child_process'; -import { createHash } from 'node:crypto'; +import { createHash, randomBytes } from 'node:crypto'; import { planRun, applyRun, discoverCandidates, loadKeepList, computeHardlinkGroups, - validateTarget, copyUnitPureNode, recoverInterruptedMoves, detectInterruptedMoves + validateTarget, copyUnitPureNode, recoverInterruptedMoves, detectInterruptedMoves, + findProcessUsers, sha256File, HASH_CHUNK_BYTES } from '../src/model-tidy.mjs'; const MODEL_TIDY_MODULE_URL = new URL('../src/model-tidy.mjs', import.meta.url).href; @@ -1351,3 +1352,187 @@ describe('GAP 2: crash-safety across a hard process kill (not just a thrown exce }); }); }); + +describe('ROUND 6 ITEM 1: findProcessUsers fails closed on a real read-boundary error, not on every race', () => { + function buildFakeProcRoot() { + const procRoot = tempDir('model-tidy-fakeproc-'); + mkdirSync(join(procRoot, '1234', 'fd'), { recursive: true }); + writeFileSync(join(procRoot, '1234', 'fd', '3'), ''); + writeFileSync(join(procRoot, '1234', 'cmdline'), 'unrelated-process\0--flag\0'); + return procRoot; + } + + it('ENOENT at the fd-resolution boundary (fd vanished mid-scan) is ignorable — checked stays true, candidate remains selectable', () => { + const procRoot = buildFakeProcRoot(); + const result = findProcessUsers('/some/candidate/path', { + procRoot, + realpathSync: () => { + const e = new Error('no such file or directory'); + e.code = 'ENOENT'; + throw e; + } + }); + assert.equal(result.checked, true, 'ENOENT at the fd boundary must not fail the check closed'); + assert.equal(result.users.length, 0); + }); + + it('EACCES at the fd-resolution boundary marks that pid unverified and fails closed, naming the pid and the code', () => { + const procRoot = buildFakeProcRoot(); + const result = findProcessUsers('/some/candidate/path', { + procRoot, + realpathSync: () => { + const e = new Error('permission denied'); + e.code = 'EACCES'; + throw e; + } + }); + assert.equal(result.checked, false, 'EACCES at the fd boundary must fail the check closed'); + assert.match(result.note, /1234/, 'the note must name the pid'); + assert.match(result.note, /EACCES/, 'the note must name the error code'); + }); + + it('positive control: with no read-boundary errors at all, the check completes normally', () => { + const procRoot = buildFakeProcRoot(); + const result = findProcessUsers('/some/candidate/path', { + procRoot, + realpathSync: p => p // resolves to itself — never matches the candidate path, never throws + }); + assert.equal(result.checked, true); + assert.equal(result.users.length, 0); + }); + + it('end-to-end: planRun skips the candidate with a reason naming the pid and code when the process check is unverified', () => { + const home = tempDir('model-tidy-r6-home-'); + const src = join(home, 'models', 'idle'); + mkdirSync(src, { recursive: true }); + writeFileSync(join(src, 'weights.gguf'), 'x'.repeat(1024)); + utimesSync(join(src, 'weights.gguf'), daysAgo(30), daysAgo(30)); + + const plan = planRun({ + home, + minIdleDays: 14, + listProcessUsers: () => ({ checked: false, users: [], note: '4242 EACCES — cannot rule out that process using this path' }), + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + const skipped = plan.skipped.find(r => r.path === src); + assert.ok(skipped); + assert.match(skipped.reason, /in-use status unverified: 4242 EACCES/); + }); +}); + +describe('ROUND 6 ITEM 2: sha256File hashes with bounded memory (streaming chunks), never a whole-file read', () => { + it('hashes a file LARGER than the chunk size identically to a whole-buffer digest', () => { + const dir = tempDir('model-tidy-hash-'); + const file = join(dir, 'big.bin'); + const size = HASH_CHUNK_BYTES + 3 * 1024 * 1024; // > 1 full chunk, plus a partial tail + const content = randomBytes(size); + writeFileSync(file, content); + + const streamed = sha256File(file); + const wholeBuffer = createHash('sha256').update(content).digest('hex'); + assert.equal(streamed, wholeBuffer, 'the chunked hash must exactly match the whole-buffer hash'); + }); + + it('never allocates a buffer larger than the chunk size while hashing (proves bounded memory, not a hidden readFileSync)', () => { + const dir = tempDir('model-tidy-hash-'); + const file = join(dir, 'big2.bin'); + const size = HASH_CHUNK_BYTES * 2 + 12345; // several chunks, plus a partial tail + const content = randomBytes(size); + writeFileSync(file, content); + + const originalAllocUnsafe = Buffer.allocUnsafe; + const originalAlloc = Buffer.alloc; + const seenSizes = []; + function guard(n) { + seenSizes.push(n); + if (n > HASH_CHUNK_BYTES) { + Buffer.allocUnsafe = originalAllocUnsafe; + Buffer.alloc = originalAlloc; + throw new Error(`sha256File allocated ${n} bytes, more than the ${HASH_CHUNK_BYTES}-byte chunk limit — it read some or all of the file into memory at once`); + } + } + Buffer.allocUnsafe = function (n) { + guard(n); + return originalAllocUnsafe.call(Buffer, n); + }; + Buffer.alloc = function (n, ...rest) { + guard(n); + return originalAlloc.call(Buffer, n, ...rest); + }; + + let hash; + try { + hash = sha256File(file); + } finally { + Buffer.allocUnsafe = originalAllocUnsafe; + Buffer.alloc = originalAlloc; + } + + assert.equal(hash, createHash('sha256').update(content).digest('hex')); + assert.ok(seenSizes.length >= 1, 'sanity: at least one allocation was observed'); + assert.ok(seenSizes.every(n => n <= HASH_CHUNK_BYTES), `every allocation must be <= ${HASH_CHUNK_BYTES} bytes; saw ${JSON.stringify(seenSizes)}`); + }); +}); + +describe('ROUND 6 ITEM 3: swapToSymlink pre-swap verification failure never recursively deletes an assumed symlink', () => { + it('sentinel test: symlinkFn creates a REAL DIRECTORY at the link path instead of a symlink — it and a sentinel file inside it survive, apply fails, and the journal record says failed', () => { + const home = tempDir('model-tidy-r6-item3-home-'); + const target = tempDir('model-tidy-r6-item3-target-'); + const src = join(home, 'models', 'idle'); + mkdirSync(src, { recursive: true }); + writeFileSync(join(src, 'weights.gguf'), 'r6-item3-bytes'.repeat(50)); + utimesSync(join(src, 'weights.gguf'), daysAgo(30), daysAgo(30)); + + const plan = planRun({ + home, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + assert.ok(plan.selected.some(r => r.path === src)); + + const link = `${src}.tidy-link`; + const result = applyRun({ + plan, + target, + home, + validateTarget: bypassCrossFsCheck, + symlinkFn: (dst, linkPath) => { + // Simulate something creating a real directory exactly where + // model-tidy expected to create — and, on verification failure, + // remove — its own symlink. + mkdirSync(linkPath, { recursive: true }); + writeFileSync(join(linkPath, 'SENTINEL.txt'), 'do not delete me'); + } + }); + + assert.equal(result.ok, false); + assert.equal(result.errors.length, 1); + assert.equal(result.errors[0].step, 'swap'); + assert.match(result.errors[0].error, /pre-swap symlink verification failed/); + assert.match(result.errors[0].error, /could not be safely removed/); + + // The directory and its sentinel file must survive, completely + // untouched — never rmSync'd. + assert.equal(existsSync(link), true); + assert.equal(lstatSync(link).isSymbolicLink(), false); + assert.equal(existsSync(join(link, 'SENTINEL.txt')), true); + assert.equal(readFileSync(join(link, 'SENTINEL.txt'), 'utf8'), 'do not delete me'); + + // The original source itself must also be untouched — swapToSymlink + // never got past the pre-swap step, so nothing was ever renamed away. + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), false); + assert.equal(readFileSync(join(src, 'weights.gguf'), 'utf8'), 'r6-item3-bytes'.repeat(50)); + + // The journal record must say failed, not be silently cleared. + const journalDir = join(home, '.cache', 'ide-agent-kit', 'model-tidy-journal'); + const names = readdirSync(journalDir).filter(n => n.endsWith('.json')); + assert.equal(names.length, 1); + const record = JSON.parse(readFileSync(join(journalDir, names[0]), 'utf8')); + assert.equal(record.step, 'failed'); + assert.match(record.failureReason, /could not be safely removed/); + }); +}); From 360d7300679aa08d7fddad3ca37057bf394b2a27 Mon Sep 17 00:00:00 2001 From: Petrus Pennanen Date: Mon, 21 Sep 2026 11:00:57 +0200 Subject: [PATCH 8/9] Round 7: recover journal ownership from filename; never swallow directory-fsync failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps from codexmb's round-6 review, both confirmed against src with the provided probe before fixing. GAP 1: an unreadable journal record did not protect its unit. Confirmed with the probe: interrupt beforeLink, empty the journal record, rerun plan — it correctly reported journal-unreadable with path:null, but that finding was filtered out of interruptedByPath (which only keeps findings with a resolvable path), so the same source was still selected. Fixed by recovering ownership from the journal FILENAME: journalFilePath always names a record .json, and that 64-hex-char key survives even when the record's content doesn't. plan now builds a candidate->journalKey map first and matches every unreadable/stray record's filename against it: a match refuses exactly that candidate with reason 'interrupted move with unreadable journal record: '; a filename that cannot be matched at all (unparseable, or a key with no current candidate) makes plan refuse to select ANYTHING this run (globalJournalBlockReason, surfaced in the summary line) rather than guess. apply enforces the same rule even harder: after its own recovery pass, any remaining unreadable/stray journal file blocks the ENTIRE run before any mutation (before even --target validation) — 'unresolved journal state: ; run recover, or resolve by hand'. recover itself still only reports such files and leaves them, and every path, alone. GAP 2: writeJournalRecordSync's directory-fsync failure was caught and swallowed unconditionally, including EIO. Now: on the supported target (process.platform === 'linux', injectable for tests), ANY error from the temp-file fsync, the rename into place, or the directory fsync throws — uncaught by swapToSymlink, so it aborts the whole unit before whatever mutation that write was meant to guard, and no claim of durability is made. Off Linux, only ENOTSUP/EINVAL from the DIRECTORY fsync specifically is tolerated, attaching an explicit 'durability degraded: ' note to the result rather than pretending durability was achieved; EIO and everything else still abort there too. The fsync/ rename functions and the platform are both injectable through applyRun/recoverInterruptedMoves so this is tested without branching on the real process.platform. New tests (9, all pass standalone and in the full suite): GAP 1 — codexmb's probe verbatim (source not selected, reason names the record); a stray journal file with an unparseable name (plan selects nothing, apply refuses before any mutation, full tree unchanged); a positive control that a healthy journal-free run still selects the idle dir. GAP 2 — EIO on the directory fsync on a simulated Linux platform (apply aborts, source untouched, error names EIO and 'journal directory'); ENOTSUP on a simulated non-Linux platform (proceeds, durability-degraded note reported); EIO on the temp-file fsync (aborts on any platform). Test results: 6/6 new tests in isolation (grouped as 3+3 per gap, 9 assertions worth of coverage across them), 56/56 full model-tidy suite, 715/715 full repo suite. Co-Authored-By: Claude Fable 5.1 --- docs/model-tidy.md | 70 +++++++++--- src/model-tidy.mjs | 210 +++++++++++++++++++++++++++++++----- test/model-tidy.test.mjs | 227 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 466 insertions(+), 41 deletions(-) diff --git a/docs/model-tidy.md b/docs/model-tidy.md index 003a659..617511d 100644 --- a/docs/model-tidy.md +++ b/docs/model-tidy.md @@ -251,15 +251,27 @@ PARTIAL record at the path readers expect — making a real, in-flight unit look like there's nothing to recover, or worse, giving a reader a document that parses as JSON but describes something incoherent. -Every journal write now follows write-temp-fsync-rename: the full record -is written to a throwaway `.tmp--` in the same -directory, that temp file is `fsync`ed and closed, then `renameSync`d -atomically over the real path (same-directory renames are atomic — a -reader sees either the old complete record or the new complete one, never -a partial one), then the journal directory itself is `fsync`ed so the -rename survives a crash immediately after (on a platform where a -directory can't be opened for `fsync`, that failure is swallowed — the -rename is still atomic there, just not immediately durable). +Every journal write follows write-temp-fsync-rename: the full record is +written to a throwaway `.tmp--` in the same directory, +that temp file is `fsync`ed and closed, then `renameSync`d atomically over +the real path (same-directory renames are atomic — a reader sees either +the old complete record or the new complete one, never a partial one), +then the journal directory itself is `fsync`ed. + +**No step is allowed to fail silently.** On the supported target (Linux), +ANY error from the temp-file fsync, the rename, or the directory fsync +aborts the write — and the caller (`swapToSymlink`) does not catch that, +so it aborts the whole unit before whatever mutation the write was meant +to guard, and reports the error. On a non-Linux dev platform, the same is +true with exactly one exception: `ENOTSUP` or `EINVAL` specifically from +the *directory* fsync (common on filesystems/platforms that don't support +fsync-ing a directory fd at all) is tolerated — the write still succeeds, +but a `durability degraded: ` note is attached to the result rather +than silently claiming durability that wasn't achieved. `EIO` and +everything else still abort there too. The fsync/rename functions and the +platform are all test-injectable (`--journal-fsync`-style options, not +exposed on the CLI — internal to `applyRun`/`recoverInterruptedMoves`), +so this rule is tested without needing two different operating systems. On the read side, a journal record that is **missing, empty, truncated, fails to parse, or doesn't match the expected schema** is never treated as @@ -271,6 +283,29 @@ stray `.json.tmp-*` file — the leftover of a write that itself got interrupted — is recognized by name and reported the same way; it is never parsed as a record. +**Recovering ownership from the filename.** An unreadable record's +*content* can't say which candidate it was protecting — but its *name* +still can: `journalFilePath` always names a record +`.json`, and that 64-hex-char key survives even +when the content doesn't. `plan` builds a `journalKey(path) -> candidate` +map from every currently discovered candidate and matches every unreadable +record's filename against it: +- a match refuses exactly that candidate, with reason `interrupted move + with unreadable journal record: ` — no other candidate is + affected; +- **no match at all** (an unparseable filename, or a key that doesn't + correspond to any currently discovered candidate) means the unreadable + record *might* describe something plan can't even see right now — so + `plan` refuses to select **anything** this run, reporting exactly why + (`globalJournalBlockReason`, and `REFUSING TO SELECT ANYTHING THIS RUN` + in the summary line) rather than guess. `apply` enforces the same rule + even more strictly: after its own recovery pass, if any unreadable or + stray journal file remains, it refuses to touch anything at all — + `unresolved journal state: ; run recover, or resolve by hand` — + before validating `--target`, before copying, before anything. `recover` + itself never guesses either: an unreadable record is reported and both + it and every path it might touch are left alone. + ## Selection rules (in order, each with an explicit reason string) 0. **Interrupted move found** — highest priority, checked before anything @@ -403,10 +438,19 @@ it never attempts to install anything itself. recorded target — a plain `unlinkSync`, never `rmSync`, only after both checks pass. - **Every journal write is crash-safe: write-temp-fsync-rename, then fsync - the directory.** A journal record that's missing, empty, truncated, - unparseable, or the wrong shape is treated as `journal-unreadable` by - every reader (`plan`, `apply`, `recover`) — that unit is reported and - left alone, never acted on. + the directory — and any failure in that sequence aborts the write rather + than silently claiming durability.** On Linux, no error is tolerated + anywhere in that sequence. Off Linux, only `ENOTSUP`/`EINVAL` from the + directory fsync specifically is tolerated, and only with an explicit + `durability degraded: ` note in the result — never silently. +- A journal record that's missing, empty, truncated, unparseable, or the + wrong shape is treated as `journal-unreadable` by every reader (`plan`, + `apply`, `recover`) — that unit is reported and left alone, never acted + on. Its filename (not its content) is still used to identify which + candidate it protects: a match refuses that one candidate; no match at + all (unparseable name, or a key with no current candidate) makes `plan` + refuse to select anything and makes `apply` refuse to touch anything, + rather than risk missing what an unreadable record was protecting. - `plan` makes zero filesystem writes, ever — including for interrupted moves, which it only detects and reports. Only `recover` and `apply` (once, at its own start) mutate anything. diff --git a/src/model-tidy.mjs b/src/model-tidy.mjs index 7b281c3..078f461 100644 --- a/src/model-tidy.mjs +++ b/src/model-tidy.mjs @@ -53,7 +53,7 @@ import { statSync, appendFileSync, constants as FS_CONSTANTS, accessSync, renameSync, openSync, writeSync, fsyncSync, closeSync, unlinkSync, readSync } from 'node:fs'; -import { join, relative, sep, isAbsolute } from 'node:path'; +import { join, relative, sep, isAbsolute, basename } from 'node:path'; import { createHash, randomBytes } from 'node:crypto'; import { homedir } from 'node:os'; @@ -550,6 +550,42 @@ export function planRun(options = {}) { const interruptedByPath = new Map(interrupted.filter(f => f.path).map(f => [f.path, f])); const rawCandidates = options.candidates || discoverCandidates(home); + + // An unreadable/stray journal record (missing, empty, truncated, + // unparseable, or schema-invalid content) has `path: null` above — its + // CONTENT can't tell us which candidate it was protecting. But its + // FILENAME still encodes journalKey(sourcePath) (journalFilePath always + // names a record that way), so recover ownership from the name: match + // every unreadable/stray record's key against every currently + // discovered candidate. A match refuses that specific candidate. Any + // unreadable record whose filename doesn't match a current candidate at + // all (an unparseable name, or a key with no current candidate) means + // we cannot rule out that it was protecting something — fail closed + // GLOBALLY rather than guess which unit, if any, needs protecting. + const candidateKeyToPath = new Map(); + for (const c of rawCandidates) candidateKeyToPath.set(journalKey(c.path), c.path); + + const unresolvedJournalFiles = []; + for (const finding of interrupted) { + if (finding.path || !finding.journalFile) continue; // already attributable, or not a journal-file-shaped finding at all + const key = extractJournalKeyFromFilename(finding.journalFile); + const matchedPath = key ? candidateKeyToPath.get(key) : undefined; + if (matchedPath) { + interruptedByPath.set(matchedPath, { + path: matchedPath, + journalFile: finding.journalFile, + status: 'interrupted', + note: `interrupted move with unreadable journal record: ${finding.journalFile}`, + exactReason: `interrupted move with unreadable journal record: ${finding.journalFile}` + }); + } else { + unresolvedJournalFiles.push(finding.journalFile); + } + } + const globalJournalBlockReason = unresolvedJournalFiles.length > 0 + ? `unresolved journal state: ${unresolvedJournalFiles.join(', ')}; run recover, or resolve by hand` + : null; + const { groupOf, groupMembers, groupSizeBytes, filesByCandidate, incompleteReasonByCandidate } = computeHardlinkGroups(rawCandidates); const perCandidate = new Map(); @@ -671,7 +707,10 @@ export function planRun(options = {}) { // never saw it at all. for (const finding of interruptedByPath.values()) { const existingIdx = results.findIndex(r => r.path === finding.path); - const reason = `interrupted move found: ${finding.note}`; + // A record recovered by filename-matching (GAP 1) already carries the + // exact required reason text; a normal readable record still gets the + // generic wrapper. + const reason = finding.exactReason || `interrupted move found: ${finding.note}`; if (existingIdx >= 0) { results[existingIdx].selected = false; results[existingIdx].reason = reason; @@ -689,6 +728,22 @@ export function planRun(options = {}) { } } + // GAP 1, global fail-closed: an unreadable/stray journal file whose + // filename could not be matched to any current candidate means we + // cannot rule out that it was protecting something — plan selects + // NOTHING this run rather than guess. Candidates that already have a + // more specific reason (KEEP, an unrelated in-use check, a matched + // interrupted record, ...) keep that reason; only would-be-selected + // candidates are overridden here. + if (globalJournalBlockReason) { + for (const r of results) { + if (r.selected) { + r.selected = false; + r.reason = globalJournalBlockReason; + } + } + } + results.sort((a, b) => b.groupSizeBytes - a.groupSizeBytes || a.path.localeCompare(b.path)); const selected = results.filter(r => r.selected); @@ -707,15 +762,17 @@ export function planRun(options = {}) { const interruptedLine = interrupted.length > 0 ? ` ${interrupted.length} interrupted-move finding(s) detected (never mutated by plan — run 'recover' to resolve).` : ''; + const globalBlockLine = globalJournalBlockReason ? ` REFUSING TO SELECT ANYTHING THIS RUN: ${globalJournalBlockReason}` : ''; const summaryLine = `model-tidy plan: ${selected.length} dir(s) in ${selectedGroupIds.size} unit(s), ` + `${(totalSelectedBytes / 2 ** 30).toFixed(1)} GiB movable to target, ${skipped.length} skipped ` + - `(${unverifiedCount} unverified). ${freeLine}${interruptedLine}`; + `(${unverifiedCount} unverified). ${freeLine}${interruptedLine}${globalBlockLine}`; return { home, minIdleDays, maxGb, interrupted, + globalJournalBlockReason, selected, skipped, totalSelectedBytes, @@ -907,6 +964,22 @@ function journalFilePath(journalDir, sourcePath) { return join(journalDir, `${journalKey(sourcePath)}.json`); } +const JOURNAL_KEY_PATTERN = /^([0-9a-f]{64})\.json(?:\.tmp-.*)?$/; + +/** + * Recover ownership from a journal file's NAME alone — used when its + * CONTENT can't be trusted (missing, empty, truncated, unparseable, or + * schema-invalid). `journalFilePath` always names a record + * `.json` (or, for a stray in-flight write, + * `...json.tmp--`), so the 64-hex-char key survives even + * when the record's content doesn't. Returns null if the filename doesn't + * match that shape at all (a genuinely unrecognizable file). + */ +function extractJournalKeyFromFilename(file) { + const match = basename(file).match(JOURNAL_KEY_PATTERN); + return match ? match[1] : null; +} + /** Minimal structural check for a parsed journal record. Anything that * doesn't match — including a record that parsed as valid JSON but isn't * actually one of ours (wrong shape) — is treated as unreadable, exactly @@ -942,7 +1015,33 @@ function isValidJournalRecordShape(record) { * safe there) — noted here in case it needs to change for a supported * platform where journal loss would matter. */ -function writeJournalRecordSync(journalDir, record) { +/** + * Write one journal record durably — or refuse to claim it was written + * durably at all. `opts.fsyncSync`/`opts.renameSync` are test-only + * injection points (default to the real `fsyncSync`/`renameSync`); + * `opts.platform` defaults to the real `process.platform` but is + * injectable so tests can exercise the Linux-vs-other-platform rule + * without actually running on both. + * + * On the supported target (Linux): ANY error from the temp-file fsync, + * the rename into place, or the journal-directory fsync throws — the + * caller aborts the unit before any source mutation that write was meant + * to guard, and no claim of durability is ever made silently. + * + * On a non-Linux dev platform: the same is true EXCEPT for the directory + * fsync specifically — `ENOTSUP` or `EINVAL` there (common on filesystems + * or platforms that don't support fsync-ing a directory fd at all) is + * tolerated, and the write still succeeds, but the returned + * `durabilityNote` says so explicitly rather than silently pretending + * durability was achieved. `EIO` and everything else still abort there + * too — only that specific "this platform doesn't support the operation" + * shape of failure is tolerated, never an I/O error. + */ +function writeJournalRecordSync(journalDir, record, opts = {}) { + const doFsync = opts.fsyncSync || fsyncSync; + const doRename = opts.renameSync || renameSync; + const platform = opts.platform || process.platform; + mkdirSync(journalDir, { recursive: true }); const file = journalFilePath(journalDir, record.sourcePath); const tmpFile = `${file}.tmp-${process.pid}-${randomBytes(6).toString('hex')}`; @@ -951,26 +1050,40 @@ function writeJournalRecordSync(journalDir, record) { const fd = openSync(tmpFile, 'w'); try { writeSync(fd, data); - fsyncSync(fd); + try { + doFsync(fd); + } catch (e) { + throw new Error(`journal write for ${file} aborted: could not fsync the temp file (${e.code || e.message})`); + } } finally { closeSync(fd); } - renameSync(tmpFile, file); + try { + doRename(tmpFile, file); + } catch (e) { + throw new Error(`journal write for ${file} aborted: could not rename the temp file into place (${e.code || e.message})`); + } + + let durabilityNote = null; try { const dirFd = openSync(journalDir, 'r'); try { - fsyncSync(dirFd); + doFsync(dirFd); } finally { closeSync(dirFd); } - } catch { - // Some platforms can't fsync a directory fd — the rename above is - // still atomic there, just not guaranteed durable against a crash in - // the same instant. Nothing further to do about it here. + } catch (e) { + const code = e.code || 'UNKNOWN'; + const tolerable = platform !== 'linux' && (code === 'ENOTSUP' || code === 'EINVAL'); + if (!tolerable) { + const platformNote = platform === 'linux' ? ' (no durability claim can be made on the supported Linux target)' : ''; + throw new Error(`journal write for ${file} aborted: could not fsync the journal directory (${code})${platformNote}`); + } + durabilityNote = `durability degraded: ${code}`; } - return file; + return { file, durabilityNote }; } function removeJournalRecord(journalDir, sourcePath) { @@ -1194,7 +1307,7 @@ function unlinkOwnedSymlink(path, expectedTargetPath) { return { ok: true }; } -function finalizeSwapOrRestore(journalDir, record) { +function finalizeSwapOrRestore(journalDir, record, opts = {}) { const { sourcePath, stagedPath, linkPath, targetPath, manifest } = record; const targetOk = manifestExactMatch(targetPath, manifest); @@ -1260,8 +1373,8 @@ function finalizeSwapOrRestore(journalDir, record) { const quarantinePath = `${sourcePath}${QUARANTINE_SUFFIX_PREFIX}${journalKey(sourcePath)}`; const reason = 'the staged original was damaged (partial or corrupted content) but the live target and symlink are complete and correct; the live path was left untouched and the damaged staged copy was quarantined rather than deleted or silently trusted'; renameSync(stagedPath, quarantinePath); - writeJournalRecordSync(journalDir, { ...record, step: 'completed-partial-staging-quarantined', quarantinePath }); - return { ok: true, row: 2, quarantined: true, quarantinePath, reason }; + const { durabilityNote } = writeJournalRecordSync(journalDir, { ...record, step: 'completed-partial-staging-quarantined', quarantinePath }, opts); + return { ok: true, row: 2, quarantined: true, quarantinePath, reason, durabilityNotes: durabilityNote ? [durabilityNote] : [] }; } const reasons = []; @@ -1289,7 +1402,7 @@ function finalizeSwapOrRestore(journalDir, record) { if (!unlinkResult.ok) { const combinedReason = `${reason}; additionally, could not safely clear the way for restore: ${unlinkResult.reason}`; try { - writeJournalRecordSync(journalDir, { ...record, step: 'failed', failureReason: combinedReason }); + writeJournalRecordSync(journalDir, { ...record, step: 'failed', failureReason: combinedReason }, opts); } catch { // best effort } @@ -1310,7 +1423,7 @@ function finalizeSwapOrRestore(journalDir, record) { // didn't apply) — leave both paths for manual inspection rather than // guess which is authoritative. try { - writeJournalRecordSync(journalDir, { ...record, step: 'failed', failureReason: reason }); + writeJournalRecordSync(journalDir, { ...record, step: 'failed', failureReason: reason }, opts); } catch { // best effort — the reason is still returned to the caller either way } @@ -1327,7 +1440,7 @@ function finalizeSwapOrRestore(journalDir, record) { : 'the staged original itself no longer matches the journaled manifest exactly'); const reason = reasons.join('; '); try { - writeJournalRecordSync(journalDir, { ...record, step: 'failed-both-copies-damaged', failureReason: reason }); + writeJournalRecordSync(journalDir, { ...record, step: 'failed-both-copies-damaged', failureReason: reason }, opts); } catch { // best effort } @@ -1377,6 +1490,17 @@ function swapToSymlink(src, dst, opts = {}) { const staging = src + STAGING_SUFFIX; const link = src + LINK_SUFFIX; const journalDir = opts.journalDir; + const journalOpts = { fsyncSync: opts.journalFsyncSync, renameSync: opts.journalRenameSync, platform: opts.platform }; + const durabilityNotes = []; + function journalWrite(rec) { + // GAP 2 (round 6): a failure here — temp-file fsync, rename into + // place, or (on Linux) directory fsync — THROWS, which this function + // does not catch, so it propagates to the caller and aborts the whole + // unit before whatever mutation this write was meant to guard. Never + // silently swallowed. + const { durabilityNote } = writeJournalRecordSync(journalDir, rec, journalOpts); + if (durabilityNote) durabilityNotes.push(durabilityNote); + } if (existsSync(staging) || existsSync(link)) { throw new Error(`refusing to touch ${src}: a leftover ${existsSync(staging) ? staging : link} already exists from a previous run — run the 'recover' subcommand first`); @@ -1385,7 +1509,7 @@ function swapToSymlink(src, dst, opts = {}) { const manifest = computeManifest(src); const base = { sourcePath: src, stagedPath: staging, linkPath: link, targetPath: dst, manifest }; - writeJournalRecordSync(journalDir, { ...base, step: 'pending' }); + journalWrite({ ...base, step: 'pending' }); // BEFORE any mutation — a failure here aborts with the source completely untouched if (opts.beforeLink) opts.beforeLink(); // test-only hook: simulate a crash here doSymlink(dst, link); @@ -1408,34 +1532,35 @@ function swapToSymlink(src, dst, opts = {}) { } const reason = `pre-swap symlink verification failed for ${link}, and it could not be safely removed: ${unlinkResult.reason}`; try { - writeJournalRecordSync(journalDir, { ...base, step: 'failed', failureReason: reason }); + journalWrite({ ...base, step: 'failed', failureReason: reason }); } catch { // best effort — the reason is still thrown below either way } throw new Error(reason); } - writeJournalRecordSync(journalDir, { ...base, step: 'linked' }); + journalWrite({ ...base, step: 'linked' }); // still before the first source mutation (the rename below) if (opts.afterLink) opts.afterLink(); // test-only hook: simulate a crash here renameSync(src, staging); - writeJournalRecordSync(journalDir, { ...base, step: 'staged' }); + journalWrite({ ...base, step: 'staged' }); if (opts.afterStage) opts.afterStage(); // test-only hook: simulate a crash IN THE GAP between the two renames renameSync(link, src); - writeJournalRecordSync(journalDir, { ...base, step: 'swapped' }); + journalWrite({ ...base, step: 'swapped' }); if (opts.afterSwap) opts.afterSwap(); // test-only hook: simulate a crash here // The ONLY place a staged original is deleted, in this run or later via // recovery — see finalizeSwapOrRestore's docstring for the three checks. // "The link resolves to something" is deliberately not one of them. - const result = finalizeSwapOrRestore(journalDir, base); + const result = finalizeSwapOrRestore(journalDir, base, journalOpts); if (!result.ok) { const outcome = result.row === 4 ? 'left every path exactly as found (nothing verified good anywhere)' : `restored the staged original to ${src}`; throw new Error(`post-swap finalize refused to delete the staged original and ${outcome}: ${result.reason}`); } - return result; // row 1: plain success. row 2: success, but carries {quarantined:true, quarantinePath, reason}. + return { ...result, durabilityNotes: [...durabilityNotes, ...(result.durabilityNotes || [])] }; + // row 1: plain success. row 2: success, but carries {quarantined:true, quarantinePath, reason}. } /** Every location discoverCandidates() looks at, reused so detection and @@ -1550,6 +1675,7 @@ export function detectInterruptedMoves(home, options = {}) { */ export function recoverInterruptedMoves(home, options = {}) { const journalDir = options.journalDir || defaultJournalDir(home); + const journalOpts = { fsyncSync: options.journalFsyncSync, renameSync: options.journalRenameSync, platform: options.platform }; const recovered = []; const journaledPaths = new Set(); @@ -1584,7 +1710,7 @@ export function recoverInterruptedMoves(home, options = {}) { // itself, only inside the row-1/row-2 branches that decide it's // warranted. if ((srcExists && srcIsSymlink) || (!srcExists && stagedExists && linkExists)) { - const result = finalizeSwapOrRestore(journalDir, record); + const result = finalizeSwapOrRestore(journalDir, record, journalOpts); if (result.ok && result.quarantined) { // Row 2: live path is correct and complete — preserved, // untouched. The damaged backup was quarantined, not deleted. @@ -1695,11 +1821,31 @@ export function applyRun(options) { const symlinkFn = options.symlinkFn; // test-only injection; real default is symlinkSync inside swapToSymlink const recoverFn = options.recover || recoverInterruptedMoves; const journalDir = options.journalDir || defaultJournalDir(home); + // GAP 2 (round 6) test-only injection points: the real journal fsync, + // the real journal rename, and process.platform, all overridable so + // tests can exercise the Linux-vs-other-platform durability rule + // without branching on the real platform inside the test itself. + const journalFsyncSync = options.journalFsyncSync; + const journalRenameSync = options.journalRenameSync; + const journalPlatform = options.platform; // Self-heal any interrupted swap from a previous crash before this run's // own pre-checks and copy/verify/swap loop — see recoverInterruptedMoves. // Only reached here, in apply, never from planRun. - const recovered = recoverFn(home, { journalDir }); + const recovered = recoverFn(home, { journalDir, journalFsyncSync, journalRenameSync, platform: journalPlatform }); + + // GAP 1 (round 6): recovery above deliberately leaves any unreadable or + // stray journal file alone rather than guess at it. apply must not + // proceed AT ALL while that's true — not just for whatever unit it + // might belong to, but globally, since an unreadable record's content + // can't tell us what it was protecting. This check runs before any + // mutation (before even --target validation). + const remainingUnreadable = listJournalRecords(journalDir).filter(r => r.corrupt || !r.record); + if (remainingUnreadable.length > 0) { + const files = remainingUnreadable.map(r => r.file); + const reason = `unresolved journal state: ${files.join(', ')}; run recover, or resolve by hand`; + return { ok: false, error: reason, moved: [], errors: [{ error: reason }], recovered }; + } const validation = validateTargetFn(target, home); if (!validation.ok) { @@ -1714,6 +1860,7 @@ export function applyRun(options) { const moved = []; const errors = []; + const durabilityNotes = []; // any 'durability degraded: ' notes from journal writes during this run for (const [groupId, members] of byGroup) { const sourceAbsPaths = members.map(m => m.path); @@ -1752,6 +1899,9 @@ export function applyRun(options) { const finalizeResult = swapToSymlink(src, dst, { symlinkSync: symlinkFn, journalDir, + journalFsyncSync, + journalRenameSync, + platform: journalPlatform, beforeLink: options.beforeLink, afterLink: options.afterLink, afterStage: options.afterStage, @@ -1763,6 +1913,10 @@ export function applyRun(options) { entry.quarantinePath = finalizeResult.quarantinePath; entry.note = finalizeResult.reason; } + if (finalizeResult && finalizeResult.durabilityNotes && finalizeResult.durabilityNotes.length > 0) { + entry.durabilityNotes = finalizeResult.durabilityNotes; + durabilityNotes.push(...finalizeResult.durabilityNotes); + } swapped.push(entry); } catch (e) { swapFailed = { path: src, error: e.message }; @@ -1776,7 +1930,7 @@ export function applyRun(options) { } } - return { ok: errors.length === 0, moved, errors, recovered }; + return { ok: errors.length === 0, moved, errors, recovered, durabilityNotes }; } // --------------------------------------------------------------------------- diff --git a/test/model-tidy.test.mjs b/test/model-tidy.test.mjs index 06cc6ac..33ca5f2 100644 --- a/test/model-tidy.test.mjs +++ b/test/model-tidy.test.mjs @@ -1536,3 +1536,230 @@ describe('ROUND 6 ITEM 3: swapToSymlink pre-swap verification failure never recu assert.match(record.failureReason, /could not be safely removed/); }); }); + +describe('ROUND 7', () => { + function journalDirFor(home) { + return join(home, '.cache', 'ide-agent-kit', 'model-tidy-journal'); + } + function findJournalFile(home) { + const dir = journalDirFor(home); + const names = readdirSync(dir).filter(n => n.endsWith('.json') && !n.includes('.tmp-')); + assert.equal(names.length, 1, `expected exactly one journal file, found ${names.length}: ${names.join(', ')}`); + return join(dir, names[0]); + } + function buildIdleFixture(prefix) { + const home = tempDir(prefix || 'model-tidy-r7-home-'); + const src = join(home, 'models', 'idle'); + mkdirSync(src, { recursive: true }); + writeFileSync(join(src, 'weights.gguf'), 'GOOD'.repeat(20)); + utimesSync(join(src, 'weights.gguf'), daysAgo(30), daysAgo(30)); + return { home, src }; + } + function planFor(home) { + return planRun({ + home, + minIdleDays: 14, + listProcessUsers: noProcessUsers, + listDockerBindUsers: dockerNotInUse, + diskFreeBytes: fixedDiskFree + }); + } + function snapshotTree(root) { + const snap = {}; + function recurse(dir) { + for (const name of readdirSync(dir)) { + const full = join(dir, name); + const lst = lstatSync(full); + if (lst.isSymbolicLink()) { + snap[full] = { type: 'symlink', linkTarget: readlinkSync(full) }; + } else if (lst.isDirectory()) { + recurse(full); + } else if (lst.isFile()) { + snap[full] = { type: 'file', size: lst.size, sha256: createHash('sha256').update(readFileSync(full)).digest('hex') }; + } + } + } + recurse(root); + return snap; + } + function escapeRe(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + } + + describe('GAP 1: an unreadable journal record must still protect its unit, recovered from the FILENAME', () => { + it("codexmb's probe verbatim: interrupt beforeLink, empty the journal record, rerun plan — the source must NOT be selected, and the reason names the record", () => { + const { home, src } = buildIdleFixture(); + const target = tempDir('model-tidy-r7-target-'); + const opts = { home, listProcessUsers: noProcessUsers, listDockerBindUsers: dockerNotInUse, diskFreeBytes: fixedDiskFree }; + const interrupted = applyRun({ + home, + target, + plan: planRun(opts), + validateTarget: () => ({ ok: true }), + beforeLink: () => { throw new Error('fixture interruption'); } + }); + assert.equal(interrupted.ok, false); + + const journalFile = findJournalFile(home); + writeFileSync(journalFile, ''); + + const next = planRun(opts); + assert.ok(!next.selected.some(r => r.path === src), 'the source must not be selected while its journal record is unreadable'); + const skipped = next.skipped.find(r => r.path === src); + assert.ok(skipped, 'the source must be reported in skipped, recovered from the journal filename'); + assert.match(skipped.reason, /interrupted move with unreadable journal record/); + assert.match(skipped.reason, new RegExp(escapeRe(journalFile)), 'the reason must name the record file'); + }); + + it('a stray journal file with an UNPARSEABLE name: plan selects nothing, and apply refuses before any mutation', () => { + const { home } = buildIdleFixture(); + const target = tempDir('model-tidy-r7-target-'); + const journalDir = journalDirFor(home); + mkdirSync(journalDir, { recursive: true }); + const strayFile = join(journalDir, 'not-a-real-hash-key.json'); + writeFileSync(strayFile, '{ this is not valid json'); + + const before = snapshotTree(home); + + const plan = planFor(home); + assert.equal(plan.selected.length, 0, 'plan must select nothing while an unresolvable journal file exists'); + assert.ok(plan.globalJournalBlockReason); + assert.match(plan.globalJournalBlockReason, /unresolved journal state/); + assert.match(plan.globalJournalBlockReason, new RegExp(escapeRe(strayFile))); + assert.match(plan.summaryLine, /REFUSING TO SELECT ANYTHING THIS RUN/); + + assert.deepEqual(snapshotTree(home), before, 'plan must not mutate anything'); + + const result = applyRun({ plan, target, home, validateTarget: bypassCrossFsCheck }); + assert.equal(result.ok, false); + assert.match(result.error, /unresolved journal state/); + assert.match(result.error, new RegExp(escapeRe(strayFile))); + + assert.deepEqual(snapshotTree(home), before, 'apply must not mutate anything either'); + assert.equal(readdirSync(target).length, 0, 'apply must refuse before even attempting to copy anything into target'); + }); + + it('positive control: a healthy, journal-free run still selects the idle dir', () => { + const { home, src } = buildIdleFixture(); + const plan = planFor(home); + assert.ok(plan.selected.some(r => r.path === src)); + assert.equal(plan.globalJournalBlockReason, null); + }); + }); + + describe('GAP 2: a directory-fsync failure must never be silently swallowed', () => { + // Scoped to the SOURCE tree only, not model-tidy's own journal + // bookkeeping under home/.cache (a durability failure legitimately + // leaves journal-directory debris: a written-but-not-fsynced record, + // or an un-renamed temp file) and not the target dir (the copy step + // runs, and legitimately succeeds, BEFORE the swap step where the + // journal write under test happens — target ending up with a copy is + // expected, not a violation; what must never change is the source). + function snapshotSource(home) { + return snapshotTree(join(home, 'models')); + } + + it('EIO on the journal-directory fsync, on a SIMULATED Linux platform: apply aborts, source untouched (full tree identical), report names the error', () => { + const { home, src } = buildIdleFixture('model-tidy-r7-gap2a-home-'); + const target = tempDir('model-tidy-r7-gap2a-target-'); + const plan = planFor(home); + assert.ok(plan.selected.some(r => r.path === src)); + + const before = snapshotSource(home); + + // Within one writeJournalRecordSync call, fsyncSync is called on the + // temp-file fd first, then the directory fd second — fail every + // SECOND call (the directory fsync) and let the first (temp-file) + // succeed, so this test isolates the directory-fsync failure + // specifically. The very first journal write (step 'pending') + // happens before any source mutation, so its directory-fsync + // failure aborts before anything is touched. + let callCount = 0; + const result = applyRun({ + plan, + target, + home, + validateTarget: bypassCrossFsCheck, + platform: 'linux', + journalFsyncSync: () => { + callCount++; + if (callCount % 2 === 0) { + throw Object.assign(new Error('input/output error'), { code: 'EIO' }); + } + } + }); + + assert.equal(result.ok, false); + assert.equal(result.moved.length, 0); + assert.ok(result.errors.length >= 1); + assert.match(result.errors[0].error, /EIO/); + assert.match(result.errors[0].error, /journal directory/); + // The copy+verify step runs (and succeeds) BEFORE the swap step + // where this journal write happens, so target legitimately holds a + // copy — that's expected, not a violation. What must never change, + // durability failure or not, is the SOURCE. + assert.deepEqual(snapshotSource(home), before, 'the source must be completely untouched'); + assert.equal(lstatSync(src).isSymbolicLink(), false, 'the source must still be a real directory, never swapped for a symlink'); + }); + + it('ENOTSUP on the journal-directory fsync, on a non-Linux platform: proceeds with a "durability degraded" note', () => { + const { home, src } = buildIdleFixture('model-tidy-r7-gap2b-home-'); + const target = tempDir('model-tidy-r7-gap2b-target-'); + const plan = planFor(home); + + let callCount = 0; + const result = applyRun({ + plan, + target, + home, + validateTarget: bypassCrossFsCheck, + platform: 'darwin', // explicitly injected, never the real process.platform + journalFsyncSync: () => { + callCount++; + if (callCount % 2 === 0) { + throw Object.assign(new Error('operation not supported'), { code: 'ENOTSUP' }); + } + } + }); + + assert.equal(result.ok, true, JSON.stringify(result.errors)); + assert.equal(result.moved.length, 1); + assert.equal(lstatSync(src).isSymbolicLink(), true); + assert.ok(result.durabilityNotes.length > 0, 'a durability-degraded note must be reported, not silently dropped'); + assert.ok(result.durabilityNotes.every(n => n.includes('durability degraded: ENOTSUP'))); + }); + + it('EIO on the journal TEMP-FILE fsync aborts, on any platform', () => { + const { home, src } = buildIdleFixture('model-tidy-r7-gap2c-home-'); + const target = tempDir('model-tidy-r7-gap2c-target-'); + const plan = planFor(home); + + const before = snapshotSource(home); + + // Fail the FIRST fsyncSync call within each write (the temp file), + // regardless of platform — a temp-file fsync failure is never + // tolerated anywhere, unlike the directory fsync on non-Linux. + let callCount = 0; + const result = applyRun({ + plan, + target, + home, + validateTarget: bypassCrossFsCheck, + platform: 'darwin', + journalFsyncSync: () => { + callCount++; + if (callCount % 2 === 1) { + throw Object.assign(new Error('input/output error'), { code: 'EIO' }); + } + } + }); + + assert.equal(result.ok, false); + assert.equal(result.moved.length, 0); + assert.ok(result.errors.length >= 1); + assert.match(result.errors[0].error, /EIO/); + assert.match(result.errors[0].error, /temp file/); + assert.deepEqual(snapshotSource(home), before, 'the source must be completely untouched'); + }); + }); +}); From 1da9d2b72d00a8b3979134b09907d02e5f543bd9 Mon Sep 17 00:00:00 2001 From: Petrus Pennanen Date: Mon, 21 Sep 2026 11:17:51 +0200 Subject: [PATCH 9/9] Round 8: run the unresolved-journal preflight BEFORE automatic recovery, not after MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit codexmb's round-7 review found an ordering bug: applyRun called the mutating recoverInterruptedMoves BEFORE checking for remaining unreadable journal state. Reproduced with the provided probe: one valid afterStage interruption plus an empty unknown.json. apply correctly returned the unresolved-journal failure, but had already promoted the valid unit's source to a symlink and deleted its staged original by the time it did. No data loss (the promotion was itself verified), but it broke the promised global no-mutation guarantee — "refusing" is supposed to mean nothing happened, not "something happened, and by the way we're refusing." Fixed by reordering: the preflight (listJournalRecords, filtered to corrupt/unreadable entries) now runs FIRST, before recoverFn is ever called. recoverInterruptedMoves never touches an unreadable/corrupt record either way (it always reports and leaves those alone), so the set of files this check sees is identical whether it runs before or after recovery — only WHEN it's allowed to act on that information changed. If anything is unresolved, apply returns immediately with recovered: [] and the reason naming every unresolved file; recovery only runs once the preflight is clean. The explicit `recover` subcommand's behavior is intentionally unchanged: it still recovers every valid journaled unit in one run while reporting (never touching) any unreadable one alongside them — documented as a single sentence in docs/model-tidy.md: "apply refuses globally on any unreadable journal; recover heals what it can and reports the rest." New tests (4, all pass standalone and in the full suite): codexmb's probe verbatim, ported (mixed valid afterStage interruption + empty unknown.json — apply refuses, full tree byte-identical before/after including the valid unit's staging still present and its source not yet a symlink, recovered: []); the lone-corrupt case (no valid interrupted unit at all) still refuses; a positive control that a valid interruption with no unreadable records still recovers and proceeds normally; and recoverInterruptedMoves on the same mixed fixture still heals the valid unit while reporting the unreadable one, confirming recover's behavior is unchanged by this fix. Test results: 4/4 new tests in isolation, 60/60 full model-tidy suite, 719/719 full repo suite. Co-Authored-By: Claude Fable 5.1 --- docs/model-tidy.md | 18 ++++-- src/model-tidy.mjs | 35 ++++++----- test/model-tidy.test.mjs | 130 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 20 deletions(-) diff --git a/docs/model-tidy.md b/docs/model-tidy.md index 617511d..996fcc9 100644 --- a/docs/model-tidy.md +++ b/docs/model-tidy.md @@ -299,12 +299,18 @@ record's filename against it: `plan` refuses to select **anything** this run, reporting exactly why (`globalJournalBlockReason`, and `REFUSING TO SELECT ANYTHING THIS RUN` in the summary line) rather than guess. `apply` enforces the same rule - even more strictly: after its own recovery pass, if any unreadable or - stray journal file remains, it refuses to touch anything at all — - `unresolved journal state: ; run recover, or resolve by hand` — - before validating `--target`, before copying, before anything. `recover` - itself never guesses either: an unreadable record is reported and both - it and every path it might touch are left alone. + even more strictly, and — after a review round found the ordering itself + was a bug — checks it **first**: before its own automatic recovery pass, + before validating `--target`, before copying, before anything. If any + unreadable or stray journal file exists, `apply` refuses the *entire* + run — `unresolved journal state: ; run recover, or resolve by + hand` — without running recovery at all, so it never mutates so much as + one otherwise-valid interrupted unit while any journal file's content + can't be trusted. **`apply` refuses globally on any unreadable journal; + `recover` heals what it can and reports the rest** — the explicit + `recover` subcommand still recovers every valid journaled unit it finds + in the same run, reporting (never touching) any unreadable one + alongside them. ## Selection rules (in order, each with an explicit reason string) diff --git a/src/model-tidy.mjs b/src/model-tidy.mjs index 078f461..2f8fab5 100644 --- a/src/model-tidy.mjs +++ b/src/model-tidy.mjs @@ -1829,24 +1829,31 @@ export function applyRun(options) { const journalRenameSync = options.journalRenameSync; const journalPlatform = options.platform; + // GAP (round 8): this preflight MUST run before recoverFn below, not + // after. Recovery is itself a mutation (it promotes valid interrupted + // units to symlinks and deletes their staged originals), and the whole + // point of this check is a GLOBAL refusal to mutate anything at all + // while any journal file's content can't be trusted — including units + // that recovery would otherwise have happily fixed. Running it after + // recovery meant apply could report "unresolved journal state, refusing" + // while having already mutated a perfectly valid unit moments earlier. + // recoverInterruptedMoves never touches an unreadable/corrupt record + // either way (it always reports and leaves those alone), so the set of + // unreadable files this check sees is identical whether it runs before + // or after recovery — only WHEN it's allowed to act differs. + const preflightUnreadable = listJournalRecords(journalDir).filter(r => r.corrupt || !r.record); + if (preflightUnreadable.length > 0) { + const files = preflightUnreadable.map(r => r.file); + const reason = `unresolved journal state: ${files.join(', ')}; run recover, or resolve by hand`; + return { ok: false, error: reason, moved: [], errors: [{ error: reason }], recovered: [] }; + } + // Self-heal any interrupted swap from a previous crash before this run's // own pre-checks and copy/verify/swap loop — see recoverInterruptedMoves. - // Only reached here, in apply, never from planRun. + // Only reached here, in apply, never from planRun. Only reached at all + // once the preflight above confirms every journal file is trustworthy. const recovered = recoverFn(home, { journalDir, journalFsyncSync, journalRenameSync, platform: journalPlatform }); - // GAP 1 (round 6): recovery above deliberately leaves any unreadable or - // stray journal file alone rather than guess at it. apply must not - // proceed AT ALL while that's true — not just for whatever unit it - // might belong to, but globally, since an unreadable record's content - // can't tell us what it was protecting. This check runs before any - // mutation (before even --target validation). - const remainingUnreadable = listJournalRecords(journalDir).filter(r => r.corrupt || !r.record); - if (remainingUnreadable.length > 0) { - const files = remainingUnreadable.map(r => r.file); - const reason = `unresolved journal state: ${files.join(', ')}; run recover, or resolve by hand`; - return { ok: false, error: reason, moved: [], errors: [{ error: reason }], recovered }; - } - const validation = validateTargetFn(target, home); if (!validation.ok) { return { ok: false, error: validation.error, moved: [], errors: [{ error: validation.error }], recovered }; diff --git a/test/model-tidy.test.mjs b/test/model-tidy.test.mjs index 33ca5f2..384b707 100644 --- a/test/model-tidy.test.mjs +++ b/test/model-tidy.test.mjs @@ -1763,3 +1763,133 @@ describe('ROUND 7', () => { }); }); }); + +describe('ROUND 8: the unresolved-journal preflight must run BEFORE any automatic recovery, not after', () => { + function journalDirFor(home) { + return join(home, '.cache', 'ide-agent-kit', 'model-tidy-journal'); + } + function buildIdleFixture(prefix) { + const home = tempDir(prefix || 'model-tidy-r8-home-'); + const src = join(home, 'models', 'idle'); + mkdirSync(src, { recursive: true }); + writeFileSync(join(src, 'weights.gguf'), 'GOOD'); + utimesSync(join(src, 'weights.gguf'), new Date(0), new Date(0)); + return { home, src }; + } + function opts(home) { + return { + home, + listProcessUsers: () => ({ checked: true, users: [] }), + listDockerBindUsers: () => ({ available: true, inUse: false, containers: [] }), + diskFreeBytes: () => 0 + }; + } + function snapshotTree(root) { + const snap = {}; + function recurse(dir) { + for (const name of readdirSync(dir)) { + const full = join(dir, name); + const lst = lstatSync(full); + if (lst.isSymbolicLink()) { + snap[full] = { type: 'symlink', linkTarget: readlinkSync(full) }; + } else if (lst.isDirectory()) { + recurse(full); + } else if (lst.isFile()) { + snap[full] = { type: 'file', size: lst.size, sha256: createHash('sha256').update(readFileSync(full)).digest('hex') }; + } + } + } + recurse(root); + return snap; + } + /** Build the mixed fixture: one VALID interrupted unit (afterStage, so + * its staging exists and its source is currently missing) plus one + * completely unrelated, unreadable journal file (empty unknown.json). */ + function buildMixedFixture() { + const { home, src } = buildIdleFixture('model-tidy-r8-mixed-home-'); + const target = tempDir('model-tidy-r8-mixed-target-'); + const o = opts(home); + const interrupted = applyRun({ + home, target, plan: planRun(o), validateTarget: () => ({ ok: true }), + afterStage: () => { throw new Error('fixture interruption'); } + }); + assert.equal(interrupted.ok, false, 'sanity: the fixture interruption must have happened'); + const journalDir = journalDirFor(home); + writeFileSync(join(journalDir, 'unknown.json'), ''); + return { home, src, target, o }; + } + + it("codexmb's probe verbatim: apply on a mixed fixture (one valid afterStage interruption + one empty unknown.json) refuses with the unresolved-journal error, and the full tree is byte-identical before/after — the valid unit's staging is still present and its source is not yet a symlink", () => { + const { home, src, target, o } = buildMixedFixture(); + const staging = `${src}.tidy-moving`; + assert.equal(existsSync(src), false, 'sanity: the valid unit is mid-swap, source currently missing'); + assert.equal(existsSync(staging), true, 'sanity: its staged original is present'); + + const before = snapshotTree(home); + + const result = applyRun({ home, target, plan: planRun(o), validateTarget: () => ({ ok: true }) }); + + assert.equal(result.ok, false); + assert.match(result.error, /unresolved journal state/); + assert.deepEqual(result.recovered, [], 'apply must not have run any recovery at all — not even for the valid unit'); + + assert.deepEqual(snapshotTree(home), before, 'the ENTIRE tree, including the valid interrupted unit, must be byte-identical before and after — apply touched nothing'); + assert.equal(existsSync(src), false, 'the valid unit must still be mid-swap: source still missing'); + assert.equal(existsSync(staging), true, 'the valid unit\'s staged original must still be present, not deleted'); + }); + + it('the lone-corrupt case (no valid interrupted unit at all) still refuses', () => { + const { home } = buildIdleFixture('model-tidy-r8-lone-home-'); + const target = tempDir('model-tidy-r8-lone-target-'); + const journalDir = journalDirFor(home); + mkdirSync(journalDir, { recursive: true }); + writeFileSync(join(journalDir, 'unknown.json'), ''); + + const plan = planRun(opts(home)); + const result = applyRun({ home, target, plan, validateTarget: () => ({ ok: true }) }); + + assert.equal(result.ok, false); + assert.match(result.error, /unresolved journal state/); + assert.deepEqual(result.recovered, []); + }); + + it('positive control: a valid interruption with NO unreadable records still recovers and proceeds normally', () => { + const { home, src } = buildIdleFixture('model-tidy-r8-positive-home-'); + const target = tempDir('model-tidy-r8-positive-target-'); + const o = opts(home); + const interrupted = applyRun({ + home, target, plan: planRun(o), validateTarget: () => ({ ok: true }), + afterStage: () => { throw new Error('fixture interruption'); } + }); + assert.equal(interrupted.ok, false); + assert.equal(existsSync(src), false); + + // No corrupt/unreadable journal file this time — just re-run apply. + const result = applyRun({ home, target, plan: planRun(o), validateTarget: () => ({ ok: true }) }); + + assert.equal(result.ok, true, JSON.stringify(result.errors)); + assert.equal(result.recovered.length, 1); + assert.equal(result.recovered[0].action, 'completed-swap-and-cleaned'); + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), true); + }); + + it("recover (the explicit subcommand's underlying function) on the mixed fixture heals the valid unit and reports the unreadable one — recover's own behavior is unchanged by this fix", () => { + const { home, src } = buildMixedFixture(); + const staging = `${src}.tidy-moving`; + + const recovered = recoverInterruptedMoves(home); + + const validFinding = recovered.find(r => r.path === src); + assert.ok(validFinding, 'the valid unit must be reported'); + assert.equal(validFinding.action, 'completed-swap-and-cleaned'); + assert.equal(existsSync(src), true); + assert.equal(lstatSync(src).isSymbolicLink(), true); + assert.equal(existsSync(staging), false); + + const unreadableFinding = recovered.find(r => r.journalFile && r.journalFile.endsWith('unknown.json')); + assert.ok(unreadableFinding, 'the unreadable record must be reported too'); + assert.equal(unreadableFinding.action, 'left-alone'); + assert.match(unreadableFinding.note, /unreadable/); + }); +});