From d364df80f306017e80c8a686ea7d01e0c1e28e80 Mon Sep 17 00:00:00 2001 From: christso <3115311+christso@users.noreply.github.com> Date: Thu, 29 Oct 2026 00:49:00 +1100 Subject: [PATCH] perf(sync): reduce repeated parent cleanup during updates --- docs/reviews/performance-followup.md | 191 ++++ .../reviews/performance-followup.samples.json | 906 ++++++++++++++++++ scripts/benchmark-cli-sync.ts | 408 ++++++++ src/core/sync.ts | 30 +- tests/unit/core/sync-dedup.test.ts | 86 +- 5 files changed, 1616 insertions(+), 5 deletions(-) create mode 100644 docs/reviews/performance-followup.md create mode 100644 docs/reviews/performance-followup.samples.json create mode 100644 scripts/benchmark-cli-sync.ts diff --git a/docs/reviews/performance-followup.md b/docs/reviews/performance-followup.md new file mode 100644 index 00000000..1f179daf --- /dev/null +++ b/docs/reviews/performance-followup.md @@ -0,0 +1,191 @@ +# CLI performance follow-up + +Baseline: `e331f44` (1.21.4), including #594. Candidate: the cleanup change +in `src/core/sync.ts` on `perf/remaining-cli-performance`. #594's unchanged +remote checks and avoided pulls are baseline behavior, not improvements from +this change. No TUI labels, progress, command contracts, or refresh decisions change. + +## Finding and change + +An initial built-CLI probe measured about 230 ms for help/version, 250 ms for +empty updates, and 1.4–1.5 seconds for repeated 1,000-skill updates. Existing +`ALLAGENTS_DEBUG=timing` output attributed 913 ms (57.5%) of a separate +1.59-second sync to selective purge and 505 ms to copying. A syscall trace +showed 3,001 `rmdir`, 27,031 `statx`, and 25,021 `readlink` calls. Tracing adds +substantial overhead; these counts and phase attribution are diagnostic, +not the final timing comparison. + +`selectivePurgeWorkspace` previously called `cleanupEmptyParents` after every +tracked deletion. Siblings repeatedly inspected and attempted to remove the +same nonempty parent. The candidate defers cleanup until that client's deletions +finish, then cleans each distinct parent once. Every deletion and cleanup retains +its existing path and symlink guards. Overlapping tracked file/directory entries +keep immediate cleanup to preserve deletion reporting. No persistent cache or +new dependency is introduced. + +Measured bundle SHA-256 values (also emitted by the harness): + +- Baseline: `47bf4f9e46fe11eab93cc4b43d95c079720192ccc59eade86dfd8716ea27f71a` +- Candidate: `250ecc1005058e56080ab2b62602d6593ce8a72c8d2adc71b0cf7ec58ec253c9` + +A CPU profile of `--help` showed time in bundle parsing and module initialization. +Startup remains unchanged; splitting/lazily loading the command graph was not +attempted in this bounded investigation. Copying and actual tracked deletion also +remain substantial costs. + +## Paired measurements + +Each scenario uses 3 interleaved warmups per version, then +15 interleaved timed baseline/candidate pairs, alternating order. Values are +whole-process wall time; p95 uses the nearest-rank sample. Fixtures have isolated +HOME/project directories, one Codex file-install client, 0/100/1,000 skills, and +user-owned sentinel files. Remote cases use a local bare Git remote redirected +through an isolated Git config. There is no injected latency or real-network +performance claim. Filesystem caches are warm; cold install is setup, not timed. +Runtime: Node v26.7.0, Bun 1.4.3, Linux 7.0.0-38-generic, Intel i7-6700T. +With 15 samples, nearest-rank p95 is the maximum observed sample, not a stable +estimate of production tail latency. + +| Scenario | Skills | Baseline p50 / p95 (ms) | Candidate p50 / p95 (ms) | +| --- | ---: | ---: | ---: | +| version | 0 | 221 / 238 | 213 / 239 | +| help | 0 | 218 / 235 | 219 / 270 | +| local-project | 0 | 247 / 292 | 243 / 291 | +| local-user | 0 | 246 / 275 | 238 / 254 | +| local-project | 100 | 410 / 451 | 386 / 406 | +| local-user | 100 | 412 / 507 | 387 / 442 | +| local-project | 1000 | 1490 / 1647 | 1269 / 1377 | +| local-user | 1000 | 1448 / 1548 | 1268 / 1388 | +| remote-project | 1000 | 1609 / 1706 | 1373 / 1476 | +| remote-user | 1000 | 1620 / 1818 | 1352 / 1462 | +| offline-project | 1000 | 1493 / 1558 | 1294 / 1349 | +| offline-user | 1000 | 1506 / 1634 | 1288 / 1361 | + +The 1,000-skill median reductions were 12.5–16.5%. Startup and +empty-update differences are treated as noise; no startup speedup is claimed. +All 360 timed invocations matched output, destination/state and Git work. All +450 invocations (including initial materialization, warmups and separate traces) +passed protected-source/config/sentinel guards. [Raw final samples and traces](performance-followup.samples.json) +include bundle hashes and +individual samples for reproduction. + +| Local 1,000-skill project syscall | Baseline | Candidate | +| --- | ---: | ---: | +| rmdir | 3000 | 2001 | +| readlink | 31018 | 21028 | +| statx | 28032 | 26071 | +| access | 4020 | 3021 | + +Git work was identical in both builds: local sources started no Git processes; +unchanged online remote updates started 7 Git processes (one `ls-remote`, two +`rev-parse`, one each `config`, `status`, `symbolic-ref`, plus the local +transport), with zero clone/pull/fetch/merge/checkout/reset. Offline cached +updates performed one `rev-parse` and no remote operation. These existing remote +skips belong to #594 and earlier work. + +Each timed sample must match baseline/candidate stdout, stderr, normalized +sync state (only `lastSync` removed), config bytes, `.codex` and canonical +`.agents` file contents/modes, root `AGENTS.md` when present, and Git operation counts. +The snapshot walks existing directories and records symlink targets using +`lstat`, including dangling links; it does not follow links outside those roots. +The benchmark fixture does not introduce dangling links or generate root +`AGENTS.md`; containment/symlink behavior is checked separately. Git work comes +from `GIT_TRACE2_EVENT`. Optional +`strace -f -c -e trace=%file` runs separately from the timed samples. + +Before materialization, the harness captures expected source resource bytes and +modes, both workspace-config bytes/modes, and both explicit user sentinel +bytes/modes. Every initial, warmup, timed, and trace invocation rechecks these +immutable expectations after the timing interval. Cached remote resources must +also match the original source's resource bytes/modes. These guards prevent an +early candidate mutation from becoming the shared fixture's new baseline. + +A one-shot wrapper that deletes the HOME sentinel on its first invocation was +rejected during the first warmup with +`Protected source/config/sentinel bytes or modes changed` (exit 1). The normal +protected local/remote/offline smoke passed. Final measurements use these guards; +earlier exploratory matrices are not used for the reported performance comparison. + +## Reproduction and validation + +The baseline bundle for this run was copied outside mutable `dist` before +editing. To recreate it from the candidate checkout without changing branches: + +```sh +baseline_source=$(mktemp -d /tmp/allagents-perf-base-XXXXXX) +baseline_artifacts=$(mktemp -d /tmp/allagents-perf-bundle-XXXXXX) +git clone --shared --no-checkout . "$baseline_source" +git -C "$baseline_source" checkout --detach e331f44 +(cd "$baseline_source" && bun install --frozen-lockfile && bun run build) +cp -a "$baseline_source/dist" "$baseline_artifacts/baseline" +bun run build +bun scripts/benchmark-cli-sync.ts \ + "$baseline_artifacts/baseline/index.js" dist/index.js \ + --samples=15 --warmups=3 --sizes=0,100,1000 --trace \ + > /tmp/allagents-perf-final.json +rm -rf "$baseline_source" "$baseline_artifacts" +``` + +The harness deletes its temporary fixtures on completion. `--trace` requires +Linux `strace`; omit it for timing/output/state/Git checks on other systems. + +To reproduce the first-warmup contamination check, wrap the candidate with: + +```sh +#!/bin/sh +if [ ! -f "$ALLAGENTS_TEST_HOME/.perf-fault-fired" ]; then + rm "$ALLAGENTS_TEST_HOME/.codex/skills/user-owned/keep.txt" + touch "$ALLAGENTS_TEST_HOME/.perf-fault-fired" +fi +exec "$ALLAGENTS_PERF_REAL_CLI" "$@" +``` + +Make the wrapper executable and run the benchmark with that candidate, +`ALLAGENTS_PERF_REAL_CLI="$PWD/dist/index.js"`, and +`--samples=1 --warmups=1 --sizes=0`. Expected: exit 1 with the protected-state +error above, before any timed result is emitted. + +Focused checks: + +```sh +bun test tests/unit/core/sync-dedup.test.ts tests/unit/core/sync.test.ts \ + tests/unit/core/sync-user.test.ts tests/unit/core/client-context.test.ts \ + tests/unit/core/sync-symlink.test.ts tests/unit/core/omp-artifacts.test.ts +bun run typecheck +bun run lint +bun run build +``` + +Result: 95 focused tests passed (803 assertions); source typecheck, explicit +benchmark-script typecheck, lint, and build passed. Lint reports the existing +17 warnings. The benchmark typecheck command was: + +```sh +bunx tsc --ignoreConfig --noEmit --strict --noUncheckedIndexedAccess \ + --target es2022 --module esnext --moduleResolution bundler --types bun \ + --skipLibCheck scripts/benchmark-cli-sync.ts +``` + +Built manual checks used the same isolated baseline and candidate to install, +repeat-sync, edit a skill, remove a source skill, and remove the plugin for both +project and user scope. JSON and redirected human output matched across all +eight normalized outputs. User-owned sentinel bytes survived each stage; source +edits reached the destination, deleted skills disappeared, and final sync state +released owned files. Regression coverage includes nested parent cleanup, +user files, write-root preservation, tracked final symlinks, rejected symlink +parent traversal, external Pi roots, and overlapping state entries. The overlap +test failed on the first candidate and passes with the conservative fallback. + +A separate built-candidate check ran JSON `update --scope project --offline +--dry-run` and `status` after materialization in an isolated HOME/project. A +whole-fixture fingerprint confirmed unchanged paths, bytes, modes, mtimes, and +symlink targets after each read-only invocation. + +Final inline and independent peer reviews found no actionable issue in the +production change. Measurement-review findings were resolved by adding canonical +resource snapshots, immutable fixture guards on every invocation, and first-warmup +fault injection. + +Limits: these measurements do not establish live-network, cold-disk-cache, +failed-remote, cancellation-latency, or multi-profile improvements. Existing +remote health/no-op decisions and refresh failure handling are unchanged. diff --git a/docs/reviews/performance-followup.samples.json b/docs/reviews/performance-followup.samples.json new file mode 100644 index 00000000..5f0aaea2 --- /dev/null +++ b/docs/reviews/performance-followup.samples.json @@ -0,0 +1,906 @@ +{ + "baseline": "/tmp/allagents-perf-e331f44/baseline/index.js", + "candidate": "/home/christso/projects/allagentsdev/allagents.worktrees/perf-remaining-cli-performance/dist/index.js", + "bundles": { + "baselineSha256": "47bf4f9e46fe11eab93cc4b43d95c079720192ccc59eade86dfd8716ea27f71a", + "candidateSha256": "250ecc1005058e56080ab2b62602d6593ce8a72c8d2adc71b0cf7ec58ec253c9" + }, + "samples": 15, + "warmups": 3, + "environment": { + "bun": "1.4.3", + "node": "v26.7.0", + "os": "7.0.0-38-generic", + "cpu": "Intel(R) Core(TM) i7-6700T CPU @ 2.80GHz" + }, + "profile": "local filesystem and local bare Git remotes; no injected delay", + "rows": [ + { + "name": "version", + "size": 0, + "args": [ + "--version" + ], + "baseline": { + "count": 15, + "p50Ms": 220.74008700000013, + "p95Ms": 237.5225230000001, + "samplesMs": [ + 220.41287899999998, + 219.86596699999973, + 237.5225230000001, + 221.51857399999972, + 220.74008700000013, + 213.52377600000045, + 211.0821850000002, + 224.8197680000003, + 236.083842, + 228.06953599999997, + 218.03548299999966, + 212.0152849999995, + 224.18860700000005, + 218.29420300000038, + 236.863026 + ] + }, + "candidate": { + "count": 15, + "p50Ms": 213.19913499999984, + "p95Ms": 239.14369299999998, + "samplesMs": [ + 221.70381199999997, + 209.91407400000003, + 220.42203699999982, + 213.19913499999984, + 239.14369299999998, + 217.3268509999998, + 214.9909859999998, + 207.57876699999997, + 208.003471, + 216.35205900000074, + 206.6368339999999, + 214.30842100000064, + 206.85127399999965, + 209.00847900000008, + 210.04485900000054 + ] + }, + "gitOperations": { + "processes": 0, + "ls-remote": 0, + "clone": 0, + "pull": 0, + "fetch": 0, + "merge": 0, + "checkout": 0, + "reset": 0, + "rev-parse": 0, + "remote": 0, + "status": 0, + "config": 0, + "symbolic-ref": 0 + }, + "compatible": true, + "protectedStateSha256": "529f97f0692188560246107ab3ac1bad11cc322e3fa876c25d37b6d38eee6532", + "sourceResourcesSha256": "7f6f3522a64d5fc1eb33f2aa27fe2c8073f29a99122e9458bd79fea560cb664e", + "fileOperations": {} + }, + { + "name": "help", + "size": 0, + "args": [ + "--help" + ], + "baseline": { + "count": 15, + "p50Ms": 218.0291879999986, + "p95Ms": 235.10321599999952, + "samplesMs": [ + 218.0291879999986, + 218.48567100000037, + 217.27520600000025, + 215.9512479999994, + 211.31902900000023, + 217.11966399999983, + 215.5284929999998, + 211.501631000001, + 218.94376699999884, + 223.09044799999901, + 235.10321599999952, + 223.02705799999967, + 218.35825300000033, + 211.38438099999985, + 226.04276299999947 + ] + }, + "candidate": { + "count": 15, + "p50Ms": 218.5982700000004, + "p95Ms": 269.970722, + "samplesMs": [ + 219.58157799999935, + 218.5982700000004, + 269.970722, + 228.15175799999997, + 217.76089200000024, + 217.41354000000138, + 224.26668099999915, + 229.38699199999974, + 211.56588399999964, + 216.14569400000073, + 209.6047269999999, + 212.5987759999989, + 222.47846000000027, + 220.66166899999916, + 214.79549800000132 + ] + }, + "gitOperations": { + "processes": 0, + "ls-remote": 0, + "clone": 0, + "pull": 0, + "fetch": 0, + "merge": 0, + "checkout": 0, + "reset": 0, + "rev-parse": 0, + "remote": 0, + "status": 0, + "config": 0, + "symbolic-ref": 0 + }, + "compatible": true, + "protectedStateSha256": "529f97f0692188560246107ab3ac1bad11cc322e3fa876c25d37b6d38eee6532", + "sourceResourcesSha256": "7f6f3522a64d5fc1eb33f2aa27fe2c8073f29a99122e9458bd79fea560cb664e", + "fileOperations": {} + }, + { + "name": "local-project", + "size": 0, + "args": [ + "--json", + "update", + "--scope", + "project", + "--offline" + ], + "baseline": { + "count": 15, + "p50Ms": 247.0878389999998, + "p95Ms": 292.3489460000019, + "samplesMs": [ + 247.0878389999998, + 248.08232700000008, + 244.35763000000225, + 246.8714919999984, + 236.37614699999904, + 261.3778349999993, + 244.37519800000155, + 258.20744099999865, + 246.47221600000194, + 292.3489460000019, + 250.5345649999981, + 277.9102830000011, + 239.04961299999923, + 263.9288439999982, + 244.53412300000127 + ] + }, + "candidate": { + "count": 15, + "p50Ms": 242.86091999999917, + "p95Ms": 290.9691120000025, + "samplesMs": [ + 249.08816900000238, + 241.33207799999946, + 232.53199400000085, + 234.22007099999973, + 245.09779999999955, + 248.65841, + 236.1991710000002, + 235.21819099999993, + 236.46042400000078, + 242.86091999999917, + 290.9691120000025, + 277.3749530000023, + 249.9268149999989, + 248.48294700000042, + 242.07784900000115 + ] + }, + "gitOperations": { + "processes": 0, + "ls-remote": 0, + "clone": 0, + "pull": 0, + "fetch": 0, + "merge": 0, + "checkout": 0, + "reset": 0, + "rev-parse": 0, + "remote": 0, + "status": 0, + "config": 0, + "symbolic-ref": 0 + }, + "compatible": true, + "protectedStateSha256": "529f97f0692188560246107ab3ac1bad11cc322e3fa876c25d37b6d38eee6532", + "sourceResourcesSha256": "7f6f3522a64d5fc1eb33f2aa27fe2c8073f29a99122e9458bd79fea560cb664e", + "fileOperations": {} + }, + { + "name": "local-user", + "size": 0, + "args": [ + "--json", + "update", + "--scope", + "user", + "--offline" + ], + "baseline": { + "count": 15, + "p50Ms": 246.33623600000283, + "p95Ms": 275.47993300000235, + "samplesMs": [ + 241.20998500000132, + 243.75213399999848, + 245.20036800000162, + 250.89588899999944, + 237.9263490000012, + 251.75971500000014, + 243.48704400000133, + 251.9351809999971, + 249.9030890000031, + 246.33623600000283, + 245.2333120000003, + 275.47993300000235, + 249.7109980000023, + 252.6094479999956, + 242.26217099999485 + ] + }, + "candidate": { + "count": 15, + "p50Ms": 237.72983699999895, + "p95Ms": 253.81224400000065, + "samplesMs": [ + 232.9629449999993, + 252.64519400000063, + 235.35010899999907, + 246.02479500000118, + 231.83905199999936, + 237.71058099999937, + 246.127325999998, + 253.81224400000065, + 236.8368100000007, + 236.91957300000286, + 239.64185400000133, + 238.88177599999835, + 240.59254900000087, + 237.72983699999895, + 234.12085500000103 + ] + }, + "gitOperations": { + "processes": 0, + "ls-remote": 0, + "clone": 0, + "pull": 0, + "fetch": 0, + "merge": 0, + "checkout": 0, + "reset": 0, + "rev-parse": 0, + "remote": 0, + "status": 0, + "config": 0, + "symbolic-ref": 0 + }, + "compatible": true, + "protectedStateSha256": "529f97f0692188560246107ab3ac1bad11cc322e3fa876c25d37b6d38eee6532", + "sourceResourcesSha256": "7f6f3522a64d5fc1eb33f2aa27fe2c8073f29a99122e9458bd79fea560cb664e", + "fileOperations": {} + }, + { + "name": "local-project", + "size": 100, + "args": [ + "--json", + "update", + "--scope", + "project", + "--offline" + ], + "baseline": { + "count": 15, + "p50Ms": 409.7766249999986, + "p95Ms": 450.99819600000046, + "samplesMs": [ + 419.5695960000012, + 409.7766249999986, + 448.2479669999957, + 434.45880799999577, + 398.4881269999969, + 450.99819600000046, + 425.5650470000037, + 402.09010200000193, + 399.0626540000012, + 392.11304600000585, + 417.32092099999863, + 417.2449620000043, + 397.6193179999973, + 401.73820599999453, + 398.87101399999665 + ] + }, + "candidate": { + "count": 15, + "p50Ms": 386.344047999999, + "p95Ms": 406.17884400000185, + "samplesMs": [ + 390.02351700000145, + 390.1246849999952, + 386.344047999999, + 374.6247730000032, + 380.7144550000012, + 385.9803329999995, + 387.8047379999989, + 389.83963100000256, + 377.22090000000026, + 377.9799070000008, + 382.55973500000255, + 388.87873799999943, + 369.71327400000155, + 393.8677069999976, + 406.17884400000185 + ] + }, + "gitOperations": { + "processes": 0, + "ls-remote": 0, + "clone": 0, + "pull": 0, + "fetch": 0, + "merge": 0, + "checkout": 0, + "reset": 0, + "rev-parse": 0, + "remote": 0, + "status": 0, + "config": 0, + "symbolic-ref": 0 + }, + "compatible": true, + "protectedStateSha256": "b5d28f734db6c88e53129fda35f057a67e135c42831ad3f86debcf324a5b509c", + "sourceResourcesSha256": "6e99b9d4881232795daef2b844d61100fa11b96fe3eabd2328dc27491709604b", + "fileOperations": {} + }, + { + "name": "local-user", + "size": 100, + "args": [ + "--json", + "update", + "--scope", + "user", + "--offline" + ], + "baseline": { + "count": 15, + "p50Ms": 412.3988790000003, + "p95Ms": 507.46993900000234, + "samplesMs": [ + 415.61962700000004, + 456.00854300000356, + 507.46993900000234, + 397.05410100000154, + 416.2473159999936, + 382.2078610000026, + 376.3102860000072, + 407.847692000003, + 412.3988790000003, + 394.7229059999954, + 418.96263900000486, + 393.2500319999963, + 418.113860999998, + 431.08152800000244, + 406.9206550000017 + ] + }, + "candidate": { + "count": 15, + "p50Ms": 387.2759670000014, + "p95Ms": 442.2745189999987, + "samplesMs": [ + 359.7037780000028, + 442.2745189999987, + 422.4687799999956, + 369.08556099999987, + 387.2759670000014, + 389.2544790000029, + 389.81524999999965, + 389.1621249999953, + 368.49948700000095, + 369.83113500000036, + 391.81477299999824, + 401.31338700000197, + 387.0102210000041, + 378.59858100000565, + 375.8920769999968 + ] + }, + "gitOperations": { + "processes": 0, + "ls-remote": 0, + "clone": 0, + "pull": 0, + "fetch": 0, + "merge": 0, + "checkout": 0, + "reset": 0, + "rev-parse": 0, + "remote": 0, + "status": 0, + "config": 0, + "symbolic-ref": 0 + }, + "compatible": true, + "protectedStateSha256": "b5d28f734db6c88e53129fda35f057a67e135c42831ad3f86debcf324a5b509c", + "sourceResourcesSha256": "6e99b9d4881232795daef2b844d61100fa11b96fe3eabd2328dc27491709604b", + "fileOperations": {} + }, + { + "name": "local-project", + "size": 1000, + "args": [ + "--json", + "update", + "--scope", + "project", + "--offline" + ], + "baseline": { + "count": 15, + "p50Ms": 1489.6614529999933, + "p95Ms": 1647.2019710000022, + "samplesMs": [ + 1505.720696999997, + 1448.7365130000107, + 1579.6300679999986, + 1489.10966799999, + 1517.8039040000003, + 1523.2474450000009, + 1467.8967039999989, + 1509.670398000002, + 1489.6614529999933, + 1510.5301719999989, + 1411.7368780000106, + 1433.3723909999972, + 1429.3180919999868, + 1487.1935079999967, + 1647.2019710000022 + ] + }, + "candidate": { + "count": 15, + "p50Ms": 1268.664374, + "p95Ms": 1377.218651999996, + "samplesMs": [ + 1239.9636649999884, + 1260.1463280000025, + 1311.7808920000098, + 1332.5903610000096, + 1377.218651999996, + 1273.9870079999964, + 1296.1009149999882, + 1198.2883690000017, + 1251.4748980000004, + 1247.2356109999964, + 1268.664374, + 1355.8520070000086, + 1221.0235909999901, + 1303.5792089999886, + 1243.82293200001 + ] + }, + "gitOperations": { + "processes": 0, + "ls-remote": 0, + "clone": 0, + "pull": 0, + "fetch": 0, + "merge": 0, + "checkout": 0, + "reset": 0, + "rev-parse": 0, + "remote": 0, + "status": 0, + "config": 0, + "symbolic-ref": 0 + }, + "compatible": true, + "protectedStateSha256": "838f25830bd563bb88f3d3ad460ef278c29cbf92143f290675d6631d659792dd", + "sourceResourcesSha256": "657679ed02250553ff9fe6a461836e2339a5fe685743eb808f3b495bc25d92fe", + "fileOperations": { + "baseline": "% time seconds usecs/call calls errors syscall\n------ ----------- ----------- --------- --------- ----------------\n 40.16 0.225040 8 28032 3007 statx\n 29.42 0.164856 5 31018 31016 readlink\n 13.35 0.074806 18 4050 6 openat\n 6.35 0.035565 17 2000 chmod\n 2.82 0.015776 15 1002 2 mkdir\n 2.65 0.014877 14 1000 utimensat\n 2.58 0.014447 4 3000 2000 rmdir\n 1.36 0.007624 7 1000 unlink\n 1.23 0.006908 1 4020 11 access\n 0.08 0.000440 88 5 3 execve\n 0.00 0.000008 4 2 2 statfs\n 0.00 0.000000 0 2 2 stat\n 0.00 0.000000 0 1 getcwd\n 0.00 0.000000 0 1 rename\n------ ----------- ----------- --------- --------- ----------------\n100.00 0.560347 7 75133 36049 total\n", + "candidate": "% time seconds usecs/call calls errors syscall\n------ ----------- ----------- --------- --------- ----------------\n 41.24 0.205723 7 26071 3010 statx\n 25.15 0.125446 5 21028 21026 readlink\n 13.77 0.068679 16 4055 4 openat\n 10.18 0.050773 25 2000 chmod\n 2.83 0.014130 14 1002 2 mkdir\n 2.36 0.011777 11 1000 utimensat\n 1.91 0.009533 4 2001 1001 rmdir\n 1.36 0.006770 6 1000 unlink\n 1.10 0.005499 1 3021 11 access\n 0.09 0.000459 91 5 3 execve\n 0.00 0.000009 4 2 2 statfs\n 0.00 0.000002 2 1 getcwd\n 0.00 0.000000 0 2 2 stat\n 0.00 0.000000 0 1 rename\n------ ----------- ----------- --------- --------- ----------------\n100.00 0.498800 8 61189 25061 total\n" + } + }, + { + "name": "local-user", + "size": 1000, + "args": [ + "--json", + "update", + "--scope", + "user", + "--offline" + ], + "baseline": { + "count": 15, + "p50Ms": 1448.4345620000095, + "p95Ms": 1547.9610800000082, + "samplesMs": [ + 1481.8722409999755, + 1444.6055500000075, + 1389.6202460000059, + 1465.9666159999906, + 1406.9536040000094, + 1399.279698999977, + 1547.9610800000082, + 1427.4291019999946, + 1448.4345620000095, + 1417.7453509999905, + 1487.8351820000098, + 1390.6970189999847, + 1457.3569279999938, + 1483.9799829999974, + 1500.1215030000021 + ] + }, + "candidate": { + "count": 15, + "p50Ms": 1268.061332000012, + "p95Ms": 1388.2408479999867, + "samplesMs": [ + 1123.20883399999, + 1154.3971350000065, + 1308.2481210000115, + 1388.2408479999867, + 1345.3546630000055, + 1223.3651430000027, + 1279.604938000004, + 1248.6993540000112, + 1273.7819629999867, + 1364.3339640000195, + 1268.061332000012, + 1304.1352179999813, + 1178.7399020000012, + 1247.1627009999938, + 1239.3272700000089 + ] + }, + "gitOperations": { + "processes": 0, + "ls-remote": 0, + "clone": 0, + "pull": 0, + "fetch": 0, + "merge": 0, + "checkout": 0, + "reset": 0, + "rev-parse": 0, + "remote": 0, + "status": 0, + "config": 0, + "symbolic-ref": 0 + }, + "compatible": true, + "protectedStateSha256": "838f25830bd563bb88f3d3ad460ef278c29cbf92143f290675d6631d659792dd", + "sourceResourcesSha256": "657679ed02250553ff9fe6a461836e2339a5fe685743eb808f3b495bc25d92fe", + "fileOperations": {} + }, + { + "name": "remote-project", + "size": 1000, + "args": [ + "--json", + "update", + "--scope", + "project" + ], + "baseline": { + "count": 15, + "p50Ms": 1609.0202139999892, + "p95Ms": 1705.6895999999833, + "samplesMs": [ + 1702.6072510000085, + 1705.6895999999833, + 1609.0202139999892, + 1625.5605630000064, + 1703.7289410000085, + 1605.3139440000232, + 1485.1047700000054, + 1491.6887329999881, + 1586.7698479999963, + 1525.6122380000015, + 1693.980268000014, + 1646.5755960000097, + 1561.8974480000034, + 1545.8150339999993, + 1691.268276000017 + ] + }, + "candidate": { + "count": 15, + "p50Ms": 1373.0047919999924, + "p95Ms": 1476.0766009999788, + "samplesMs": [ + 1435.6673449999944, + 1416.44976399999, + 1319.2252050000243, + 1421.1956519999949, + 1470.5041449999844, + 1353.6341229999962, + 1373.0047919999924, + 1384.7370440000086, + 1318.5982089999889, + 1365.114578000008, + 1305.4428859999753, + 1416.5977660000208, + 1476.0766009999788, + 1236.1709869999904, + 1359.3827470000251 + ] + }, + "gitOperations": { + "processes": 7, + "ls-remote": 1, + "clone": 0, + "pull": 0, + "fetch": 0, + "merge": 0, + "checkout": 0, + "reset": 0, + "rev-parse": 2, + "remote": 0, + "status": 1, + "config": 1, + "symbolic-ref": 1 + }, + "compatible": true, + "protectedStateSha256": "1ac8d8f67f474189870693d21651fc6a443bfa250ec1aa3da4a189f4f4477776", + "sourceResourcesSha256": "657679ed02250553ff9fe6a461836e2339a5fe685743eb808f3b495bc25d92fe", + "fileOperations": { + "baseline": "% time seconds usecs/call calls errors syscall\n------ ----------- ----------- --------- --------- ----------------\n 36.94 0.204593 7 28034 3007 statx\n 27.35 0.151450 4 31019 31017 readlink\n 13.37 0.074056 13 5301 47 openat\n 7.59 0.042026 21 2000 chmod\n 4.20 0.023262 23 1002 2 mkdir\n 2.61 0.014450 14 1000 utimensat\n 2.49 0.013798 4 3000 2000 rmdir\n 1.87 0.010375 4 2216 72 newfstatat\n 1.42 0.007848 1 4079 24 access\n 1.34 0.007423 7 1001 unlink\n 0.79 0.004387 141 31 21 execve\n 0.01 0.000065 2 23 getcwd\n 0.01 0.000037 37 1 rename\n 0.00 0.000009 4 2 2 statfs\n 0.00 0.000008 1 8 chdir\n 0.00 0.000000 0 2 2 stat\n------ ----------- ----------- --------- --------- ----------------\n100.00 0.553787 7 78719 36194 total\n", + "candidate": "% time seconds usecs/call calls errors syscall\n------ ----------- ----------- --------- --------- ----------------\n 38.55 0.216376 8 26073 3010 statx\n 25.87 0.145211 6 21029 21027 readlink\n 17.54 0.098443 18 5306 45 openat\n 5.98 0.033564 16 2000 chmod\n 3.02 0.016958 16 1002 2 mkdir\n 2.83 0.015873 15 1000 utimensat\n 1.89 0.010622 5 2001 1001 rmdir\n 1.54 0.008619 3 2216 72 newfstatat\n 1.35 0.007589 7 1001 unlink\n 0.78 0.004379 1 3080 24 access\n 0.64 0.003584 115 31 21 execve\n 0.01 0.000058 2 23 getcwd\n 0.00 0.000012 1 8 chdir\n 0.00 0.000011 5 2 2 statfs\n 0.00 0.000005 2 2 2 stat\n 0.00 0.000000 0 1 rename\n------ ----------- ----------- --------- --------- ----------------\n100.00 0.561304 8 64775 25206 total\n" + } + }, + { + "name": "remote-user", + "size": 1000, + "args": [ + "--json", + "update", + "--scope", + "user" + ], + "baseline": { + "count": 15, + "p50Ms": 1620.1408159999992, + "p95Ms": 1817.7452969999867, + "samplesMs": [ + 1526.7394750000094, + 1620.1408159999992, + 1640.9863560000085, + 1569.300111999968, + 1564.4577790000476, + 1614.7442409999785, + 1665.7619290000293, + 1666.3019940000377, + 1817.7452969999867, + 1494.4045240000123, + 1621.897759000014, + 1653.3812380000018, + 1628.8349419999868, + 1602.9962940000114, + 1461.8402789999964 + ] + }, + "candidate": { + "count": 15, + "p50Ms": 1352.3557400000282, + "p95Ms": 1462.4618519999785, + "samplesMs": [ + 1457.6618510000117, + 1343.2860219999857, + 1321.93690900004, + 1415.5531370000099, + 1274.8904099999927, + 1417.5873680000077, + 1462.4618519999785, + 1323.0399070000276, + 1414.2936540000373, + 1336.737649000017, + 1352.3557400000282, + 1345.3674050000263, + 1292.8800059999921, + 1382.4735130000045, + 1444.3861230000039 + ] + }, + "gitOperations": { + "processes": 7, + "ls-remote": 1, + "clone": 0, + "pull": 0, + "fetch": 0, + "merge": 0, + "checkout": 0, + "reset": 0, + "rev-parse": 2, + "remote": 0, + "status": 1, + "config": 1, + "symbolic-ref": 1 + }, + "compatible": true, + "protectedStateSha256": "1ac8d8f67f474189870693d21651fc6a443bfa250ec1aa3da4a189f4f4477776", + "sourceResourcesSha256": "657679ed02250553ff9fe6a461836e2339a5fe685743eb808f3b495bc25d92fe", + "fileOperations": {} + }, + { + "name": "offline-project", + "size": 1000, + "args": [ + "--json", + "update", + "--scope", + "project", + "--offline" + ], + "baseline": { + "count": 15, + "p50Ms": 1493.1307819999638, + "p95Ms": 1558.4486840000027, + "samplesMs": [ + 1493.1307819999638, + 1551.0881139999838, + 1464.7942930000136, + 1460.9590580000076, + 1497.7590559999808, + 1426.238858000026, + 1498.4200259999488, + 1497.3119980000192, + 1458.6280249999836, + 1416.1413869999815, + 1443.4056850000052, + 1538.8462590000127, + 1492.9001530000241, + 1545.2829720000154, + 1558.4486840000027 + ] + }, + "candidate": { + "count": 15, + "p50Ms": 1293.9743460000027, + "p95Ms": 1348.5875909999595, + "samplesMs": [ + 1293.9743460000027, + 1306.5714750000043, + 1215.5335429999977, + 1317.0463379999856, + 1235.9483140000375, + 1271.7581529999734, + 1208.5519600000116, + 1333.6742830000003, + 1231.7793219999876, + 1278.6492550000548, + 1294.9694280000404, + 1326.6609980000067, + 1309.1398889999837, + 1278.57414099999, + 1348.5875909999595 + ] + }, + "gitOperations": { + "processes": 1, + "ls-remote": 0, + "clone": 0, + "pull": 0, + "fetch": 0, + "merge": 0, + "checkout": 0, + "reset": 0, + "rev-parse": 1, + "remote": 0, + "status": 0, + "config": 0, + "symbolic-ref": 0 + }, + "compatible": true, + "protectedStateSha256": "1ac8d8f67f474189870693d21651fc6a443bfa250ec1aa3da4a189f4f4477776", + "sourceResourcesSha256": "657679ed02250553ff9fe6a461836e2339a5fe685743eb808f3b495bc25d92fe", + "fileOperations": { + "baseline": "% time seconds usecs/call calls errors syscall\n------ ----------- ----------- --------- --------- ----------------\n 37.09 0.206645 7 28033 3007 statx\n 27.67 0.154163 4 31018 31016 readlink\n 18.45 0.102780 25 4081 10 openat\n 4.99 0.027826 13 2000 chmod\n 3.54 0.019748 19 1002 2 mkdir\n 2.52 0.014033 4 3000 2000 rmdir\n 2.40 0.013388 13 1000 utimensat\n 1.66 0.009238 2 4027 12 access\n 1.39 0.007722 7 1000 unlink\n 0.25 0.001381 153 9 6 execve\n 0.02 0.000128 4 31 9 newfstatat\n 0.00 0.000013 6 2 2 statfs\n 0.00 0.000012 3 4 getcwd\n 0.00 0.000005 2 2 2 stat\n 0.00 0.000000 0 1 chdir\n 0.00 0.000000 0 1 rename\n------ ----------- ----------- --------- --------- ----------------\n100.00 0.557082 7 75211 36066 total\n", + "candidate": "% time seconds usecs/call calls errors syscall\n------ ----------- ----------- --------- --------- ----------------\n 45.57 0.222180 8 26072 3010 statx\n 23.41 0.114168 5 21028 21026 readlink\n 14.07 0.068630 16 4086 8 openat\n 5.79 0.028240 14 2000 chmod\n 3.49 0.017040 17 1002 2 mkdir\n 2.75 0.013403 13 1000 utimensat\n 2.03 0.009917 4 2001 1001 rmdir\n 1.46 0.007095 7 1000 unlink\n 1.27 0.006209 2 3028 12 access\n 0.13 0.000625 69 9 6 execve\n 0.01 0.000071 2 31 9 newfstatat\n 0.00 0.000015 15 1 rename\n 0.00 0.000006 1 4 getcwd\n 0.00 0.000006 6 1 chdir\n 0.00 0.000004 2 2 2 statfs\n 0.00 0.000000 0 2 2 stat\n------ ----------- ----------- --------- --------- ----------------\n100.00 0.487609 7 61267 25078 total\n" + } + }, + { + "name": "offline-user", + "size": 1000, + "args": [ + "--json", + "update", + "--scope", + "user", + "--offline" + ], + "baseline": { + "count": 15, + "p50Ms": 1505.7270309999585, + "p95Ms": 1634.2074610000127, + "samplesMs": [ + 1451.6899490000214, + 1634.2074610000127, + 1556.1285759999882, + 1514.3086089999997, + 1505.7270309999585, + 1459.8567400000175, + 1492.8708120000083, + 1385.6670959999901, + 1458.8953149999725, + 1429.3056229999638, + 1516.8779280000017, + 1506.0872310000123, + 1554.487392000039, + 1478.4899720000103, + 1518.5602000000072 + ] + }, + "candidate": { + "count": 15, + "p50Ms": 1287.5524250000017, + "p95Ms": 1361.05453199998, + "samplesMs": [ + 1233.7196599999443, + 1258.9402160000172, + 1312.6113049999694, + 1328.1868029999896, + 1247.4381280000089, + 1289.0423929999815, + 1336.6764059999841, + 1264.0481659999932, + 1258.5877999999793, + 1260.9332660000073, + 1243.8812540000072, + 1361.05453199998, + 1310.0881480000098, + 1287.5524250000017, + 1306.279841999989 + ] + }, + "gitOperations": { + "processes": 1, + "ls-remote": 0, + "clone": 0, + "pull": 0, + "fetch": 0, + "merge": 0, + "checkout": 0, + "reset": 0, + "rev-parse": 1, + "remote": 0, + "status": 0, + "config": 0, + "symbolic-ref": 0 + }, + "compatible": true, + "protectedStateSha256": "1ac8d8f67f474189870693d21651fc6a443bfa250ec1aa3da4a189f4f4477776", + "sourceResourcesSha256": "657679ed02250553ff9fe6a461836e2339a5fe685743eb808f3b495bc25d92fe", + "fileOperations": {} + } + ] +} diff --git a/scripts/benchmark-cli-sync.ts b/scripts/benchmark-cli-sync.ts new file mode 100644 index 00000000..e010d22a --- /dev/null +++ b/scripts/benchmark-cli-sync.ts @@ -0,0 +1,408 @@ +#!/usr/bin/env bun +/** Paired built-CLI timings on bounded, isolated local fixtures; no network latency claims. */ +import { createHash } from 'node:crypto'; +import { + type Stats, + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + readlinkSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { cpus, release } from 'node:os'; +import { join, resolve } from 'node:path'; + +const [baselineArg, candidateArg, ...flags] = process.argv.slice(2); +if (!baselineArg || !candidateArg || flags.includes('--help')) { + console.log( + 'Usage: bun scripts/benchmark-cli-sync.ts BASELINE CANDIDATE [--samples=15] [--warmups=3] [--sizes=0,100,1000] [--trace]', + ); + process.exit(baselineArg && candidateArg ? 0 : 2); +} +const baseline = resolve(baselineArg); +const candidate = resolve(candidateArg); +for (const path of [baseline, candidate]) + if (!existsSync(path)) throw new Error(`Missing CLI: ${path}`); +const option = (name: string, fallback: string) => + flags.find((flag) => flag.startsWith(`--${name}=`))?.split('=')[1] ?? + fallback; +const samples = Number(option('samples', '15')); +const warmups = Number(option('warmups', '3')); +const sizes = option('sizes', '0,100,1000').split(',').map(Number); +if ( + !Number.isInteger(samples) || + samples < 1 || + !Number.isInteger(warmups) || + warmups < 0 || + sizes.some((n) => !Number.isInteger(n) || n < 0 || n > 1000) +) + throw new Error('Invalid bounded sample/fixture sizes'); +const root = mkdtempSync('/tmp/allagents-cli-sync-bench-'); +const decoder = new TextDecoder(); +const rows: unknown[] = []; +const hash = (value: string) => + createHash('sha256').update(value).digest('hex'); +const operations = [ + 'ls-remote', + 'clone', + 'pull', + 'fetch', + 'merge', + 'checkout', + 'reset', + 'rev-parse', + 'remote', + 'status', + 'config', + 'symbolic-ref', +]; +function git(cwd: string, args: string[]): string { + const result = Bun.spawnSync(['git', '-C', cwd, ...args], { + stdout: 'pipe', + stderr: 'pipe', + env: { + ...process.env, + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_CONFIG_NOSYSTEM: '1', + }, + }); + if (result.exitCode) throw new Error(decoder.decode(result.stderr)); + return decoder.decode(result.stdout).trim(); +} +function snapshot(base: string): string { + const entries: string[] = []; + function visit(path: string, relative = '') { + let stat: Stats; + try { + stat = lstatSync(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; + throw error; + } + if (stat.isSymbolicLink()) + entries.push(`${relative}:link:${readlinkSync(path)}`); + else if (stat.isDirectory()) { + entries.push(`${relative}:directory`); + for (const name of readdirSync(path).sort()) + visit(join(path, name), join(relative, name)); + } else { + let content = readFileSync(path, 'utf8'); + if (path.endsWith('/sync-state.json')) { + const state = JSON.parse(content); + delete state.lastSync; + content = JSON.stringify(state); + } + entries.push(`${relative}:${stat.mode & 0o777}:${content}`); + } + } + for (const name of [ + '.agents', + '.codex', + 'AGENTS.md', + '.allagents/workspace.yaml', + '.allagents/sync-state.json', + ]) + visit(join(base, name), name); + return hash(entries.join('\n')); +} + +function protectedSnapshot(paths: string[]): string { + const entries: string[] = []; + function visit(path: string, label: string) { + let stat: Stats; + try { + stat = lstatSync(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + entries.push(`${label}:missing`); + return; + } + const mode = stat.mode & 0o777; + if (stat.isSymbolicLink()) { + entries.push(`${label}:${mode}:link:${readlinkSync(path)}`); + } else if (stat.isDirectory()) { + entries.push(`${label}:${mode}:directory`); + for (const name of readdirSync(path).sort()) visit(join(path, name), join(label, name)); + } else { + entries.push(`${label}:${mode}:file:${readFileSync(path).toString('base64')}`); + } + } + for (const [index, path] of paths.entries()) visit(path, String(index)); + return hash(entries.join('\n')); +} + +function summary(values: number[]) { + const sorted = [...values].sort((a, b) => a - b); + return { + count: values.length, + p50Ms: sorted[Math.ceil(sorted.length * 0.5) - 1], + p95Ms: sorted[Math.ceil(sorted.length * 0.95) - 1], + samplesMs: values, + }; +} +try { + for (const size of sizes) { + const fixture = join(root, `skills-${size}`); + const project = join(fixture, 'project'); + const home = join(fixture, 'home'); + const source = join(fixture, 'source'); + for (const path of [ + project, + home, + source, + join(home, '.allagents'), + join(project, '.allagents'), + ]) + mkdirSync(path, { recursive: true }); + for (let i = 0; i < size; i++) { + const skill = join(source, 'skills', `skill-${i}`); + mkdirSync(skill, { recursive: true }); + writeFileSync( + join(skill, 'SKILL.md'), + `---\nname: skill-${i}\ndescription: benchmark fixture\n---\n# Skill ${i}\n`, + ); + } + // Prevent the unrelated human-output update notice from starting a background request. + writeFileSync( + join(home, '.allagents/version-check.json'), + JSON.stringify({ + latestVersion: '0.0.0', + lastCheckedAt: new Date().toISOString(), + }), + ); + const gitConfig = join(fixture, 'gitconfig'); + writeFileSync(gitConfig, ''); + const env = { + ...process.env, + HOME: home, + ALLAGENTS_TEST_HOME: home, + USERPROFILE: home, + XDG_CONFIG_HOME: join(home, '.config'), + GIT_CONFIG_GLOBAL: gitConfig, + GIT_CONFIG_NOSYSTEM: '1', + GIT_TERMINAL_PROMPT: '0', + NO_COLOR: '1', + }; + const protectedPaths = [ + join(source, 'skills'), + ...[project, home].flatMap((base) => [ + join(base, '.allagents/workspace.yaml'), + join(base, '.codex/skills/user-owned/keep.txt'), + ]), + ]; + let expectedProtected: string; + let expectedResources: string; + let cachedResources: string | undefined; + function configure(plugin: string | null) { + const config = `version: 2\nrepositories: []\nclients: [codex]\nplugins: ${plugin ? `\n - source: ${JSON.stringify(plugin)}` : '[]'}\n`; + for (const base of [project, home]) + writeFileSync(join(base, '.allagents/workspace.yaml'), config); + for (const base of [project, home]) { + mkdirSync(join(base, '.codex/skills/user-owned'), { recursive: true }); + writeFileSync( + join(base, '.codex/skills/user-owned/keep.txt'), + 'preserve user content', + ); + } + expectedProtected = protectedSnapshot(protectedPaths); + expectedResources = protectedSnapshot([join(source, 'skills')]); + cachedResources = plugin === 'uat/cli-performance' + ? join(home, '.allagents/plugins/marketplaces/uat-cli-performance/skills') + : undefined; + } + function run(cli: string, args: string[], traceOutput?: string) { + const trace = join(fixture, 'git-trace.jsonl'); + writeFileSync(trace, ''); + const command = traceOutput + ? [ + 'strace', + '-f', + '-c', + '-e', + 'trace=%file', + '-o', + traceOutput, + cli, + ...args, + ] + : [cli, ...args]; + const start = performance.now(); + const result = Bun.spawnSync(command, { + cwd: project, + env: { ...env, GIT_TRACE2_EVENT: trace }, + stdout: 'pipe', + stderr: 'pipe', + }); + const ms = performance.now() - start; + // Check immutable fixture expectations even during initial setup/warmups; + // shared fixtures must not let an early mutation become the new baseline. + if (protectedSnapshot(protectedPaths) !== expectedProtected) { + throw new Error('Protected source/config/sentinel bytes or modes changed'); + } + if (cachedResources && protectedSnapshot([cachedResources]) !== expectedResources) { + throw new Error('Cached source resource bytes or modes changed'); + } + if (result.exitCode) + throw new Error( + `${args.join(' ')}: ${decoder.decode(result.stdout)}${decoder.decode(result.stderr)}`, + ); + const counts: Record = { processes: 0 }; + for (const operation of operations) counts[operation] = 0; + for (const line of readFileSync(trace, 'utf8') + .split('\n') + .filter(Boolean)) { + const event = JSON.parse(line); + if (event.event !== 'start') continue; + counts.processes = (counts.processes ?? 0) + 1; + for (const operation of operations) + if (event.argv?.includes(operation)) counts[operation] = (counts[operation] ?? 0) + 1; + } + return { + ms, + counts, + stdout: hash(decoder.decode(result.stdout)), + stderr: hash(decoder.decode(result.stderr)), + state: hash(snapshot(project) + snapshot(home)), + }; + } + function measure(name: string, args: string[]) { + run(baseline, args); // Materialize the selected scope before timing repeated reconciliation. + const values = { baseline: [] as number[], candidate: [] as number[] }; + const signatures = new Set(); + const counts = new Set(); + for (let i = -warmups; i < samples; i++) { + const order = + i % 2 === 0 + ? (['baseline', 'candidate'] as const) + : (['candidate', 'baseline'] as const); + for (const label of order) { + const result = run(label === 'baseline' ? baseline : candidate, args); + if (i >= 0) { + values[label].push(result.ms); + signatures.add( + JSON.stringify([result.stdout, result.stderr, result.state]), + ); + counts.add(JSON.stringify(result.counts)); + } + } + } + if (signatures.size !== 1 || counts.size !== 1) + throw new Error( + `${name}: baseline/candidate output, state, or Git work differs`, + ); + const fileOperations: Record = {}; + if ( + flags.includes('--trace') && + name.includes('project') && + size === Math.max(...sizes) + ) { + for (const [label, cli] of [ + ['baseline', baseline], + ['candidate', candidate], + ]) { + const output = join(fixture, `strace-${label}.txt`); + run(cli as string, args, output); + fileOperations[label as string] = readFileSync(output, 'utf8'); + } + } + rows.push({ + name, + size, + args, + baseline: summary(values.baseline), + candidate: summary(values.candidate), + gitOperations: JSON.parse([...counts][0]!), + compatible: true, + protectedStateSha256: expectedProtected, + sourceResourcesSha256: expectedResources, + fileOperations, + }); + console.error(`Measured ${name} (${size} skills)`); + } + configure(size ? source : null); + if (size === sizes[0]) { + measure('version', ['--version']); + measure('help', ['--help']); + } + for (const scope of ['project', 'user']) + measure(`local-${scope}`, [ + '--json', + 'update', + '--scope', + scope, + '--offline', + ]); + if (size === Math.max(...sizes) && size > 0) { + git(source, ['init', '-b', 'main']); + git(source, ['config', '--local', 'user.name', 'Benchmark']); + git(source, [ + 'config', + '--local', + 'user.email', + 'benchmark@example.test', + ]); + git(source, ['add', '.']); + git(source, ['commit', '-m', 'fixture']); + const remote = join(fixture, 'remote.git'); + git(fixture, ['clone', '--bare', source, remote]); + const originalHead = git(source, ['rev-parse', 'HEAD']); + writeFileSync( + gitConfig, + `[url "file://${remote}"]\n insteadOf = https://github.com/uat/cli-performance.git\n`, + ); + configure('uat/cli-performance'); + for (const scope of ['project', 'user']) + measure(`remote-${scope}`, ['--json', 'update', '--scope', scope]); + const cache = join( + home, + '.allagents/plugins/marketplaces/uat-cli-performance', + ); + if ( + git(cache, ['rev-parse', 'HEAD']) !== originalHead || + git(cache, ['status', '--porcelain']) !== '' + ) + throw new Error('Cache integrity changed'); + for (const scope of ['project', 'user']) + measure(`offline-${scope}`, [ + '--json', + 'update', + '--scope', + scope, + '--offline', + ]); + } + } + console.log( + JSON.stringify( + { + baseline, + candidate, + bundles: { + baselineSha256: hash(readFileSync(baseline, 'utf8')), + candidateSha256: hash(readFileSync(candidate, 'utf8')), + }, + samples, + warmups, + environment: { + bun: Bun.version, + node: decoder + .decode(Bun.spawnSync(['node', '--version']).stdout) + .trim(), + os: release(), + cpu: cpus()[0]?.model, + }, + profile: + 'local filesystem and local bare Git remotes; no injected delay', + rows, + }, + null, + 2, + ), + ); +} finally { + rmSync(root, { recursive: true, force: true }); +} diff --git a/src/core/sync.ts b/src/core/sync.ts index 54740647..44c0b253 100644 --- a/src/core/sync.ts +++ b/src/core/sync.ts @@ -982,7 +982,22 @@ export async function selectivePurgeWorkspace( const mapping = clientMappings[client]; if (!mapping) continue; const context = clientContexts?.get(client); + const writeRoot = resolve(context?.writeRoot ?? workspacePath); const purgedPaths: string[] = []; + const cleanupPaths = new Map(); + const trackedPaths = new Set( + previousFiles.map((path) => resolveTrackedPath(workspacePath, path)), + ); + // Overlapping state entries depend on immediate cleanup: removing the last + // file can also remove a directory that appears later in the tracked list. + const hasTrackedAncestor = (path: string): boolean => { + let parent = dirname(path); + while (parent !== writeRoot && pathIsWithin(writeRoot, parent)) { + if (trackedPaths.has(parent)) return true; + parent = dirname(parent); + } + return false; + }; for (const filePath of previousFiles) { if (!trackedPathIsAllowed(workspacePath, filePath, mapping, context)) { @@ -1010,15 +1025,22 @@ export async function selectivePurgeWorkspace( await unlink(cleanPath); } purgedPaths.push(filePath); - await cleanupEmptyParents( - context?.writeRoot ?? workspacePath, - cleanPath, - ); + if (hasTrackedAncestor(cleanPath)) { + await cleanupEmptyParents(writeRoot, cleanPath); + } else { + cleanupPaths.set(dirname(cleanPath), cleanPath); + } } catch { // Best effort - continue with other files } } + // Sibling deletions share a parent. Wait until they finish so cleanup does + // not repeatedly check and try to remove the same nonempty directory. + for (const cleanPath of cleanupPaths.values()) { + await cleanupEmptyParents(writeRoot, cleanPath); + } + if (purgedPaths.length > 0) { result.push({ client, paths: purgedPaths }); } diff --git a/tests/unit/core/sync-dedup.test.ts b/tests/unit/core/sync-dedup.test.ts index 7afd71c0..7615167d 100644 --- a/tests/unit/core/sync-dedup.test.ts +++ b/tests/unit/core/sync-dedup.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach, afterEach } from 'bun:test'; -import { mkdtemp, rm, mkdir, writeFile, readFile } from 'node:fs/promises'; +import { mkdtemp, rm, mkdir, writeFile, readFile, symlink } from 'node:fs/promises'; import { existsSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; @@ -157,6 +157,90 @@ describe('collectSyncedPaths with shared paths', () => { }); describe('external resolved path state and purge containment', () => { + it('unlinks tracked final symlinks but refuses traversal through symlink parents', async () => { + const root = await mkdtemp(join(tmpdir(), 'allagents-purge-symlinks-')); + const outside = join(root, 'outside'); + const workspace = join(root, 'workspace'); + const finalLink = '.claude/skills/final'; + const traversal = '.claude/skills/parent/SKILL.md'; + try { + await mkdir(outside, { recursive: true }); + await mkdir(join(workspace, '.claude/skills'), { recursive: true }); + await writeFile(join(outside, 'SKILL.md'), 'outside'); + await symlink(outside, join(workspace, finalLink), 'dir'); + await symlink(outside, join(workspace, '.claude/skills/parent'), 'dir'); + const state: SyncState = { + version: 1, + lastSync: new Date().toISOString(), + files: { claude: [finalLink, traversal] }, + }; + + expect(await selectivePurgeWorkspace(workspace, state, ['claude'])).toEqual([ + { client: 'claude', paths: [finalLink] }, + ]); + expect(existsSync(join(workspace, finalLink))).toBe(false); + expect(await readFile(join(workspace, traversal), 'utf8')).toBe('outside'); + expect(await readFile(join(outside, 'SKILL.md'), 'utf8')).toBe('outside'); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it('cleans shared empty parents while retaining user files and the write root', async () => { + const root = await mkdtemp(join(tmpdir(), 'allagents-purge-siblings-')); + const paths = [ + '.claude/commands/group/first.md', + '.claude/commands/group/second.md', + '.claude/skills/first/', + '.claude/skills/second/', + ]; + try { + await mkdir(join(root, '.claude/commands/group'), { recursive: true }); + await mkdir(join(root, '.claude/skills/first'), { recursive: true }); + await mkdir(join(root, '.claude/skills/second'), { recursive: true }); + for (const path of paths.slice(0, 2)) await writeFile(join(root, path), 'managed'); + await writeFile(join(root, '.claude/skills/user.md'), 'user'); + const state: SyncState = { + version: 1, + lastSync: new Date().toISOString(), + files: { claude: paths }, + }; + + expect(await selectivePurgeWorkspace(root, state, ['claude'])).toEqual([ + { client: 'claude', paths }, + ]); + expect(existsSync(join(root, '.claude/commands'))).toBe(false); + for (const path of paths) expect(existsSync(join(root, path))).toBe(false); + expect(await readFile(join(root, '.claude/skills/user.md'), 'utf8')).toBe('user'); + expect(existsSync(root)).toBe(true); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it('preserves purge reporting when state tracks both a file and its empty parent', async () => { + const root = await mkdtemp(join(tmpdir(), 'allagents-purge-overlap-')); + const file = '.claude/skills/managed/SKILL.md'; + const directory = '.claude/skills/managed/'; + try { + await mkdir(join(root, directory), { recursive: true }); + await writeFile(join(root, file), 'managed'); + const state: SyncState = { + version: 1, + lastSync: new Date().toISOString(), + files: { claude: [file, directory] }, + }; + + expect(await selectivePurgeWorkspace(root, state, ['claude'])).toEqual([ + { client: 'claude', paths: [file] }, + ]); + expect(existsSync(join(root, '.claude'))).toBe(false); + expect(existsSync(root)).toBe(true); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + it('tracks an external Pi root as an absolute path without traversal', () => { const contexts = resolveClientContexts(['pi'], 'user', { homeDir: '/home/tester',