diff --git a/crates/bashkit/Cargo.toml b/crates/bashkit/Cargo.toml index 28116567e..02e5f6077 100644 --- a/crates/bashkit/Cargo.toml +++ b/crates/bashkit/Cargo.toml @@ -349,6 +349,10 @@ harness = false name = "file_ops" harness = false +[[bench]] +name = "remote_fs" +harness = false + [[bench]] name = "awk_regex" harness = false diff --git a/crates/bashkit/benches/remote_fs.rs b/crates/bashkit/benches/remote_fs.rs new file mode 100644 index 000000000..8d1771177 --- /dev/null +++ b/crates/bashkit/benches/remote_fs.rs @@ -0,0 +1,226 @@ +//! Tree-walking builtins over a simulated remote filesystem. +//! +//! Remote VFS backends (a database, an object store, a network share) pay +//! one round trip per `FileSystem` call, ~10 ms in production. What makes a +//! command slow there is its call count, not CPU. This bench loads a real +//! tree, the bashkit checkout itself (`git ls-files`, ~3.7k files), into a +//! `CountingFs` and: +//! +//! 1. prints, per command and backend, the call counts and the time they +//! would cost at 10 ms per call (`--counts` or always, before timing); +//! 2. times each command with a simulated per-call latency +//! (`BASHKIT_BENCH_LATENCY_US`, default 100 us) under criterion. +//! +//! Backends: `plain` (no index, `walk` = `read_dir` per directory), `index` +//! (`SearchCapable` content index), `index+walk` (index plus a one-call bulk +//! `walk`, like a path-prefix query). +//! +//! Run with: `cargo bench --bench remote_fs` +//! Counts only: `cargo bench --bench remote_fs -- --counts` + +use bashkit::testing::{CallCounts, CountingFs}; +use bashkit::{Bash, FileSystem, FsLimits}; +use criterion::{BenchmarkId, Criterion, criterion_group}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; +use tokio::runtime::Runtime; + +/// Per-call latency the production report is about. +const REMOTE_CALL_MS: f64 = 10.0; +/// Files above this are skipped (generated fixtures, lockfiles). +const MAX_FILE_BYTES: u64 = 1 << 20; + +/// Commands under test, run with cwd `/workspace` (the checkout root). +const COMMANDS: &[&str] = &[ + // The production report's commands (no match in this checkout: worst case). + "rg -l markkit crates", + "grep -rln markkit crates", + "find /workspace -path '*markkit*'", + // A rare match and a common one. + "rg -l SearchCapable crates", + "rg -l --no-ignore SearchCapable crates", + "grep -rl SearchCapable crates", + "rg -c fn crates/bashkit/src/builtins", + "find crates -type f -name '*.rs'", + "find crates -ls", + "ls -R crates", + "du -s crates", + "tree crates", + "tar -cf /tmp/src.tar crates/bashkit/src/builtins", + "cp -r crates/bashkit/src/builtins /tmp/builtins-copy", +]; + +#[derive(Clone, Copy)] +enum Backend { + Plain, + Index, + IndexWalk, +} + +impl Backend { + const ALL: [Backend; 3] = [Backend::Plain, Backend::Index, Backend::IndexWalk]; + + fn name(self) -> &'static str { + match self { + Backend::Plain => "plain", + Backend::Index => "index", + Backend::IndexWalk => "index+walk", + } + } + + fn fs(self) -> CountingFs { + let fs = CountingFs::new(); + match self { + Backend::Plain => fs, + Backend::Index => fs.with_index(), + Backend::IndexWalk => fs.with_index().with_native_walk(), + } + } +} + +/// Repository root and its tracked files. +fn checkout_files() -> (PathBuf, Vec) { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let out = std::process::Command::new("git") + .arg("-C") + .arg(&root) + .args(["ls-files", "-z"]) + .output() + .expect("git ls-files (the bench needs a git checkout)"); + let files = out + .stdout + .split(|b| *b == 0) + .filter(|f| !f.is_empty()) + .map(|f| String::from_utf8_lossy(f).into_owned()) + .collect(); + (root, files) +} + +/// Copy the checkout into a fresh backend under `/workspace`; `latency` +/// applies after seeding. +fn load(rt: &Runtime, backend: Backend, latency: Duration) -> (Arc, usize) { + let (root, files) = checkout_files(); + let fs = Arc::new(backend.fs().with_limits(FsLimits::unlimited())); + let mut loaded = 0; + rt.block_on(async { + // `.git` marks the tree as a repository, so `.gitignore` applies. + fs.mkdir(Path::new("/workspace/.git"), true).await.unwrap(); + for rel in &files { + let host = root.join(rel); + let Ok(meta) = std::fs::symlink_metadata(&host) else { + continue; + }; + if !meta.is_file() || meta.len() > MAX_FILE_BYTES { + continue; + } + let Ok(content) = std::fs::read(&host) else { + continue; + }; + let path = PathBuf::from(format!("/workspace/{rel}")); + if let Some(parent) = path.parent() { + fs.mkdir(parent, true).await.unwrap(); + } + fs.write_file(&path, &content).await.unwrap(); + loaded += 1; + } + }); + fs.set_latency(latency); + (fs, loaded) +} + +fn bash_for(fs: &Arc) -> Bash { + let fs_dyn: Arc = fs.clone(); + Bash::builder() + .fs(fs_dyn) + .cwd("/workspace") + .limits(bashkit::ExecutionLimits::default().timeout(Duration::from_secs(600))) + .build() +} + +fn run_once(rt: &Runtime, fs: &Arc, cmd: &str) -> (CallCounts, Duration) { + let mut bash = bash_for(fs); + rt.block_on(async { + let _ = bash.exec("rm -rf /tmp/src.tar /tmp/builtins-copy").await; + }); + fs.reset(); + let start = std::time::Instant::now(); + let r = rt.block_on(bash.exec(&format!("{cmd} > /dev/null"))); + let elapsed = start.elapsed(); + if let Ok(r) = &r + && r.exit_code > 1 + { + eprintln!("warning: `{cmd}` exited {}: {}", r.exit_code, r.stderr); + } + (fs.counts(), elapsed) +} + +/// Markdown table of call counts and projected remote time. +fn print_counts(rt: &Runtime) { + println!("\n## Round trips per command (bashkit checkout, cwd = repo root)\n"); + let mut header_done = false; + for backend in Backend::ALL { + let (fs, loaded) = load(rt, backend, Duration::ZERO); + if !header_done { + println!("{loaded} files loaded. Projected time = calls x {REMOTE_CALL_MS} ms.\n"); + println!( + "| command | backend | calls | read_file | read_dir | stat+lstat+exists | walk | search | at 10 ms |" + ); + println!("|---|---|---:|---:|---:|---:|---:|---:|---:|"); + header_done = true; + } + for cmd in COMMANDS { + let (c, _) = run_once(rt, &fs, cmd); + println!( + "| `{cmd}` | {} | {} | {} | {} | {} | {} | {} | {:.1} s |", + backend.name(), + c.total(), + c.read_file, + c.read_dir, + c.metadata(), + c.walk, + c.search, + c.total() as f64 * REMOTE_CALL_MS / 1000.0 + ); + } + } + println!(); +} + +fn latency() -> Duration { + let us = std::env::var("BASHKIT_BENCH_LATENCY_US") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(100); + Duration::from_micros(us) +} + +fn bench_remote(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let latency = latency(); + let mut group = c.benchmark_group(format!("remote_fs_{}us", latency.as_micros())); + group.sample_size(10); + group.measurement_time(Duration::from_secs(5)); + for backend in Backend::ALL { + let (fs, _) = load(&rt, backend, latency); + for cmd in COMMANDS { + group.bench_with_input(BenchmarkId::new(backend.name(), cmd), cmd, |b, cmd| { + b.iter(|| run_once(&rt, &fs, cmd)) + }); + } + } + group.finish(); +} + +criterion_group!(benches, bench_remote); + +fn main() { + let counts_only = std::env::args().any(|a| a == "--counts"); + let rt = Runtime::new().unwrap(); + print_counts(&rt); + if counts_only { + return; + } + benches(); + Criterion::default().configure_from_args().final_summary(); +} diff --git a/crates/bashkit/benches/results/criterion-remote_fs-linux-x86_64-1793656706.md b/crates/bashkit/benches/results/criterion-remote_fs-linux-x86_64-1793656706.md new file mode 100644 index 000000000..e71ec0e41 --- /dev/null +++ b/crates/bashkit/benches/results/criterion-remote_fs-linux-x86_64-1793656706.md @@ -0,0 +1,120 @@ +# remote_fs bench, bashkit checkout (598ebb4 + this change) + +Host: cloud container, linux x86_64. Tree: `git ls-files` of this repo, 3,720 files under `/workspace` (a `.git` dir present so `.gitignore` applies). Command: `cargo bench --bench remote_fs`. + +Call counts are the signal: on a remote backend each call is a round trip ("at 10 ms" column). + +## Before (origin/main, same CountingFs, no `walk` hook) + +| command | backend | calls | read_file | read_dir | stat+lstat+exists | walk | search | at 10 ms | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| `rg -l markkit crates` | index | 176281 | 7101 | 1981 | 167199 | 0 | 0 | 1762.8 s | +| `grep -rln markkit crates` | index | 23709 | 3140 | 1982 | 18587 | 0 | 0 | 237.1 s | +| `find /workspace -path '*markkit*'` | index | 7831 | 0 | 2055 | 5776 | 0 | 0 | 78.3 s | +| `rg -l SearchCapable crates` | index | 176281 | 7101 | 1981 | 167199 | 0 | 0 | 1762.8 s | +| `rg -l --no-ignore SearchCapable crates` | index | 5116 | 3133 | 1981 | 2 | 0 | 0 | 51.2 s | +| `grep -rl SearchCapable crates` | index | 23709 | 3140 | 1982 | 18587 | 0 | 0 | 237.1 s | +| `rg -c fn crates/bashkit/src/builtins` | index | 1255 | 264 | 17 | 974 | 0 | 0 | 12.6 s | +| `find crates -type f -name '*.rs'` | index | 7105 | 0 | 1982 | 5123 | 0 | 0 | 71.0 s | +| `find crates -ls` | index | 9087 | 0 | 3964 | 5123 | 0 | 0 | 90.9 s | +| `ls -R crates` | index | 1982 | 0 | 1981 | 1 | 0 | 0 | 19.8 s | +| `du -s crates` | index | 7105 | 0 | 1982 | 5123 | 0 | 0 | 71.0 s | +| `tree crates` | index | 1982 | 0 | 1981 | 1 | 0 | 0 | 19.8 s | +| `tar -cf /tmp/src.tar crates/bashkit/src/builtins` | index | 463 | 212 | 17 | 233 | 0 | 0 | 4.6 s | +| `cp -r crates/bashkit/src/builtins /tmp/builtins-copy` | index | 2785 | 0 | 17 | 2522 | 0 | 0 | 27.9 s | + +## After + +| command | backend | calls | read_file | read_dir | stat+lstat+exists | walk | search | at 10 ms | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| `rg -l markkit crates` | plain | 5127 | 3135 | 1983 | 9 | 0 | 0 | 51.3 s | +| `grep -rln markkit crates` | plain | 5125 | 3140 | 1982 | 3 | 0 | 0 | 51.2 s | +| `find /workspace -path '*markkit*'` | plain | 2056 | 0 | 2055 | 1 | 0 | 0 | 20.6 s | +| `rg -l SearchCapable crates` | plain | 5127 | 3135 | 1983 | 9 | 0 | 0 | 51.3 s | +| `rg -l --no-ignore SearchCapable crates` | plain | 5116 | 3133 | 1981 | 2 | 0 | 0 | 51.2 s | +| `grep -rl SearchCapable crates` | plain | 5125 | 3140 | 1982 | 3 | 0 | 0 | 51.2 s | +| `rg -c fn crates/bashkit/src/builtins` | plain | 245 | 214 | 22 | 9 | 0 | 0 | 2.5 s | +| `find crates -type f -name '*.rs'` | plain | 1983 | 0 | 1982 | 1 | 0 | 0 | 19.8 s | +| `find crates -ls` | plain | 1983 | 0 | 1982 | 1 | 0 | 0 | 19.8 s | +| `ls -R crates` | plain | 1982 | 0 | 1981 | 1 | 0 | 0 | 19.8 s | +| `du -s crates` | plain | 1983 | 0 | 1982 | 1 | 0 | 0 | 19.8 s | +| `tree crates` | plain | 1982 | 0 | 1981 | 1 | 0 | 0 | 19.8 s | +| `tar -cf /tmp/src.tar crates/bashkit/src/builtins` | plain | 233 | 212 | 17 | 3 | 0 | 0 | 2.3 s | +| `cp -r crates/bashkit/src/builtins /tmp/builtins-copy` | plain | 780 | 0 | 17 | 517 | 0 | 0 | 7.8 s | +| `rg -l markkit crates` | index | 3 | 0 | 0 | 2 | 0 | 1 | 0.0 s | +| `grep -rln markkit crates` | index | 2 | 0 | 0 | 1 | 0 | 1 | 0.0 s | +| `find /workspace -path '*markkit*'` | index | 2056 | 0 | 2055 | 1 | 0 | 0 | 20.6 s | +| `rg -l SearchCapable crates` | index | 33 | 15 | 8 | 9 | 0 | 1 | 0.3 s | +| `rg -l --no-ignore SearchCapable crates` | index | 16 | 13 | 0 | 2 | 0 | 1 | 0.2 s | +| `grep -rl SearchCapable crates` | index | 28 | 13 | 0 | 14 | 0 | 1 | 0.3 s | +| `rg -c fn crates/bashkit/src/builtins` | index | 242 | 210 | 22 | 9 | 0 | 1 | 2.4 s | +| `find crates -type f -name '*.rs'` | index | 1983 | 0 | 1982 | 1 | 0 | 0 | 19.8 s | +| `find crates -ls` | index | 1983 | 0 | 1982 | 1 | 0 | 0 | 19.8 s | +| `ls -R crates` | index | 1982 | 0 | 1981 | 1 | 0 | 0 | 19.8 s | +| `du -s crates` | index | 1983 | 0 | 1982 | 1 | 0 | 0 | 19.8 s | +| `tree crates` | index | 1982 | 0 | 1981 | 1 | 0 | 0 | 19.8 s | +| `tar -cf /tmp/src.tar crates/bashkit/src/builtins` | index | 233 | 212 | 17 | 3 | 0 | 0 | 2.3 s | +| `cp -r crates/bashkit/src/builtins /tmp/builtins-copy` | index | 780 | 0 | 17 | 517 | 0 | 0 | 7.8 s | +| `rg -l markkit crates` | index+walk | 3 | 0 | 0 | 2 | 0 | 1 | 0.0 s | +| `grep -rln markkit crates` | index+walk | 2 | 0 | 0 | 1 | 0 | 1 | 0.0 s | +| `find /workspace -path '*markkit*'` | index+walk | 2 | 0 | 0 | 1 | 1 | 0 | 0.0 s | +| `rg -l SearchCapable crates` | index+walk | 33 | 15 | 8 | 9 | 0 | 1 | 0.3 s | +| `rg -l --no-ignore SearchCapable crates` | index+walk | 16 | 13 | 0 | 2 | 0 | 1 | 0.2 s | +| `grep -rl SearchCapable crates` | index+walk | 28 | 13 | 0 | 14 | 0 | 1 | 0.3 s | +| `rg -c fn crates/bashkit/src/builtins` | index+walk | 242 | 210 | 22 | 9 | 0 | 1 | 2.4 s | +| `find crates -type f -name '*.rs'` | index+walk | 2 | 0 | 0 | 1 | 1 | 0 | 0.0 s | +| `find crates -ls` | index+walk | 2 | 0 | 0 | 1 | 1 | 0 | 0.0 s | +| `ls -R crates` | index+walk | 1982 | 0 | 1981 | 1 | 0 | 0 | 19.8 s | +| `du -s crates` | index+walk | 2 | 0 | 0 | 1 | 1 | 0 | 0.0 s | +| `tree crates` | index+walk | 1982 | 0 | 1981 | 1 | 0 | 0 | 19.8 s | +| `tar -cf /tmp/src.tar crates/bashkit/src/builtins` | index+walk | 233 | 212 | 17 | 3 | 0 | 0 | 2.3 s | +| `cp -r crates/bashkit/src/builtins /tmp/builtins-copy` | index+walk | 780 | 0 | 17 | 517 | 0 | 0 | 7.8 s | + +## Criterion wall time, 100 us simulated latency per call + +Run concurrently with a test build, so noisy. Walk-heavy rows are CPU-bound by `InMemoryFs::read_dir`, which scans every entry per call (O(entries x dirs)); that is the in-memory backend's cost, not round trips. + +| bench | median | +|---|---:| +| `plain/rg -l markkit crates` | 3.1882 s | +| `plain/grep -rln markkit crates` | 2.9633 s | +| `plain/find /workspace -path '*markkit*'` | 2.5221 s | +| `plain/rg -l SearchCapable crates` | 3.2156 s | +| `plain/rg -l --no-ignore SearchCapable crates` | 3.0005 s | +| `plain/grep -rl SearchCapable crates` | 2.9670 s | +| `plain/rg -c fn crates/bashkit/src/builtins` | 67.001 ms | +| `plain/find crates -type f -name '*.rs'` | 2.3600 s | +| `plain/find crates -ls` | 2.3169 s | +| `plain/ls -R crates` | 2.4517 s | +| `plain/du -s crates` | 2.4584 s | +| `plain/tree crates` | 2.6678 s | +| `plain/tar -cf /tmp/src.tar crates/bashkit/src/builtins` | 48.890 ms | +| `plain/cp -r crates/bashkit/src/builtins /tmp/builtins-copy` | 199.99 ms | +| `index/rg -l markkit crates` | 79.563 ms | +| `index/grep -rln markkit crates` | 80.336 ms | +| `index/find /workspace -path '*markkit*'` | 2.9004 s | +| `index/rg -l SearchCapable crates` | 94.123 ms | +| `index/rg -l --no-ignore SearchCapable crates` | 82.366 ms | +| `index/grep -rl SearchCapable crates` | 80.259 ms | +| `index/rg -c fn crates/bashkit/src/builtins` | 92.512 ms | +| `index/find crates -type f -name '*.rs'` | 2.8683 s | +| `index/find crates -ls` | 2.8529 s | +| `index/ls -R crates` | 2.9771 s | +| `index/du -s crates` | 3.0325 s | +| `index/tree crates` | 3.0087 s | +| `index/tar -cf /tmp/src.tar crates/bashkit/src/builtins` | 56.475 ms | +| `index/cp -r crates/bashkit/src/builtins /tmp/builtins-copy` | 263.19 ms | +| `index+walk/rg -l markkit crates` | 80.651 ms | +| `index+walk/grep -rln markkit crates` | 78.429 ms | +| `index+walk/find /workspace -path '*markkit*'` | 2.6263 s | +| `index+walk/rg -l SearchCapable crates` | 96.695 ms | +| `index+walk/rg -l --no-ignore SearchCapable crates` | 81.962 ms | +| `index+walk/grep -rl SearchCapable crates` | 96.617 ms | +| `index+walk/rg -c fn crates/bashkit/src/builtins` | 96.170 ms | +| `index+walk/find crates -type f -name '*.rs'` | 2.5372 s | +| `index+walk/find crates -ls` | 2.5755 s | +| `index+walk/ls -R crates` | 2.8837 s | +| `index+walk/du -s crates` | 2.6339 s | +| `index+walk/tree crates` | 3.0670 s | +| `index+walk/tar -cf /tmp/src.tar crates/bashkit/src/builtins` | 51.691 ms | +| `index+walk/cp -r crates/bashkit/src/builtins /tmp/builtins-copy` | 249.46 ms | diff --git a/crates/bashkit/benches/results/criterion-remote_fs-linux-x86_64-1793662034.md b/crates/bashkit/benches/results/criterion-remote_fs-linux-x86_64-1793662034.md new file mode 100644 index 000000000..4403a18cb --- /dev/null +++ b/crates/bashkit/benches/results/criterion-remote_fs-linux-x86_64-1793662034.md @@ -0,0 +1,103 @@ +# remote_fs bench, bashkit checkout (bounded `walk` batches) + +Host: cloud container, linux x86_64. Tree: `git ls-files` of this repo, 3,725 files under `/workspace` (`.git` present so `.gitignore` applies). Command: `cargo bench --bench remote_fs`. + +Follows `criterion-remote_fs-linux-x86_64-1793656706.md` (the before/after for the round-trip fix, with origin/main numbers). This run adds bounded `walk` answers (`WalkOptions::max_entries`, 10k entries) and per-file rg input charging. Call counts are unchanged at this tree size: every walked subtree fits in one batch, so `find`/`du` on a bulk backend still take one `walk` call. Trees above 10k entries take one `walk` per batch, holding at most one batch in memory. + +Call counts are the signal: on a remote backend each call is a round trip ("at 10 ms" column). + +## Round trips per command + +| command | backend | calls | read_file | read_dir | stat+lstat+exists | walk | search | at 10 ms | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| `rg -l markkit crates` | plain | 5133 | 3140 | 1984 | 9 | 0 | 0 | 51.3 s | +| `grep -rln markkit crates` | plain | 5131 | 3145 | 1983 | 3 | 0 | 0 | 51.3 s | +| `find /workspace -path '*markkit*'` | plain | 2057 | 0 | 2056 | 1 | 0 | 0 | 20.6 s | +| `rg -l SearchCapable crates` | plain | 5133 | 3140 | 1984 | 9 | 0 | 0 | 51.3 s | +| `rg -l --no-ignore SearchCapable crates` | plain | 5122 | 3138 | 1982 | 2 | 0 | 0 | 51.2 s | +| `grep -rl SearchCapable crates` | plain | 5131 | 3145 | 1983 | 3 | 0 | 0 | 51.3 s | +| `rg -c fn crates/bashkit/src/builtins` | plain | 246 | 215 | 22 | 9 | 0 | 0 | 2.5 s | +| `find crates -type f -name '*.rs'` | plain | 1984 | 0 | 1983 | 1 | 0 | 0 | 19.8 s | +| `find crates -ls` | plain | 1984 | 0 | 1983 | 1 | 0 | 0 | 19.8 s | +| `ls -R crates` | plain | 1983 | 0 | 1982 | 1 | 0 | 0 | 19.8 s | +| `du -s crates` | plain | 1984 | 0 | 1983 | 1 | 0 | 0 | 19.8 s | +| `tree crates` | plain | 1983 | 0 | 1982 | 1 | 0 | 0 | 19.8 s | +| `tar -cf /tmp/src.tar crates/bashkit/src/builtins` | plain | 234 | 213 | 17 | 3 | 0 | 0 | 2.3 s | +| `cp -r crates/bashkit/src/builtins /tmp/builtins-copy` | plain | 783 | 0 | 17 | 519 | 0 | 0 | 7.8 s | +| `rg -l markkit crates` | index | 20 | 4 | 6 | 9 | 0 | 1 | 0.2 s | +| `grep -rln markkit crates` | index | 6 | 2 | 0 | 3 | 0 | 1 | 0.1 s | +| `find /workspace -path '*markkit*'` | index | 2057 | 0 | 2056 | 1 | 0 | 0 | 20.6 s | +| `rg -l SearchCapable crates` | index | 39 | 18 | 11 | 9 | 0 | 1 | 0.4 s | +| `rg -l --no-ignore SearchCapable crates` | index | 19 | 16 | 0 | 2 | 0 | 1 | 0.2 s | +| `grep -rl SearchCapable crates` | index | 34 | 16 | 0 | 17 | 0 | 1 | 0.3 s | +| `rg -c fn crates/bashkit/src/builtins` | index | 243 | 211 | 22 | 9 | 0 | 1 | 2.4 s | +| `find crates -type f -name '*.rs'` | index | 1984 | 0 | 1983 | 1 | 0 | 0 | 19.8 s | +| `find crates -ls` | index | 1984 | 0 | 1983 | 1 | 0 | 0 | 19.8 s | +| `ls -R crates` | index | 1983 | 0 | 1982 | 1 | 0 | 0 | 19.8 s | +| `du -s crates` | index | 1984 | 0 | 1983 | 1 | 0 | 0 | 19.8 s | +| `tree crates` | index | 1983 | 0 | 1982 | 1 | 0 | 0 | 19.8 s | +| `tar -cf /tmp/src.tar crates/bashkit/src/builtins` | index | 234 | 213 | 17 | 3 | 0 | 0 | 2.3 s | +| `cp -r crates/bashkit/src/builtins /tmp/builtins-copy` | index | 783 | 0 | 17 | 519 | 0 | 0 | 7.8 s | +| `rg -l markkit crates` | index+walk | 20 | 4 | 6 | 9 | 0 | 1 | 0.2 s | +| `grep -rln markkit crates` | index+walk | 6 | 2 | 0 | 3 | 0 | 1 | 0.1 s | +| `find /workspace -path '*markkit*'` | index+walk | 2 | 0 | 0 | 1 | 1 | 0 | 0.0 s | +| `rg -l SearchCapable crates` | index+walk | 39 | 18 | 11 | 9 | 0 | 1 | 0.4 s | +| `rg -l --no-ignore SearchCapable crates` | index+walk | 19 | 16 | 0 | 2 | 0 | 1 | 0.2 s | +| `grep -rl SearchCapable crates` | index+walk | 34 | 16 | 0 | 17 | 0 | 1 | 0.3 s | +| `rg -c fn crates/bashkit/src/builtins` | index+walk | 243 | 211 | 22 | 9 | 0 | 1 | 2.4 s | +| `find crates -type f -name '*.rs'` | index+walk | 2 | 0 | 0 | 1 | 1 | 0 | 0.0 s | +| `find crates -ls` | index+walk | 2 | 0 | 0 | 1 | 1 | 0 | 0.0 s | +| `ls -R crates` | index+walk | 1983 | 0 | 1982 | 1 | 0 | 0 | 19.8 s | +| `du -s crates` | index+walk | 2 | 0 | 0 | 1 | 1 | 0 | 0.0 s | +| `tree crates` | index+walk | 1983 | 0 | 1982 | 1 | 0 | 0 | 19.8 s | +| `tar -cf /tmp/src.tar crates/bashkit/src/builtins` | index+walk | 234 | 213 | 17 | 3 | 0 | 0 | 2.3 s | +| `cp -r crates/bashkit/src/builtins /tmp/builtins-copy` | index+walk | 783 | 0 | 17 | 519 | 0 | 0 | 7.8 s | + +## Criterion wall time, 100 us simulated latency per call + +Median of 10 samples. Walk-heavy rows are CPU-bound by `InMemoryFs::read_dir` (a scan over all entries per call), not by round trips. + +| command | backend | median | +|---|---|---:| +| `rg -l markkit crates` | plain | 3.0597 s | +| `grep -rln markkit crates` | plain | 2.8932 s | +| `find /workspace -path '*markkit*'` | plain | 2.4749 s | +| `rg -l SearchCapable crates` | plain | 3.0207 s | +| `rg -l --no-ignore SearchCapable crates` | plain | 2.8959 s | +| `grep -rl SearchCapable crates` | plain | 2.9299 s | +| `rg -c fn crates/bashkit/src/builtins` | plain | 67.735 ms | +| `find crates -type f -name '*.rs'` | plain | 2.3173 s | +| `find crates -ls` | plain | 2.2931 s | +| `ls -R crates` | plain | 2.2760 s | +| `du -s crates` | plain | 2.2095 s | +| `tree crates` | plain | 2.3023 s | +| `tar -cf /tmp/src.tar crates/bashkit/src/builtins` | plain | 47.803 ms | +| `cp -r crates/bashkit/src/builtins /tmp/builtins-copy` | plain | 189.74 ms | +| `rg -l markkit crates` | index | 87.746 ms | +| `grep -rln markkit crates` | index | 76.834 ms | +| `find /workspace -path '*markkit*'` | index | 2.7527 s | +| `rg -l SearchCapable crates` | index | 102.13 ms | +| `rg -l --no-ignore SearchCapable crates` | index | 80.975 ms | +| `grep -rl SearchCapable crates` | index | 79.148 ms | +| `rg -c fn crates/bashkit/src/builtins` | index | 85.890 ms | +| `find crates -type f -name '*.rs'` | index | 2.6288 s | +| `find crates -ls` | index | 2.7281 s | +| `ls -R crates` | index | 2.7668 s | +| `du -s crates` | index | 2.7436 s | +| `tree crates` | index | 2.7659 s | +| `tar -cf /tmp/src.tar crates/bashkit/src/builtins` | index | 59.947 ms | +| `cp -r crates/bashkit/src/builtins /tmp/builtins-copy` | index | 217.61 ms | +| `rg -l markkit crates` | index+walk | 86.722 ms | +| `grep -rln markkit crates` | index+walk | 76.385 ms | +| `find /workspace -path '*markkit*'` | index+walk | 2.4043 s | +| `rg -l SearchCapable crates` | index+walk | 94.729 ms | +| `rg -l --no-ignore SearchCapable crates` | index+walk | 76.746 ms | +| `grep -rl SearchCapable crates` | index+walk | 77.005 ms | +| `rg -c fn crates/bashkit/src/builtins` | index+walk | 87.444 ms | +| `find crates -type f -name '*.rs'` | index+walk | 2.4254 s | +| `find crates -ls` | index+walk | 2.4682 s | +| `ls -R crates` | index+walk | 2.6807 s | +| `du -s crates` | index+walk | 2.4679 s | +| `tree crates` | index+walk | 2.8006 s | +| `tar -cf /tmp/src.tar crates/bashkit/src/builtins` | index+walk | 58.151 ms | +| `cp -r crates/bashkit/src/builtins /tmp/builtins-copy` | index+walk | 226.55 ms | diff --git a/crates/bashkit/docs/threat-model.md b/crates/bashkit/docs/threat-model.md index 8125adfdf..b1aa5598b 100644 --- a/crates/bashkit/docs/threat-model.md +++ b/crates/bashkit/docs/threat-model.md @@ -34,6 +34,7 @@ through configurable limits. | Width-derived allocation (TM-DOS-103) | `printf x \| od -w1152921504606846976` | Reject widths above 65,536 bytes; stack-pad one numeric field; budget output capacity before fallible growth | MITIGATED | | Pagination amplification (TM-DOS-137) | Long `pr -h` headers repeated on many short pages | Check the 16 MiB cap before every append; lease input, rows and output capacity before growth; borrow cells/page slices and retain date timestamps; charge input/work and poll cancellation/deadlines, including skipped pages | MITIGATED | | Embedded CLI limit escalation (TM-DOS-139) | `markkit extract x --max-input-size 99999999999` | `markkit` limits can only be lowered below their defaults; `fetchkit` bodies capped by HTTP limits; `--ocr` refused | MITIGATED | +| Tree-walk memory (TM-DOS-140) | `find`, `du` or `rg` over a tree with millions of files | `FileSystem::walk` answers are bounded batches (10k entries) refilled as `find`/`du` descend; `rg` charges each file to the aggregate-input budget as it reads and stops at the first refusal | MITIGATED | | Variable explosion (TM-DOS-003) | `x=$(cat /dev/urandom)` | /dev/urandom returns bounded 8KB | MITIGATED | | Array growth (TM-DOS-004) | `arr+=(element)` in loop | Command limit | MITIGATED | diff --git a/crates/bashkit/src/builtins/archive.rs b/crates/bashkit/src/builtins/archive.rs index 544ebd5bf..c9746e0ac 100644 --- a/crates/bashkit/src/builtins/archive.rs +++ b/crates/bashkit/src/builtins/archive.rs @@ -580,14 +580,12 @@ async fn create_tar( } let path = resolve_path(&base_dir, file); - if !ctx.fs.exists(&path).await.unwrap_or(false) { + let Ok(metadata) = ctx.fs.stat(&path).await else { return Ok(ExecResult::err( format!("tar: {}: Cannot stat: No such file or directory\n", file), 2, )); - } - - let metadata = ctx.fs.stat(&path).await?; + }; if metadata.file_type.is_dir() { // Add directory recursively @@ -595,6 +593,7 @@ async fn create_tar( ctx, &path, file, + metadata, &mut output_data, &mut verbose_output, names, @@ -607,6 +606,7 @@ async fn create_tar( ctx, &path, file, + Some(metadata), &mut output_data, &mut verbose_output, names, @@ -658,10 +658,13 @@ async fn create_tar( } /// Add a file to tar archive +/// `known` is the followed metadata when the caller has it (a listing entry +/// that is not a symlink), which saves a `stat` round trip per member. async fn add_file_to_tar( ctx: &Context<'_>, path: &Path, name: &str, + known: Option, output: &mut BudgetedBytes, verbose_output: &mut BudgetedString, names: TarNames, @@ -670,7 +673,10 @@ async fn add_file_to_tar( verbose, absolute_names, } = names; - let metadata = ctx.fs.stat(path).await?; + let metadata = match known { + Some(metadata) => metadata, + None => ctx.fs.stat(path).await?, + }; let content = ctx.fs.read_file(path).await?; ctx.consume_budget_input(content.len())?; ctx.consume_budget_work(u64::try_from(content.len().div_ceil(64)).unwrap_or(u64::MAX))?; @@ -744,10 +750,12 @@ fn tar_member_name(name: &str, absolute_names: bool) -> &str { } /// Add a directory to tar archive recursively +#[allow(clippy::too_many_arguments)] fn add_directory_to_tar<'a>( ctx: &'a Context<'_>, path: &'a Path, name: &'a str, + metadata: crate::fs::Metadata, output: &'a mut BudgetedBytes, verbose_output: &'a mut BudgetedString, names: TarNames, @@ -764,8 +772,6 @@ fn add_directory_to_tar<'a>( verbose_output.try_push_str("/\n")?; } - let metadata = ctx.fs.stat(path).await?; - // Create tar header for directory let mut header = [0u8; TAR_BLOCK_SIZE]; @@ -823,6 +829,7 @@ fn add_directory_to_tar<'a>( ctx, &child_path, &child_name, + entry.metadata, output, verbose_output, names, @@ -830,8 +837,17 @@ fn add_directory_to_tar<'a>( ) .await?; } else { - add_file_to_tar(ctx, &child_path, &child_name, output, verbose_output, names) - .await?; + let known = (!entry.metadata.file_type.is_symlink()).then_some(entry.metadata); + add_file_to_tar( + ctx, + &child_path, + &child_name, + known, + output, + verbose_output, + names, + ) + .await?; } } diff --git a/crates/bashkit/src/builtins/disk.rs b/crates/bashkit/src/builtins/disk.rs index e6c175269..1325df8cb 100644 --- a/crates/bashkit/src/builtins/disk.rs +++ b/crates/bashkit/src/builtins/disk.rs @@ -5,14 +5,19 @@ //! `-c` total, GNU ceiling rounding for `-h`/`--si`/`-B`). The VFS has no //! block allocation, so "disk usage" is the apparent size; without `-b` it //! is reported in ceil(bytes / 1024) blocks. +//! +//! Decision: `du` reads each directory once (via `FileSystem::walk`) and takes +//! nested entries' sizes from the listings. Remote backends pay one round +//! trip per call, so a `stat` per file would dominate. use async_trait::async_trait; use std::path::Path; use super::arg_parser::{OptArg, gnu_getopt}; +use super::walk_cache::WalkCache; use super::{Builtin, Context}; use crate::error::Result; -use crate::fs::vfs_join; +use crate::fs::{Metadata, vfs_join}; use crate::interpreter::ExecResult; /// The du builtin - estimate file space usage. @@ -226,13 +231,20 @@ impl Builtin for Du { } else { vfs_join(ctx.cwd, operand) }; - if ctx.fs.stat(&path).await.is_err() { + // Operands follow symlinks (GNU stats command-line args). + let Ok(metadata) = ctx.fs.stat(&path).await else { stderr.push_str(&format!( "du: cannot access '{operand}': No such file or directory\n" )); continue; - } - grand_total += walk(&ctx, &path, operand, 0, &opts, &mut output).await?; + }; + let mut walker = DuWalk { + ctx: &ctx, + opts: &opts, + listings: WalkCache::new(true), + output: &mut output, + }; + grand_total += walker.walk(&path, operand, 0, metadata).await?; } if opts.total { output.push_str(&format!( @@ -250,53 +262,64 @@ impl Builtin for Du { } } -/// Sum `path` (displayed as `display`), printing entries GNU-style. -fn walk<'a>( - ctx: &'a Context<'_>, - path: &'a Path, - display: &'a str, - depth: usize, +struct DuWalk<'a, 'c> { + ctx: &'a Context<'c>, opts: &'a DuOptions, + /// du lists every directory below the operand, so listings come in + /// bounded `walk` batches (one query each on bulk backends); nested + /// entries take their metadata from them, never a per-file stat. + listings: WalkCache, output: &'a mut String, -) -> std::pin::Pin> + Send + 'a>> { - Box::pin(async move { - ctx.consume_budget_work(1)?; - // Operands follow symlinks (GNU stats command-line args); nested - // entries count the link itself. - let metadata = if depth == 0 { - ctx.fs.stat(path).await? - } else { - ctx.fs.lstat(path).await? - }; - let shown = opts.max_depth.is_none_or(|d| depth <= d); +} + +impl DuWalk<'_, '_> { + /// Sum `path` (displayed as `display`, with `metadata` already known), + /// printing entries GNU-style. Nested entries count a symlink itself. + fn walk<'s>( + &'s mut self, + path: &'s Path, + display: &'s str, + depth: usize, + metadata: Metadata, + ) -> std::pin::Pin> + Send + 's>> { + Box::pin(async move { + self.ctx.consume_budget_work(1)?; + let shown = self.opts.max_depth.is_none_or(|d| depth <= d); + + if !metadata.file_type.is_dir() { + if shown && (self.opts.all || depth == 0) { + self.output.push_str(&format!( + "{}\t{display}\n", + format_size(metadata.size, self.opts.scale) + )); + } + return Ok(metadata.size); + } - if !metadata.file_type.is_dir() { - if shown && (opts.all || depth == 0) { - output.push_str(&format!( + let mut total = 0u64; + let entries = self.listings.list(&*self.ctx.fs, path, None).await?; + self.ctx + .consume_budget_work(u64::try_from(entries.len()).unwrap_or(u64::MAX))?; + for entry in entries { + let child_path = vfs_join(path, &entry.name); + let child_display = if display.ends_with('/') { + format!("{display}{}", entry.name) + } else { + format!("{display}/{}", entry.name) + }; + total += self + .walk(&child_path, &child_display, depth + 1, entry.metadata) + .await?; + } + if shown { + self.output.push_str(&format!( "{}\t{display}\n", - format_size(metadata.size, opts.scale) + format_size(total, self.opts.scale) )); } - return Ok(metadata.size); - } - - let mut total = 0u64; - let entries = ctx.fs.read_dir(path).await?; - ctx.consume_budget_work(u64::try_from(entries.len()).unwrap_or(u64::MAX))?; - for entry in entries { - let child_path = vfs_join(path, &entry.name); - let child_display = if display.ends_with('/') { - format!("{display}{}", entry.name) - } else { - format!("{display}/{}", entry.name) - }; - total += walk(ctx, &child_path, &child_display, depth + 1, opts, output).await?; - } - if shown { - output.push_str(&format!("{}\t{display}\n", format_size(total, opts.scale))); - } - Ok(total) - }) + Ok(total) + }) + } } /// The df builtin - report file system disk space usage. diff --git a/crates/bashkit/src/builtins/fileops.rs b/crates/bashkit/src/builtins/fileops.rs index f8423e763..0e8dc4de8 100644 --- a/crates/bashkit/src/builtins/fileops.rs +++ b/crates/bashkit/src/builtins/fileops.rs @@ -428,12 +428,27 @@ pub(crate) fn copy_tree<'a>( src: &'a Path, dst: &'a Path, no_clobber: bool, +) -> CopyTreeFuture<'a> { + copy_tree_entry(ctx, src, dst, no_clobber, None) +} + +/// [`copy_tree`] for an entry whose `lstat` metadata may already be known +/// from its parent's listing (saves one round trip per entry). +fn copy_tree_entry<'a>( + ctx: &'a Context<'_>, + src: &'a Path, + dst: &'a Path, + no_clobber: bool, + known: Option, ) -> CopyTreeFuture<'a> { Box::pin(async move { ctx.consume_budget_work(1)?; - let meta = match ctx.fs.lstat(src).await { - Ok(m) => m, - Err(e) => return Ok(Err(format!("cannot stat '{}': {e}", src.display()))), + let meta = match known { + Some(m) => m, + None => match ctx.fs.lstat(src).await { + Ok(m) => m, + Err(e) => return Ok(Err(format!("cannot stat '{}': {e}", src.display()))), + }, }; if meta.file_type.is_dir() { match ctx.fs.stat(dst).await { @@ -462,7 +477,9 @@ pub(crate) fn copy_tree<'a>( for entry in entries { let from = vfs_join(src, &entry.name); let to = vfs_join(dst, &entry.name); - if let Err(msg) = copy_tree(ctx, &from, &to, no_clobber).await? { + if let Err(msg) = + copy_tree_entry(ctx, &from, &to, no_clobber, Some(entry.metadata)).await? + { return Ok(Err(msg)); } } diff --git a/crates/bashkit/src/builtins/find/mod.rs b/crates/bashkit/src/builtins/find/mod.rs index f726f1c0b..7b62cabdd 100644 --- a/crates/bashkit/src/builtins/find/mod.rs +++ b/crates/bashkit/src/builtins/find/mod.rs @@ -20,6 +20,14 @@ //! so those read the modification time. //! - THREAT[TM-DOS-121]: `-L` loop detection, budget-charged listings, capped //! output, bounded `{} +` batches. +//! - Round trips (remote VFS backends pay ~one network call per FS call): an +//! entry's metadata comes from its parent's `read_dir` listing; `lstat` runs +//! only for roots, and symlinks add `read_link` + a target `stat`. A +//! directory listing read for `%n`/`-ls` is reused when descending. Without +//! side-effecting or traversal-cutting actions (`-exec*`, `-delete`, +//! `-prune`, `-quit`), directories are listed through bounded +//! `FileSystem::walk` batches (`WalkCache`): one query per ~10k entries on +//! bulk backends, and never more than one batch held in memory. //! - `-ok`/`-okdir`, `-samefile`, `-inum`, `-links`, //! `-fstype`, `-context` are not implemented (L-FIND-001). @@ -37,12 +45,13 @@ use async_trait::async_trait; use parse::{Cmp, Expr, Follow, Identity, NewerRef, Options, PermKind, TimeField}; use super::limits::FIND_MAX_OUTPUT_BYTES; +use super::walk_cache::WalkCache; use super::{ Builtin, Context, Date, ExecutionPlan, PlanDriver, PlanStep, SubCommand, fnmatch, resolve_path, }; use crate::StreamData; use crate::error::Result; -use crate::fs::{FileSystem, Metadata, vfs_join}; +use crate::fs::{DirEntry, FileSystem, Metadata, vfs_join}; use crate::interpreter::ExecResult; const HELP: &str = "Usage: find [-H] [-L] [-P] [PATH...] [EXPRESSION]\nSearch for files in a directory hierarchy.\n\nOperators: ( EXPR ), ! EXPR, -not EXPR, EXPR -a EXPR, EXPR -o EXPR, EXPR , EXPR\nOptions: -maxdepth N -mindepth N -depth -follow -daystart -regextype TYPE -xdev\nTests: -name -iname -path -ipath -lname -regex -iregex -type -xtype -size -empty\n -mtime -mmin -atime -amin -ctime -cmin -newer -newerXY -perm -readable\n -writable -executable -user -group -uid -gid -nouser -nogroup -true -false\nActions: -print -print0 -printf FMT -delete -prune -quit\n -exec CMD {} ; -exec CMD {} + -execdir CMD {} ; -execdir CMD {} +\n --help\tdisplay this help and exit\n --version\toutput version information and exit\n"; @@ -122,6 +131,8 @@ pub(super) struct Entry { pub(super) link_target: Option, follow: bool, subdirs: usize, + /// Listing read for `%n`/`-ls`, reused when descending. + listing: Option>, } impl Entry { @@ -187,6 +198,8 @@ enum Work { root: usize, depth: usize, ancestors: Option>, + /// `lstat` metadata already known from the parent's listing. + lmeta: Option, }, /// `-depth`: evaluate a directory after its children. Post(Box), @@ -502,6 +515,9 @@ pub(super) struct FindRun { budget: Option>, exec_templates: HashMap>, needs_nlink: bool, + /// Listings fetched ahead in bounded `walk` batches (only when no action + /// changes or cuts the traversal), consumed as directories are descended. + walk: WalkCache, next_root: usize, stack: Vec, current: Option, @@ -589,6 +605,15 @@ impl FindRun { } let mut exec_templates = HashMap::new(); let mut needs_nlink = false; + let mut bulk_walk = true; + expr.walk(&mut |e| { + if matches!( + e, + Expr::Exec { .. } | Expr::Delete { .. } | Expr::Prune | Expr::Quit + ) { + bulk_walk = false; + } + }); expr.walk_mut(&mut |e| match e { Expr::Newer { reference, .. } => { if let NewerRef::File(key) | NewerRef::Date(key) = reference { @@ -625,6 +650,7 @@ impl FindRun { budget: ctx.execution_budget(), exec_templates, needs_nlink, + walk: WalkCache::new(bulk_walk), next_root: 0, stack: Vec::new(), current: None, @@ -737,16 +763,17 @@ impl FindRun { async fn start_root(&mut self, index: usize) { let root = self.cfg.roots[index].clone(); let abs = resolve_path(&self.cwd, &root); - if self.cfg.fs.lstat(&abs).await.is_err() { + let Ok(lmeta) = self.cfg.fs.lstat(&abs).await else { self.error(format!("find: '{root}': No such file or directory\n")); return; - } + }; self.stack.push(Work::Visit { abs, display: root, root: index, depth: 0, ancestors: None, + lmeta: Some(lmeta), }); } @@ -759,7 +786,7 @@ impl FindRun { } async fn visit(&mut self, work: Work) -> Result<()> { - let (abs, display, root, depth, ancestors) = match work { + let (abs, display, root, depth, ancestors, lmeta) = match work { Work::Post(entry) => { self.current = Some(Pending { entry: *entry, @@ -775,15 +802,19 @@ impl FindRun { root, depth, ancestors, - } => (abs, display, root, depth, ancestors), + lmeta, + } => (abs, display, root, depth, ancestors, lmeta), }; let fs = self.cfg.fs.clone(); - let lmeta = match fs.lstat(&abs).await { - Ok(m) => m, - Err(e) => { - self.error(format!("find: '{display}': {e}\n")); - return Ok(()); - } + let lmeta = match lmeta { + Some(m) => m, + None => match fs.lstat(&abs).await { + Ok(m) => m, + Err(e) => { + self.error(format!("find: '{display}': {e}\n")); + return Ok(()); + } + }, }; let follow = self.follows(depth); let (link_target, fmeta, dir_path) = if lmeta.file_type.is_symlink() { @@ -813,14 +844,19 @@ impl FindRun { link_target, follow, subdirs: 0, + listing: None, }; let is_dir = entry.meta().file_type.is_dir(); if is_dir && self.needs_nlink { - entry.subdirs = fs - .read_dir(&entry.dir_path) - .await - .map(|c| c.iter().filter(|d| d.metadata.file_type.is_dir()).count()) - .unwrap_or(0); + let levels = self.levels_below(entry.depth); + let listing = self.walk.list(&*fs, &entry.dir_path, levels).await; + if let Ok(list) = listing { + entry.subdirs = list + .iter() + .filter(|d| d.metadata.file_type.is_dir()) + .count(); + entry.listing = Some(list); + } } if is_dir && self.cfg.opts.follow != Follow::Never { let mut cur = ancestors.as_deref(); @@ -837,7 +873,7 @@ impl FindRun { } } if self.cfg.opts.depth_first && is_dir { - let children = self.children(&entry, ancestors).await?; + let children = self.children(&mut entry, ancestors).await?; self.stack.push(Work::Post(Box::new(entry))); self.stack.extend(children.into_iter().rev()); } else { @@ -853,11 +889,16 @@ impl FindRun { Ok(()) } + /// Directory levels still to list below `depth` (`None` = unlimited). + fn levels_below(&self, depth: usize) -> Option { + self.cfg.opts.max_depth.map(|m| m.saturating_sub(depth)) + } + /// List `entry`'s children as Visit work items (empty for non-directories /// and at -maxdepth). async fn children( &mut self, - entry: &Entry, + entry: &mut Entry, ancestors: Option>, ) -> Result> { if !entry.meta().file_type.is_dir() @@ -865,11 +906,18 @@ impl FindRun { { return Ok(Vec::new()); } - let mut list = match self.cfg.fs.read_dir(&entry.dir_path).await { - Ok(list) => list, - Err(e) => { - self.error(format!("find: '{}': {e}\n", entry.display)); - return Ok(Vec::new()); + let mut list = match entry.listing.take() { + Some(list) => list, + None => { + let levels = self.levels_below(entry.depth); + let fs = self.cfg.fs.clone(); + match self.walk.list(&*fs, &entry.dir_path, levels).await { + Ok(list) => list, + Err(e) => { + self.error(format!("find: '{}': {e}\n", entry.display)); + return Ok(Vec::new()); + } + } } }; self.consume_work(list.len())?; @@ -893,6 +941,7 @@ impl FindRun { root: entry.root, depth: entry.depth + 1, ancestors: chain.clone(), + lmeta: Some(child.metadata), }) .collect()) } @@ -967,7 +1016,8 @@ impl PlanDriver for FindRun { Ok(_) => { self.commit(effects, &pending.cache); if !pending.post && !prune && !self.quit { - let children = self.children(&pending.entry, pending.ancestors).await?; + let children = + self.children(&mut pending.entry, pending.ancestors).await?; self.stack.extend(children.into_iter().rev()); } } diff --git a/crates/bashkit/src/builtins/grep.rs b/crates/bashkit/src/builtins/grep.rs index ff3688c92..7b8e40fb6 100644 --- a/crates/bashkit/src/builtins/grep.rs +++ b/crates/bashkit/src/builtins/grep.rs @@ -4,6 +4,9 @@ //! recorded GNU goldens. //! //! Decisions: +//! - Round trips: `-r` takes file sizes from the directory listing (no +//! `stat` per file) and resolves only the root and symlinked directories; +//! an index answer with zero matches is final, never re-walked. //! - Patterns: `-G` (default) and `-E` are GNU BRE/ERE translated by //! `grep_pattern` and matched leftmost-longest (`-o` prints the longest //! alternative); back-references fall back to fancy-regex. `-P` is @@ -1139,6 +1142,7 @@ impl Builtin for Grep { &mut errors, &mut read_failed, stream.as_ref(), + None, ) .await? }; @@ -1206,6 +1210,8 @@ struct GrepAncestor { struct GrepDirectory { path: std::path::PathBuf, + /// Symlink-free path, when known without resolving. + canonical: Option, ancestors: Option>, _lease: Option, } @@ -1214,11 +1220,15 @@ impl GrepDirectory { fn new( ctx: &Context<'_>, path: &std::path::Path, + canonical: Option, ancestors: Option>, ) -> Result { - let lease = ctx.lease_budget_bytes(path.as_os_str().len())?; + let lease = ctx.lease_budget_bytes( + path.as_os_str().len() + canonical.as_ref().map_or(0, |c| c.as_os_str().len()), + )?; Ok(Self { path: path.to_path_buf(), + canonical, ancestors, _lease: lease, }) @@ -1263,16 +1273,22 @@ async fn scan_file( errors: &mut String, read_failed: &mut bool, stream: Option<&super::StdoutStream>, + size_hint: Option, ) -> Result<(usize, bool)> { ctx.consume_budget_work(1)?; - let meta = match ctx.fs.stat(path).await { - Ok(meta) => meta, - Err(e) => { - read_error(sh.opts, name, &e, errors, read_failed); - return Ok((0, false)); - } + // A recursive walk already has the size from the directory listing; a + // `stat` per file would double the round trips on remote backends. + let size = match size_hint { + Some(size) => size, + None => match ctx.fs.stat(path).await { + Ok(meta) => meta.size, + Err(e) => { + read_error(sh.opts, name, &e, errors, read_failed); + return Ok((0, false)); + } + }, }; - let size = usize::try_from(meta.size).unwrap_or(usize::MAX); + let size = usize::try_from(size).unwrap_or(usize::MAX); ctx.consume_budget_input(size)?; let _lease = ctx.lease_budget_bytes(size)?; let bytes = match ctx.fs.read_file(path).await { @@ -1316,7 +1332,8 @@ async fn scan_recursive( { for path in found.paths.iter() { let name = recursive_display(operand, &root, path); - let (n, _) = scan_file(ctx, sh, &name, path, out, errors, read_failed, None).await?; + let (n, _) = + scan_file(ctx, sh, &name, path, out, errors, read_failed, None, None).await?; selected = selected.saturating_add(n); if n > 0 && opts.quiet { break; @@ -1325,18 +1342,24 @@ async fn scan_recursive( return Ok(selected); } let mut stack = budgeted_vec(ctx)?; - stack.try_push(GrepDirectory::new(ctx, &root, None)?)?; + stack.try_push(GrepDirectory::new(ctx, &root, None, None)?)?; // THREAT[TM-DOS-011,TM-DOS-096]: A hop cap bounds one lookup, not a // branching walk. Reject ancestor cycles and meter synchronous VFS work. while let Some(dir) = stack.pop() { ctx.consume_budget_work(1)?; - let canonical = match crate::fs::canonicalize(ctx.fs.as_ref(), &dir.path).await { - Ok(path) => path, - Err(e) => { - let name = recursive_display(operand, &root, &dir.path); - read_error(opts, &name, &e, errors, read_failed); - continue; - } + // Only the root and directories reached through a symlink need a + // resolve (one `lstat` per path component); a plain subdirectory's + // canonical path is its parent's plus its name. + let canonical = match dir.canonical.clone() { + Some(path) => path, + None => match crate::fs::canonicalize(ctx.fs.as_ref(), &dir.path).await { + Ok(path) => path, + Err(e) => { + let name = recursive_display(operand, &root, &dir.path); + read_error(opts, &name, &e, errors, read_failed); + continue; + } + }, }; let mut ancestor = dir.ancestors.as_deref(); let mut cycle = false; @@ -1365,7 +1388,7 @@ async fn scan_recursive( .saturating_add(std::mem::size_of::()), )?; let ancestors = std::sync::Arc::new(GrepAncestor { - path: canonical, + path: canonical.clone(), parent: dir.ancestors, _lease: ancestor_lease, }); @@ -1392,12 +1415,17 @@ async fn scan_recursive( ctx.consume_budget_work(1)?; let entry_path = vfs_join(&dir.path, &entry.name); let mut ft = entry.metadata.file_type; - if ft.is_symlink() { + let mut size = entry.metadata.size; + let via_link = ft.is_symlink(); + if via_link { if !opts.dereference { continue; } match ctx.fs.stat(&entry_path).await { - Ok(meta) => ft = meta.file_type, + Ok(meta) => { + ft = meta.file_type; + size = meta.size; + } Err(_) => continue, } } @@ -1412,14 +1440,25 @@ async fn scan_recursive( subdirs.try_push(GrepDirectory::new( ctx, &entry_path, + (!via_link).then(|| vfs_join(&canonical, &entry.name)), Some(ancestors.clone()), )?)?; } else if ft.is_file() && should_include_file(&entry.name, &opts.include_patterns, &opts.exclude_patterns) { let name = recursive_display(operand, &root, &entry_path); - let (n, _) = - scan_file(ctx, sh, &name, &entry_path, out, errors, read_failed, None).await?; + let (n, _) = scan_file( + ctx, + sh, + &name, + &entry_path, + out, + errors, + read_failed, + None, + Some(size), + ) + .await?; selected = selected.saturating_add(n); if n > 0 && opts.quiet { return Ok(selected); @@ -1582,9 +1621,8 @@ async fn try_indexed_search( inputs.try_push(candidate)?; } - if inputs.is_empty() { - return Ok(None); - } + // An index that answered with no (in-scope) match is still an answer: + // walking the tree to find nothing again costs a read per file. Ok(Some(IndexedInputs { paths: inputs, _leases: path_leases, @@ -2188,7 +2226,10 @@ mod tests { } #[tokio::test] - async fn test_grep_recursive_indexed_search_falls_back_for_multiple_roots() { + /// The index is authoritative for every root it covers: an empty answer + /// is final, not a cue to re-walk (a walk costs a read per file on + /// remote backends). An index that may be stale must not be offered. + async fn test_grep_recursive_indexed_search_empty_answer_is_final() { let grep = Grep; let inner = InMemoryFs::new(); inner.mkdir(Path::new("/safe"), true).await.unwrap(); @@ -2232,8 +2273,8 @@ mod tests { }; let result = grep.execute(ctx).await.unwrap(); - assert_eq!(result.exit_code, 0); - assert!(result.stdout.contains("/other/hit.txt:SECRET")); + assert_eq!(result.exit_code, 1); + assert_eq!(result.stdout, ""); } #[tokio::test] diff --git a/crates/bashkit/src/builtins/mod.rs b/crates/bashkit/src/builtins/mod.rs index 0f0fd39ea..81caf4dbb 100644 --- a/crates/bashkit/src/builtins/mod.rs +++ b/crates/bashkit/src/builtins/mod.rs @@ -136,6 +136,7 @@ mod verify; #[cfg(feature = "terminal")] mod vi; mod wait; +mod walk_cache; mod wc; mod yes; #[cfg(feature = "jq")] diff --git a/crates/bashkit/src/builtins/rg/mod.rs b/crates/bashkit/src/builtins/rg/mod.rs index e70060aa3..5d7dabcd4 100644 --- a/crates/bashkit/src/builtins/rg/mod.rs +++ b/crates/bashkit/src/builtins/rg/mod.rs @@ -14,12 +14,21 @@ //! rg -m NUM PATTERN file # max count per file //! rg --no-filename PATTERN # suppress filename //! rg --color never PATTERN # disable ANSI color output +//! +//! Decisions (round trips; remote VFS backends pay ~one network call per FS +//! call, see `knowledge/foundations/vfs.md`): +//! - A `SearchCapable` index answers even with ignore rules on: matches are +//! filtered through the walk's rules ([`RgIndexIgnore`]), loaded only along +//! the directories that lead to a match. +//! - The walk lists a directory before loading its ignore files and opens +//! only those the listing shows; "inside a git repo" is carried down the +//! walk instead of a `stat` per ancestor. use async_trait::async_trait; use fancy_regex::RegexBuilder as FancyRegexBuilder; use regex::{Regex, RegexBuilder}; use serde_json::json; -use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::io::Read; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -3004,18 +3013,20 @@ async fn read_rg_text_file( async fn collect_rg_inputs( ctx: Context<'_>, opts: &RgOptions, + budget: &mut RgReadBudget, ) -> std::result::Result { if opts.paths.is_empty() { if !opts.stdin_consumed_for_patterns && let Some(stdin) = ctx.stdin { + let stdin = stdin.to_string(); + budget.charge(stdin.len()); return Ok(RgCollectedInputs::new(vec![RgInput::explicit( - "(stdin)", - stdin.to_string(), + "(stdin)", stdin, )])); } - if let Some(inputs) = try_indexed_search(&*ctx.fs, opts, ctx.cwd).await { + if let Some(inputs) = try_indexed_search(&*ctx.fs, opts, ctx.cwd, budget).await { return Ok(RgCollectedInputs::new(inputs)); } @@ -3025,10 +3036,10 @@ async fn collect_rg_inputs( display_hint: RgDisplayHint::None, }; let files = collect_rg_files_recursive(&*ctx.fs, &[root], opts, ctx.cwd).await; - return Ok(read_rg_files(&*ctx.fs, files, ctx.cwd, opts).await); + return Ok(read_rg_files(&*ctx.fs, files, ctx.cwd, opts, budget).await); } - if let Some(inputs) = try_indexed_search(&*ctx.fs, opts, ctx.cwd).await { + if let Some(inputs) = try_indexed_search(&*ctx.fs, opts, ctx.cwd, budget).await { return Ok(RgCollectedInputs::new(inputs)); } @@ -3056,6 +3067,9 @@ async fn collect_rg_inputs( }); } None => { + if budget.exhausted() { + break; + } let text = match read_rg_text_file(&*ctx.fs, &path, ctx.cwd, p, opts).await { Ok(t) => t, Err(e) => { @@ -3066,6 +3080,7 @@ async fn collect_rg_inputs( continue; } }; + budget.charge(text.len()); collected.inputs.push(RgInput::explicit( apply_path_separator_to_display(p, opts), text, @@ -3079,13 +3094,60 @@ async fn collect_rg_inputs( if opts.sort != RgSort::Path || opts.sort_reverse { sort_rg_candidates(&mut candidates, opts); } - let read = read_rg_files(&*ctx.fs, candidates, ctx.cwd, opts).await; + let read = read_rg_files(&*ctx.fs, candidates, ctx.cwd, opts, budget).await; collected.had_errors |= read.had_errors; collected.stderr.push_str(&read.stderr); collected.inputs.extend(read.inputs); Ok(collected) } +/// Charges file contents to the execution budget as they are read. +/// +/// THREAT[TM-DOS-140]: rg holds every searched file's text until output is +/// built, so a tree bigger than the input budget must stop at the budget, +/// not load everything first and fail after. Reading stops at the first +/// refused charge; the caller turns it into the limit error. +struct RgReadBudget { + budget: Option, + exceeded: Option, +} + +impl RgReadBudget { + fn new(budget: Option) -> Self { + Self { + budget, + exceeded: None, + } + } + + /// Charge one file's text; false once the budget is spent. + fn charge(&mut self, bytes: usize) -> bool { + if self.exceeded.is_some() { + return false; + } + let Some(budget) = &self.budget else { + return true; + }; + let work = u64::try_from(bytes.div_ceil(64)) + .unwrap_or(u64::MAX) + .saturating_add(1); + match budget + .consume_input(bytes) + .and_then(|()| budget.consume_work(work)) + { + Ok(()) => true, + Err(e) => { + self.exceeded = Some(e); + false + } + } + } + + fn exhausted(&self) -> bool { + self.exceeded.is_some() + } +} + #[derive(Default)] struct RgCollectedInputs { inputs: Vec, @@ -3158,7 +3220,7 @@ async fn load_rg_ignore_files( for ignore_file in opts.ignore_file_paths.clone() { let path = resolve_path(cwd, &ignore_file); - let content = match read_rg_ignore_file(fs, &path).await { + let content = match read_rg_ignore_file(fs, &path, None).await { Ok(content) => content, Err(Error::Execution(message)) if message.starts_with("rg: ignore file") => { return Err(ExecResult::err(format!("{message}\n"), 2)); @@ -3316,9 +3378,47 @@ fn parse_rg_ignore_rules( Ok(rules) } +/// Names of the per-directory ignore files, lowest precedence first. +const RG_VCS_IGNORE_FILE: &str = ".gitignore"; +const RG_DOT_IGNORE_FILES: [&str; 2] = [".ignore", ".rgignore"]; + +/// Whether a directory listing holds a `.git` directory. +fn listing_has_git_dir(listing: &[crate::fs::DirEntry]) -> bool { + listing + .iter() + .any(|e| e.name == ".git" && e.metadata.file_type.is_dir()) +} + +/// What a directory listing says about an ignore file: `None` when the +/// listing shows it is absent (skip it, no round trip), otherwise the size +/// hint to validate against (`None` inside: unknown, `stat` it first). +fn listed_ignore_file(listing: Option<&[crate::fs::DirEntry]>, name: &str) -> Option> { + let Some(listing) = listing else { + return Some(None); + }; + let entry = listing.iter().find(|e| e.name == name)?; + Some( + entry + .metadata + .file_type + .is_file() + .then_some(entry.metadata.size), + ) +} + +/// Load `dir`'s own ignore files. +/// +/// `listing` is `dir`'s `read_dir` result when the caller already has it: +/// ignore files it does not list are skipped without a lookup, so a walk +/// costs one `read_dir` per directory plus one read per ignore file that +/// exists. `in_git` is whether `dir` or an ancestor up to `root` holds a +/// `.git` directory, when the caller tracks it; otherwise it is probed. +#[allow(clippy::too_many_arguments)] async fn load_local_ignore_rules( fs: &dyn crate::fs::FileSystem, dir: &Path, + listing: Option<&[crate::fs::DirEntry]>, + in_git: Option, root: &Path, opts: &RgOptions, inherited_rule_count: usize, @@ -3329,11 +3429,31 @@ async fn load_local_ignore_rules( } // Ignore evaluation is last-match-wins, so load lower-precedence files first. - if !opts.no_ignore_vcs && (!opts.require_git || has_git_dir_in_ancestors(fs, dir, root).await) { - if !opts.no_ignore_exclude { + let git_applies = !opts.no_ignore_vcs + && (!opts.require_git + || match in_git { + Some(in_git) => in_git, + None => has_git_dir_in_ancestors(fs, dir, root).await, + }); + if git_applies { + let has_git = listing.is_none_or(listing_has_git_dir); + if !opts.no_ignore_exclude && has_git { load_optional_ignore_file_with_rule_base( fs, &vfs_join(dir, ".git/info/exclude"), + None, + dir, + opts.ignore_file_case_insensitive, + inherited_rule_count, + rules, + ) + .await?; + } + if let Some(size) = listed_ignore_file(listing, RG_VCS_IGNORE_FILE) { + load_optional_ignore_file_with_rule_base( + fs, + &vfs_join(dir, RG_VCS_IGNORE_FILE), + size, dir, opts.ignore_file_case_insensitive, inherited_rule_count, @@ -3341,35 +3461,22 @@ async fn load_local_ignore_rules( ) .await?; } - load_optional_ignore_file_with_rule_base( - fs, - &vfs_join(dir, ".gitignore"), - dir, - opts.ignore_file_case_insensitive, - inherited_rule_count, - rules, - ) - .await?; } if !opts.no_ignore_dot { - load_optional_ignore_file_with_rule_base( - fs, - &vfs_join(dir, ".ignore"), - dir, - opts.ignore_file_case_insensitive, - inherited_rule_count, - rules, - ) - .await?; - load_optional_ignore_file_with_rule_base( - fs, - &vfs_join(dir, ".rgignore"), - dir, - opts.ignore_file_case_insensitive, - inherited_rule_count, - rules, - ) - .await?; + for name in RG_DOT_IGNORE_FILES { + if let Some(size) = listed_ignore_file(listing, name) { + load_optional_ignore_file_with_rule_base( + fs, + &vfs_join(dir, name), + size, + dir, + opts.ignore_file_case_insensitive, + inherited_rule_count, + rules, + ) + .await?; + } + } } Ok(()) } @@ -3390,8 +3497,26 @@ async fn load_parent_ignore_rules( let mut ancestors: Vec = parent.ancestors().map(Path::to_path_buf).collect(); ancestors.reverse(); + // Top-down, so "a `.git` at or above this directory" accumulates from + // the listings instead of a `stat` per ancestor per level. + let mut in_git = false; for ancestor in ancestors { - load_local_ignore_rules(fs, &ancestor, Path::new("/"), opts, 0, rules).await?; + let listing = fs.read_dir(&ancestor).await.ok(); + in_git = match &listing { + Some(listing) => in_git || listing_has_git_dir(listing), + None => has_git_dir_in_ancestors(fs, &ancestor, Path::new("/")).await, + }; + load_local_ignore_rules( + fs, + &ancestor, + listing.as_deref(), + Some(in_git), + Path::new("/"), + opts, + 0, + rules, + ) + .await?; } Ok(()) } @@ -3419,18 +3544,19 @@ async fn load_optional_ignore_file( case_insensitive: bool, rules: &mut Vec, ) -> Result<()> { - load_optional_ignore_file_with_rule_base(fs, path, base, case_insensitive, 0, rules).await + load_optional_ignore_file_with_rule_base(fs, path, None, base, case_insensitive, 0, rules).await } async fn load_optional_ignore_file_with_rule_base( fs: &dyn crate::fs::FileSystem, path: &Path, + size_hint: Option, base: &Path, case_insensitive: bool, inherited_rule_count: usize, rules: &mut Vec, ) -> Result<()> { - let content = match read_rg_ignore_file(fs, path).await { + let content = match read_rg_ignore_file(fs, path, size_hint).await { Ok(content) => content, Err(Error::Execution(message)) if message.starts_with("rg: ignore file") => { return Err(Error::Execution(message)); @@ -3442,9 +3568,20 @@ async fn load_optional_ignore_file_with_rule_base( append_rg_ignore_rules_with_base(rules, parsed, inherited_rule_count) } -async fn read_rg_ignore_file(fs: &dyn crate::fs::FileSystem, path: &Path) -> Result> { - if let Ok(meta) = fs.stat(path).await { - validate_rg_ignore_file_size(path, meta.size)?; +/// Read an ignore file, refusing oversized ones before loading them. +/// `size_hint` is the size from a directory listing, which saves the `stat`. +async fn read_rg_ignore_file( + fs: &dyn crate::fs::FileSystem, + path: &Path, + size_hint: Option, +) -> Result> { + match size_hint { + Some(size) => validate_rg_ignore_file_size(path, size)?, + None => { + if let Ok(meta) = fs.stat(path).await { + validate_rg_ignore_file_size(path, meta.size)?; + } + } } let content = fs.read_file(path).await?; validate_rg_ignore_file_size(path, content.len() as u64)?; @@ -3551,6 +3688,8 @@ struct RgWalkItem { rules: Arc, ancestors: Vec, display_hint: RgDisplayHint, + /// A `.git` directory sits in an ancestor between the walk root and here. + in_git: bool, } #[derive(Clone, Copy)] @@ -3636,26 +3775,33 @@ async fn collect_rg_files_recursive( rules: Arc::new(RgIgnoreRuleSet::root(rules)), ancestors: vec![root.actual.clone()], display_hint: root.display_hint, + in_git: false, }); } while let Some(item) = stack.pop() { - let mut local_rules = Vec::new(); - let _ = load_local_ignore_rules( - fs, - &item.actual, - &item.actual_root, - opts, - item.rules.len(), - &mut local_rules, - ) - .await; - let rules = if local_rules.is_empty() { - item.rules.clone() - } else { - Arc::new(RgIgnoreRuleSet::child(item.rules.clone(), local_rules)) - }; - if let Ok(entries) = fs.read_dir(&item.actual).await { + // List first: the listing says which ignore files exist, so missing + // ones cost no lookup (one `read_dir` per directory on remote VFSes). + let listing = fs.read_dir(&item.actual).await; + if let Ok(entries) = listing { + let in_git = item.in_git || listing_has_git_dir(&entries); + let mut local_rules = Vec::new(); + let _ = load_local_ignore_rules( + fs, + &item.actual, + Some(&entries), + Some(in_git), + &item.actual_root, + opts, + item.rules.len(), + &mut local_rules, + ) + .await; + let rules = if local_rules.is_empty() { + item.rules.clone() + } else { + Arc::new(RgIgnoreRuleSet::child(item.rules.clone(), local_rules)) + }; for entry in entries { let path = vfs_join(&item.logical, &entry.name); let actual_path = vfs_join(&item.actual, &entry.name); @@ -3692,6 +3838,7 @@ async fn collect_rg_files_recursive( rules: rules.clone(), ancestors: child_ancestors, display_hint: item.display_hint, + in_git, }); } } else if entry_metadata.file_type.is_file() @@ -3839,11 +3986,16 @@ async fn read_rg_files( files: Vec, cwd: &Path, opts: &RgOptions, + budget: &mut RgReadBudget, ) -> RgCollectedInputs { let mut collected = RgCollectedInputs::default(); for file in files { + if budget.exhausted() { + break; + } let display = candidate_display_path(&file, cwd, opts); match read_rg_text_file(fs, &file.actual, cwd, &display, opts).await { + Ok(text) if !budget.charge(text.len()) => break, Ok(text) if file.explicit => collected.inputs.push(RgInput::explicit(display, text)), Ok(text) => collected.inputs.push(RgInput::discovered(display, text)), Err(e) => { @@ -3867,12 +4019,12 @@ async fn try_indexed_search( fs: &dyn crate::fs::FileSystem, opts: &RgOptions, cwd: &Path, + budget: &mut RgReadBudget, ) -> Option> { let index_can_use_literal = opts.fixed_strings && !opts.word_boundary && !opts.line_regexp; if opts.invert_match || opts.files_without_matches || opts.crlf - || opts.uses_ignore_files() || opts.patterns.len() != 1 || opts.max_filesize.is_some() || !opts.type_includes.is_empty() @@ -3912,12 +4064,11 @@ async fn try_indexed_search( if !caps.content_search || (!opts.fixed_strings && !caps.regex) { return None; } - let explicit_file_root = root_arg.is_some() - && fs - .stat(&root) - .await - .ok() - .is_some_and(|meta| meta.file_type.is_file()); + // A missing operand goes to the walk, which reports it like ripgrep. + let explicit_file_root = match root_arg { + Some(_) => fs.stat(&root).await.ok()?.file_type.is_file(), + None => false, + }; let pattern = if index_can_use_literal { opts.patterns[0].clone() } else { @@ -3937,6 +4088,9 @@ async fn try_indexed_search( }; let results = provider.search(&query).ok()?; + // Built on the first match that needs it: no matches, no listings. + let mut ignore: Option> = None; + let uses_ignore = opts.uses_ignore_files() && !explicit_file_root; for m in &results.matches { let candidate = if m.path.is_absolute() { crate::fs::normalize_path(&m.path) @@ -3950,12 +4104,29 @@ async fn try_indexed_search( || (!explicit_file_match && !opts.matches_globs(&candidate, cwd)) || (!explicit_file_match && !path_allowed_by_hidden_filter(&candidate, &root, opts, cwd)) + || (!explicit_file_match && !within_rg_max_depth(&candidate, &root, opts)) { continue; } + if uses_ignore && !explicit_file_match { + if ignore.is_none() { + ignore = Some(RgIndexIgnore::new(fs, &root, opts).await); + } + if let Some(ignore) = ignore.as_mut() + && !ignore.allows(&candidate).await + { + continue; + } + } + if budget.exhausted() { + break; + } if let Ok(content) = fs.read_file(&candidate).await { let display = display_path_for(&candidate, cwd, root_arg.as_deref(), opts); let content = decode_rg_content(&content, opts); + if !budget.charge(content.len()) { + break; + } let input = if explicit_file_match { RgInput::explicit(display, content) } else { @@ -3969,6 +4140,123 @@ async fn try_indexed_search( Some(inputs) } +/// `--max-depth` for an index match: the root's children are depth 1. +fn within_rg_max_depth(path: &Path, root: &Path, opts: &RgOptions) -> bool { + opts.max_depth.is_none_or(|max_depth| { + path.strip_prefix(root) + .map(|rel| rel.components().count() <= max_depth) + .unwrap_or(false) + }) +} + +/// Ignore rules for index matches, loaded only along the directories that +/// lead to a match. +/// +/// The walk loads every directory's ignore files; an index answer only needs +/// the directories between the root and each matching file. Each such +/// directory is listed once (the listing says which ignore files exist and +/// whether it is a git repository), and a directory that the rules exclude +/// is never listed: everything below it is dropped. The rules, their order +/// and their precedence are the walk's (`collect_rg_files_recursive`). +struct RgIndexIgnore<'a> { + fs: &'a dyn crate::fs::FileSystem, + opts: &'a RgOptions, + root: PathBuf, + /// Rules in effect inside each visited directory and its `in_git` flag; + /// `None` marks a directory that is ignored (or hidden), with its subtree. + dirs: HashMap, bool)>>, +} + +impl<'a> RgIndexIgnore<'a> { + async fn new(fs: &'a dyn crate::fs::FileSystem, root: &Path, opts: &'a RgOptions) -> Self { + let mut rules = Vec::new(); + append_global_ignore_rules_for_root(fs, root, opts, &mut rules).await; + rules.extend(opts.explicit_ignore_rules.clone()); + let _ = load_parent_ignore_rules(fs, root, opts, &mut rules).await; + let base = Arc::new(RgIgnoreRuleSet::root(rules)); + let mut this = Self { + fs, + opts, + root: root.to_path_buf(), + dirs: HashMap::new(), + }; + let state = this.enter(root, base, false).await; + this.dirs.insert(root.to_path_buf(), Some(state)); + this + } + + /// Rules in effect inside `dir`, given the rules of its parent. + async fn enter( + &self, + dir: &Path, + parent_rules: Arc, + parent_in_git: bool, + ) -> (Arc, bool) { + let listing = self.fs.read_dir(dir).await.ok(); + let in_git = parent_in_git || listing.as_deref().is_some_and(listing_has_git_dir); + let mut local = Vec::new(); + let _ = load_local_ignore_rules( + self.fs, + dir, + listing.as_deref(), + Some(in_git), + &self.root, + self.opts, + parent_rules.len(), + &mut local, + ) + .await; + let rules = if local.is_empty() { + parent_rules + } else { + Arc::new(RgIgnoreRuleSet::child(parent_rules, local)) + }; + (rules, in_git) + } + + /// Whether the walk would have searched `path`, a file under the root. + async fn allows(&mut self, path: &Path) -> bool { + let Ok(rel) = path.strip_prefix(&self.root) else { + return false; + }; + let names: Vec = rel + .components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .collect(); + let Some((_, dirs)) = names.split_last() else { + return false; + }; + let mut dir = self.root.clone(); + let Some(Some((mut rules, mut in_git))) = self.dirs.get(&dir).cloned() else { + return false; + }; + for name in dirs { + let child = vfs_join(&dir, name); + let state = match self.dirs.get(&child) { + Some(state) => state.clone(), + None => { + let state = if (!self.opts.hidden && is_hidden_name(name)) + || rules.is_ignored(&child, true) + { + None + } else { + Some(self.enter(&child, rules.clone(), in_git).await) + }; + self.dirs.insert(child.clone(), state.clone()); + state + } + }; + let Some((child_rules, child_in_git)) = state else { + return false; + }; + rules = child_rules; + in_git = child_in_git; + dir = child; + } + !rules.is_ignored(path, false) + } +} + fn path_allowed_by_hidden_filter(path: &Path, root: &Path, opts: &RgOptions, cwd: &Path) -> bool { // Match ripgrep: positive globs unhide hidden files, not hidden directories. if opts.hidden { @@ -5324,19 +5612,13 @@ impl Builtin for Rg { let execution_budget = ctx .execution_budget() .and_then(|budget| budget.try_with(Clone::clone).ok()); - let collected_inputs = match collect_rg_inputs(ctx, &opts).await { + let mut read_budget = RgReadBudget::new(execution_budget); + let collected_inputs = match collect_rg_inputs(ctx, &opts, &mut read_budget).await { Ok(inputs) => inputs, Err(result) => return Ok(result), }; - if let Some(budget) = execution_budget { - let input_bytes = collected_inputs.inputs.iter().fold(0usize, |total, input| { - total.saturating_add(input.content.len()) - }); - budget.consume_input(input_bytes)?; - budget.consume_work( - u64::try_from(input_bytes.div_ceil(64)).unwrap_or(u64::MAX) - + u64::try_from(collected_inputs.inputs.len()).unwrap_or(u64::MAX), - )?; + if let Some(exceeded) = read_budget.exceeded { + return Err(exceeded.into()); } let inputs = collected_inputs.inputs; diff --git a/crates/bashkit/src/builtins/walk_cache.rs b/crates/bashkit/src/builtins/walk_cache.rs new file mode 100644 index 000000000..2c873382c --- /dev/null +++ b/crates/bashkit/src/builtins/walk_cache.rs @@ -0,0 +1,137 @@ +//! Bounded `FileSystem::walk` prefetch for tree walkers (`find`, `du`). +//! THREAT[TM-DOS-140]: walks over millions of files hold one batch. +//! +//! Decisions: +//! - Listings come in bounded batches (`WalkOptions::max_entries`), never the +//! whole tree: a directory missing from the cache triggers another `walk` +//! rooted at it, so memory stays at about one batch however large the tree, +//! and bulk backends still answer each batch in one query. +//! - Listings are removed as they are consumed. If an answer does not hold the +//! requested directory (path spelling mismatch), bulk walking is turned off +//! for the run and plain `read_dir` takes over, so a mismatch can never turn +//! into one batch-sized walk per directory. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use crate::error::Result; +use crate::fs::{DirEntry, FileSystem, WalkOptions}; + +/// Directory listings fetched ahead in bounded `walk` batches. +pub(crate) struct WalkCache { + listings: HashMap>, + enabled: bool, + max_entries: usize, +} + +impl WalkCache { + /// `enabled = false` makes every lookup a plain `read_dir`. + pub(crate) fn new(enabled: bool) -> Self { + Self { + listings: HashMap::new(), + enabled, + max_entries: WalkOptions::DEFAULT_MAX_ENTRIES, + } + } + + /// Stop prefetching (an action may change or cut the traversal). + pub(crate) fn disable(&mut self) { + self.enabled = false; + self.listings.clear(); + } + + /// Listing of `dir`. `max_depth` counts directory levels still to list + /// from `dir` (`None` = unlimited; `dir` itself is level 1). + pub(crate) async fn list( + &mut self, + fs: &dyn FileSystem, + dir: &Path, + max_depth: Option, + ) -> Result> { + if let Some(list) = self.listings.remove(dir) { + return Ok(list); + } + if self.enabled && max_depth != Some(0) { + let opts = WalkOptions::new(max_depth).max_entries(self.max_entries); + if let Ok(batch) = fs.walk(dir, opts).await { + let mut own = None; + for listing in batch { + if listing.path == dir { + own = Some(listing.entries); + } else { + self.listings.insert(listing.path, listing.entries); + } + } + match own { + Some(list) => return Ok(list), + None => self.disable(), + } + } + } + fs.read_dir(dir).await + } + + /// Entries currently held (for tests and assertions). + #[cfg(test)] + pub(crate) fn held(&self) -> usize { + self.listings.values().map(Vec::len).sum() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fs::InMemoryFs; + + async fn tree(dirs: usize, files: usize) -> InMemoryFs { + let fs = InMemoryFs::new(); + for d in 0..dirs { + for f in 0..files { + let p = PathBuf::from(format!("/t/d{d}/f{f}")); + fs.mkdir(p.parent().unwrap(), true).await.unwrap(); + fs.write_file(&p, b"x").await.unwrap(); + } + } + fs + } + + #[tokio::test] + async fn batches_stay_bounded_and_cover_every_directory() { + let fs = tree(50, 20).await; + let mut cache = WalkCache::new(true); + cache.max_entries = 100; + let mut seen = 0; + let mut peak = 0; + let mut stack = vec![PathBuf::from("/t")]; + while let Some(dir) = stack.pop() { + let list = cache.list(&fs, &dir, None).await.unwrap(); + peak = peak.max(cache.held()); + for e in list { + seen += 1; + if e.metadata.file_type.is_dir() { + stack.push(dir.join(&e.name)); + } + } + } + assert_eq!(seen, 50 + 50 * 20); + // One batch plus at most one listing over the bound. + assert!(peak <= 100 + 50, "peak {peak}"); + assert_eq!(cache.held(), 0); + } + + #[tokio::test] + async fn missing_directory_reports_read_dir_error() { + let fs = tree(1, 1).await; + let mut cache = WalkCache::new(true); + assert!(cache.list(&fs, Path::new("/nope"), None).await.is_err()); + } + + #[tokio::test] + async fn disabled_cache_holds_nothing() { + let fs = tree(5, 5).await; + let mut cache = WalkCache::new(false); + let list = cache.list(&fs, Path::new("/t"), None).await.unwrap(); + assert_eq!(list.len(), 5); + assert_eq!(cache.held(), 0); + } +} diff --git a/crates/bashkit/src/execution_capability.rs b/crates/bashkit/src/execution_capability.rs index 92f8eee3b..4811c7043 100644 --- a/crates/bashkit/src/execution_capability.rs +++ b/crates/bashkit/src/execution_capability.rs @@ -473,8 +473,54 @@ impl crate::FileSystem for ExecutionFileSystem { self.run(self.inner.set_modified_time(path, time)).await? } - // Returning the raw search provider would bypass the execution lease. + async fn walk( + &self, + root: &std::path::Path, + opts: crate::WalkOptions, + ) -> crate::Result> { + self.run(self.inner.walk(root, opts)).await? + } + + // The raw provider would outlive the execution lease, so providers are + // wrapped: every search re-checks the lease (see `LeasedSearchProvider`). fn as_search_capable(&self) -> Option<&dyn crate::SearchCapable> { - None + self.scope.ensure_active().ok()?; + self.inner.as_search_capable()?; + Some(self) + } +} + +impl crate::SearchCapable for ExecutionFileSystem { + fn search_provider(&self, path: &std::path::Path) -> Option> { + self.scope.ensure_active().ok()?; + let inner = self.inner.as_search_capable()?.search_provider(path)?; + Some(Box::new(LeasedSearchProvider { + inner, + scope: Arc::clone(&self.scope), + })) + } +} + +/// Search provider that stops working when its execution lease is revoked, +/// so a builtin cannot keep querying the index after its command returns. +struct LeasedSearchProvider { + inner: Box, + scope: Arc, +} + +impl crate::SearchProvider for LeasedSearchProvider { + fn search(&self, query: &crate::SearchQuery) -> crate::Result { + self.scope + .ensure_active() + .map_err(|_| ExecutionFileSystem::revoked())?; + self.inner.search(query) + } + + fn capabilities(&self) -> crate::SearchCapabilities { + // A revoked lease answers nothing, capabilities included. + if self.scope.ensure_active().is_err() { + return crate::SearchCapabilities::default(); + } + self.inner.capabilities() } } diff --git a/crates/bashkit/src/fs/follow.rs b/crates/bashkit/src/fs/follow.rs index e37b3b5ad..652995d1e 100644 --- a/crates/bashkit/src/fs/follow.rs +++ b/crates/bashkit/src/fs/follow.rs @@ -143,6 +143,16 @@ impl FollowFs { } /// Path to write: the link's target when the final component is a link. + /// `path` with every link resolved, for operations on an existing file. + /// Fast path: one inner `stat` shows no link is involved (inner layers + /// fail on a link in a parent and report a final link as a link). + async fn resolve_existing(&self, path: &Path) -> Result { + match self.inner.stat(path).await { + Ok(meta) if !meta.file_type.is_symlink() => Ok(normalize_path(path)), + _ => self.resolve(path, true).await, + } + } + async fn write_target(&self, path: &Path) -> Result { match self.inner.stat(path).await { Ok(meta) if meta.file_type.is_symlink() => self.resolve(path, true).await, @@ -261,6 +271,32 @@ impl FileSystem for FollowFs { retry_resolved!(self, followed, path, |p| self.inner.read_dir(p).await) } + async fn walk(&self, root: &Path, opts: super::WalkOptions) -> Result> { + // Inner layers never follow, so only `root` itself may need resolving; + // listings below it come back with link entries unresolved, the same + // as `read_dir`. + match self.inner.walk(root, opts).await { + Err(e) if may_be_link_error(&e) => match self.followed(root).await? { + Some(resolved) => { + let mut listings = self.inner.walk(&resolved, opts).await?; + let root = normalize_path(root); + for listing in &mut listings { + if let Ok(rest) = listing.path.strip_prefix(&resolved) { + listing.path = if rest.as_os_str().is_empty() { + root.clone() + } else { + super::vfs_join(&root, rest) + }; + } + } + Ok(listings) + } + None => Err(e), + }, + other => other, + } + } + async fn exists(&self, path: &Path) -> Result { // `test -e` follows links: a dangling link does not exist. match self.stat(path).await { @@ -290,6 +326,16 @@ impl FileSystem for FollowFs { } async fn copy(&self, from: &Path, to: &Path) -> Result<()> { + // Fast path: `from` is no link and `to` is no link (or missing), so + // the inner layers copy as given. Resolving both paths up front costs + // one `stat` per path component, a round trip each on remote backends. + let from_plain = matches!(self.inner.stat(from).await, Ok(m) if !m.file_type.is_symlink()); + if from_plain && !matches!(self.inner.stat(to).await, Ok(m) if m.file_type.is_symlink()) { + match self.inner.copy(from, to).await { + Err(e) if may_be_link_error(&e) => {} + other => return other, + } + } let from_r = self.resolve(from, true).await?; let to_r = self.write_target(to).await?; self.inner.copy(&from_r, &to_r).await @@ -310,12 +356,12 @@ impl FileSystem for FollowFs { } async fn chmod(&self, path: &Path, mode: u32) -> Result<()> { - let resolved = self.resolve(path, true).await?; + let resolved = self.resolve_existing(path).await?; self.inner.chmod(&resolved, mode).await } async fn set_modified_time(&self, path: &Path, time: SystemTime) -> Result<()> { - let resolved = self.resolve(path, true).await?; + let resolved = self.resolve_existing(path).await?; self.inner.set_modified_time(&resolved, time).await } diff --git a/crates/bashkit/src/fs/mod.rs b/crates/bashkit/src/fs/mod.rs index 892acab5e..10da9874e 100644 --- a/crates/bashkit/src/fs/mod.rs +++ b/crates/bashkit/src/fs/mod.rs @@ -437,7 +437,10 @@ pub use search::{ SearchCapabilities, SearchCapable, SearchMatch, SearchProvider, SearchQuery, SearchResults, }; pub(crate) use std_streams::{StdStreamsFs, args_name_std_stream}; -pub use traits::{DirEntry, FileSystem, FileSystemExt, FileType, Metadata, fs_errors}; +pub use traits::{ + DirEntry, DirListing, FileSystem, FileSystemExt, FileType, Metadata, WalkOptions, fs_errors, + walk_with_read_dir, +}; use crate::error::Result; use std::io::{Error as IoError, ErrorKind}; diff --git a/crates/bashkit/src/fs/mountable.rs b/crates/bashkit/src/fs/mountable.rs index a822f807e..973eed1e4 100644 --- a/crates/bashkit/src/fs/mountable.rs +++ b/crates/bashkit/src/fs/mountable.rs @@ -12,6 +12,7 @@ use async_trait::async_trait; use std::collections::BTreeMap; use std::io::Error as IoError; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, RwLock}; use super::limits::{FsLimits, FsUsage}; @@ -156,6 +157,10 @@ pub struct MountableFs { /// Mount points: path -> filesystem /// BTreeMap ensures iteration in path order mounts: RwLock>>, + /// Bumped on every mount and unmount; search providers handed out before + /// a change refuse to answer, so callers walk instead of trusting an index + /// that no longer covers the whole subtree. + generation: Arc, } enum CrossMountEntry { @@ -293,6 +298,7 @@ impl MountableFs { Self { root, mounts: RwLock::new(BTreeMap::new()), + generation: Arc::new(AtomicU64::new(0)), } } @@ -353,6 +359,7 @@ impl MountableFs { let path = Self::normalize_path(path.as_ref()); let mut mounts = self.mounts.write().unwrap(); mounts.insert(path, fs); + self.generation.fetch_add(1, Ordering::SeqCst); Ok(()) } @@ -399,6 +406,7 @@ impl MountableFs { mounts .remove(&path) .ok_or_else(|| IoError::other("mount not found"))?; + self.generation.fetch_add(1, Ordering::SeqCst); Ok(()) } @@ -464,6 +472,107 @@ impl MountableFs { } } } + + /// Mount point that serves `path` (`None`: the root filesystem). + fn mount_point_of(&self, path: &Path) -> Option { + let mounts = self.mounts.read().unwrap(); + mounts + .keys() + .filter(|m| path.starts_with(m)) + .max_by_key(|m| m.components().count()) + .cloned() + } + + /// Whether a mount sits strictly below `path`, so one backend cannot + /// answer a subtree query for it alone. + fn has_mount_below(&self, path: &Path) -> bool { + let mounts = self.mounts.read().unwrap(); + mounts.keys().any(|m| m != path && m.starts_with(path)) + } +} + +/// Map a backend-relative path back under its mount point. +fn under_mount(mount: Option<&Path>, inner: &Path) -> PathBuf { + match mount { + None => inner.to_path_buf(), + Some(m) => { + let rel = inner.strip_prefix("/").unwrap_or(inner); + if rel.as_os_str().is_empty() { + m.to_path_buf() + } else { + vfs_join(m, rel) + } + } + } +} + +/// Search provider of a mounted backend, translating paths at the mount. +struct MountedSearchProvider { + inner: Box, + mount: Option, + /// Mount table generation when the provider was handed out. + generation: u64, + current: Arc, +} + +impl super::SearchProvider for MountedSearchProvider { + fn search(&self, query: &super::SearchQuery) -> Result { + // A mount added or removed since `search_provider` may sit below the + // root: the index would no longer cover the subtree. Erroring makes + // callers fall back to the walk instead of trusting a partial answer. + if self.current.load(Ordering::SeqCst) != self.generation { + return Err(IoError::other("mount table changed during search").into()); + } + let Some(mount) = &self.mount else { + return self.inner.search(query); + }; + let root = MountableFs::normalize_path(&query.root); + // This backend only holds paths under its mount point. + let Ok(rel) = root.strip_prefix(mount) else { + return Ok(super::SearchResults::default()); + }; + let mut translated = query.clone(); + translated.root = vfs_join(Path::new("/"), rel); + let mut results = self.inner.search(&translated)?; + for m in &mut results.matches { + let inner = if is_posix_absolute(&m.path) { + m.path.clone() + } else { + vfs_join(&translated.root, &m.path) + }; + m.path = under_mount(Some(mount), &inner); + } + Ok(results) + } + + fn capabilities(&self) -> super::SearchCapabilities { + self.inner.capabilities() + } +} + +/// Indexed search passes through to the backend that serves the whole +/// requested subtree. A subtree with another mount below it has no single +/// index, so callers fall back to walking. +impl super::SearchCapable for MountableFs { + fn search_provider(&self, path: &Path) -> Option> { + self.validate_path(path).ok()?; + let path = Self::normalize_path(path); + // Read before checking the table: a change after this read fails the + // provider's `search`. + let generation = self.generation.load(Ordering::SeqCst); + if self.has_mount_below(&path) { + return None; + } + let mount = self.mount_point_of(&path); + let (fs, resolved) = self.resolve(&path); + let inner = fs.as_search_capable()?.search_provider(&resolved)?; + Some(Box::new(MountedSearchProvider { + inner, + mount, + generation, + current: Arc::clone(&self.generation), + })) + } } #[async_trait] @@ -515,32 +624,52 @@ impl FileSystem for MountableFs { let mut entries = fs.read_dir(&resolved).await?; - // Add mount points that are direct children of this directory - let mounts = self.mounts.read().unwrap(); - for mount_path in mounts.keys() { - if mount_path.parent() == Some(&path) - && let Some(name) = mount_path.file_name() - { - // Check if this entry already exists - let name_str = name.to_string_lossy().to_string(); - if !entries.iter().any(|e| e.name == name_str) { - entries.push(DirEntry { - name: name_str, - metadata: Metadata { - file_type: FileType::Directory, - size: 0, - mode: 0o755, - modified: crate::time_compat::SystemTime::now(), - created: crate::time_compat::SystemTime::now(), - }, - }); - } + // Add mount points that are direct children of this directory. Their + // entry carries the mounted root's metadata, the same `stat` returns, + // so callers can trust listing metadata without a per-entry `stat`. + let children: Vec<(String, Arc)> = { + let mounts = self.mounts.read().unwrap(); + mounts + .iter() + .filter(|(mount_path, _)| mount_path.parent() == Some(&path)) + .filter_map(|(mount_path, mfs)| { + let name = mount_path.file_name()?.to_string_lossy().into_owned(); + Some((name, Arc::clone(mfs))) + }) + .collect() + }; + for (name, mfs) in children { + let metadata = mfs.stat(Path::new("/")).await.unwrap_or_else(|_| Metadata { + file_type: FileType::Directory, + size: 0, + mode: 0o755, + modified: crate::time_compat::SystemTime::now(), + created: crate::time_compat::SystemTime::now(), + }); + match entries.iter_mut().find(|e| e.name == name) { + Some(entry) => entry.metadata = metadata, + None => entries.push(DirEntry { name, metadata }), } } Ok(entries) } + async fn walk(&self, root: &Path, opts: super::WalkOptions) -> Result> { + self.validate_path(root)?; + let root = Self::normalize_path(root); + if self.has_mount_below(&root) { + return super::walk_with_read_dir(self, &root, opts).await; + } + let mount = self.mount_point_of(&root); + let (fs, resolved) = self.resolve(&root); + let mut listings = fs.walk(&resolved, opts).await?; + for listing in &mut listings { + listing.path = under_mount(mount.as_deref(), &listing.path); + } + Ok(listings) + } + async fn exists(&self, path: &Path) -> Result { self.validate_path(path)?; let path = Self::normalize_path(path); @@ -614,6 +743,10 @@ impl FileSystem for MountableFs { let (fs, resolved) = self.resolve(path); fs.set_modified_time(&resolved, time).await } + + fn as_search_capable(&self) -> Option<&dyn super::SearchCapable> { + Some(self) + } } #[async_trait] @@ -663,6 +796,35 @@ impl FileSystemExt for MountableFs { #[cfg(test)] mod tests { + + /// A mount added after a provider was handed out fails its search, so + /// callers walk instead of trusting an index that no longer covers the + /// subtree. + #[tokio::test] + async fn search_provider_refuses_after_mount_table_changes() { + use crate::fs::SearchCapable; + let indexed = Arc::new(crate::testing::CountingFs::new().with_index()); + indexed.mkdir(Path::new("/a"), true).await.unwrap(); + indexed + .write_file(Path::new("/a/f.txt"), b"needle") + .await + .unwrap(); + let fs = MountableFs::new(indexed); + let provider = fs.search_provider(Path::new("/")).expect("indexed root"); + let query = super::super::SearchQuery { + pattern: "needle".into(), + is_regex: false, + case_insensitive: false, + root: "/".into(), + glob_filter: None, + max_results: None, + }; + assert_eq!(provider.search(&query).unwrap().matches.len(), 1); + fs.mount("/a/m", Arc::new(InMemoryFs::new())).unwrap(); + assert!(provider.search(&query).is_err()); + // A fresh provider sees the mount below the root and is withheld. + assert!(fs.search_provider(Path::new("/")).is_none()); + } use super::*; use crate::fs::InMemoryFs; diff --git a/crates/bashkit/src/fs/proc_sub_fds.rs b/crates/bashkit/src/fs/proc_sub_fds.rs index 4fc3969a2..45202c934 100644 --- a/crates/bashkit/src/fs/proc_sub_fds.rs +++ b/crates/bashkit/src/fs/proc_sub_fds.rs @@ -277,6 +277,10 @@ impl FileSystem for ProcSubFs { async fn read_dir(&self, path: &Path) -> Result> { self.inner.read_dir(path).await } + + async fn walk(&self, root: &Path, opts: super::WalkOptions) -> Result> { + self.inner.walk(root, opts).await + } async fn exists(&self, path: &Path) -> Result { match self.slot(path) { Slot::Vfs => self.inner.exists(path).await, diff --git a/crates/bashkit/src/fs/readonly.rs b/crates/bashkit/src/fs/readonly.rs index 0163c4333..c67fb2a82 100644 --- a/crates/bashkit/src/fs/readonly.rs +++ b/crates/bashkit/src/fs/readonly.rs @@ -59,6 +59,10 @@ impl FileSystem for ReadOnlyFs { self.inner.read_dir(path).await } + async fn walk(&self, root: &Path, opts: super::WalkOptions) -> Result> { + self.inner.walk(root, opts).await + } + async fn exists(&self, path: &Path) -> Result { self.inner.exists(path).await } diff --git a/crates/bashkit/src/fs/rootfs.rs b/crates/bashkit/src/fs/rootfs.rs index 8affe9f93..802e535ec 100644 --- a/crates/bashkit/src/fs/rootfs.rs +++ b/crates/bashkit/src/fs/rootfs.rs @@ -469,6 +469,15 @@ impl FileSystem for RootFs { } } + async fn walk(&self, root: &Path, opts: super::WalkOptions) -> Result> { + // Subtrees without system paths are the session filesystem's alone. + let p = normalize_path(root); + if Self::sys_path(&p).is_some() || SYS_TREES.iter().any(|t| Path::new(t).starts_with(&p)) { + return super::walk_with_read_dir(self, root, opts).await; + } + self.inner.walk(root, opts).await + } + async fn exists(&self, path: &Path) -> Result { if self.inner.exists(path).await? { return Ok(true); diff --git a/crates/bashkit/src/fs/search.rs b/crates/bashkit/src/fs/search.rs index 10c5591e4..da593b0b1 100644 --- a/crates/bashkit/src/fs/search.rs +++ b/crates/bashkit/src/fs/search.rs @@ -127,6 +127,13 @@ use crate::error::Result; pub trait SearchCapable: super::FileSystem { /// Returns a search provider scoped to the given path. /// Returns `None` if no index covers this path. + /// + /// A provider is authoritative for `path`: builtins read only the files + /// it reports and treat an empty answer as "no match" without walking + /// the tree, so answers must be complete for the queried root. Return + /// `None` when the index may be stale or partial for this subtree; a + /// provider that finds out later returns an error from `search`, and + /// `rg`/`grep -r` then walk. fn search_provider(&self, path: &Path) -> Option>; } diff --git a/crates/bashkit/src/fs/std_streams.rs b/crates/bashkit/src/fs/std_streams.rs index 8a9cfe77e..216ac551e 100644 --- a/crates/bashkit/src/fs/std_streams.rs +++ b/crates/bashkit/src/fs/std_streams.rs @@ -240,6 +240,10 @@ impl FileSystem for StdStreamsFs { async fn read_dir(&self, path: &Path) -> Result> { self.inner.read_dir(path).await } + + async fn walk(&self, root: &Path, opts: super::WalkOptions) -> Result> { + self.inner.walk(root, opts).await + } async fn exists(&self, path: &Path) -> Result { match std_stream_fd(path) { Some(_) => Ok(true), diff --git a/crates/bashkit/src/fs/traits.rs b/crates/bashkit/src/fs/traits.rs index b50efc63b..6c4b35cec 100644 --- a/crates/bashkit/src/fs/traits.rs +++ b/crates/bashkit/src/fs/traits.rs @@ -417,6 +417,126 @@ pub trait FileSystem: FileSystemExt { fn as_search_capable(&self) -> Option<&dyn super::SearchCapable> { None } + + /// List `root` and the directories below it in one call. + /// + /// Returns one [`DirListing`] per directory: `root` itself (depth 0) and + /// descendant directories at depth `d` with `d + 1 <= opts.max_depth`, so + /// `max_depth = Some(1)` returns only `root`'s listing and `None` walks + /// the whole subtree. Entries carry the same metadata `read_dir` returns. + /// Symlinks are listed, never followed. A descendant directory that + /// cannot be listed is left out; callers that need its error call + /// `read_dir` on it. Listing order is unspecified. + /// + /// The answer is a bounded batch, not the whole tree: implementations + /// stop once the returned listings hold `opts.max_entries` entries or more + /// (going over by at most one listing; a listing is never split). A + /// directory missing from the answer was either cut by that bound or + /// unreadable; callers list it themselves, with `read_dir` or another + /// `walk` rooted there. This keeps a walk over millions of files at one + /// batch of listings in memory. + /// + /// The default issues one `read_dir` per directory. Backends where each + /// call is a network round trip (a database or object store) should + /// override it with one bulk query, e.g. a path-prefix scan with a row + /// limit. `find` and `du` use it when they would list every directory + /// anyway. + /// + /// # Errors + /// + /// Returns the `read_dir` error for `root`. + async fn walk(&self, root: &Path, opts: WalkOptions) -> Result> { + walk_with_read_dir(self, root, opts).await + } +} + +/// Bounds for one [`FileSystem::walk`] answer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WalkOptions { + /// Directory depth below the root to list (`None` = unlimited); see + /// [`FileSystem::walk`]. + pub max_depth: Option, + /// Stop once the answer holds this many entries (at most one listing + /// more). Callers list the rest on demand. + pub max_entries: usize, +} + +impl WalkOptions { + /// Default [`WalkOptions::max_entries`]: about a megabyte of listings, + /// a few hundred round trips saved per batch on remote backends. + pub const DEFAULT_MAX_ENTRIES: usize = 10_000; + + /// Walk to `max_depth` with the default entry bound. + pub fn new(max_depth: Option) -> Self { + Self { + max_depth, + max_entries: Self::DEFAULT_MAX_ENTRIES, + } + } + + /// Replace the entry bound. + pub fn max_entries(mut self, entries: usize) -> Self { + self.max_entries = entries; + self + } +} + +impl Default for WalkOptions { + fn default() -> Self { + Self::new(None) + } +} + +/// One directory listing returned by [`FileSystem::walk`]. +#[derive(Debug, Clone)] +pub struct DirListing { + /// Absolute path of the listed directory. + pub path: std::path::PathBuf, + /// Its entries, as [`FileSystem::read_dir`] returns them. + pub entries: Vec, +} + +/// The default [`FileSystem::walk`]: one `read_dir` per directory, depth +/// first, stopping at `opts.max_entries`. +/// +/// Public so overriding implementations can fall back to it for paths they +/// cannot answer in bulk. Directories are read only when popped, so pending +/// work is a path list no longer than the entries already returned. +pub async fn walk_with_read_dir( + fs: &F, + root: &Path, + opts: WalkOptions, +) -> Result> { + let mut out = Vec::new(); + if opts.max_depth == Some(0) { + return Ok(out); + } + let entries = fs.read_dir(root).await?; + let mut held = 0usize; + let mut pending = vec![(root.to_path_buf(), 0usize, Some(entries))]; + while let Some((path, depth, listed)) = pending.pop() { + if held >= opts.max_entries { + break; + } + let entries = match listed { + Some(entries) => entries, + None => match fs.read_dir(&path).await { + Ok(entries) => entries, + Err(_) => continue, + }, + }; + let child_depth = depth + 1; + if opts.max_depth.is_none_or(|m| child_depth < m) { + for entry in &entries { + if entry.metadata.file_type.is_dir() { + pending.push((super::vfs_join(&path, &entry.name), child_depth, None)); + } + } + } + held = held.saturating_add(entries.len()); + out.push(DirListing { path, entries }); + } + Ok(out) } /// File or directory metadata. diff --git a/crates/bashkit/src/lib.rs b/crates/bashkit/src/lib.rs index 60c89118f..7fbf74cd8 100644 --- a/crates/bashkit/src/lib.rs +++ b/crates/bashkit/src/lib.rs @@ -509,11 +509,11 @@ pub use execution_capability::{ CapabilityCleanupReport, ExecutionCapability, ExecutionCapabilityError, ExecutionExtensions, }; pub use fs::{ - DirEntry, FileSystem, FileSystemExt, FileType, FsBackend, FsLimitExceeded, FsLimits, FsUsage, - InMemoryFs, LazyLoader, Metadata, MountableFs, NamespaceAccess, NamespaceFs, + DirEntry, DirListing, FileSystem, FileSystemExt, FileType, FsBackend, FsLimitExceeded, + FsLimits, FsUsage, InMemoryFs, LazyLoader, Metadata, MountableFs, NamespaceAccess, NamespaceFs, NamespaceFsBuilder, OverlayFs, PosixFs, ReadOnlyFs, SearchCapabilities, SearchCapable, SearchMatch, SearchProvider, SearchQuery, SearchResults, VfsEntry, VfsEntryKind, VfsSnapshot, - normalize_path, verify_filesystem_requirements, vfs_join, + WalkOptions, normalize_path, verify_filesystem_requirements, vfs_join, walk_with_read_dir, }; #[cfg(feature = "realfs")] pub use fs::{RealFs, RealFsMode}; diff --git a/crates/bashkit/src/testing.rs b/crates/bashkit/src/testing.rs index 3a99946b2..a795793ca 100644 --- a/crates/bashkit/src/testing.rs +++ b/crates/bashkit/src/testing.rs @@ -18,6 +18,10 @@ //! fuzzer can forge the canary into its own input — see //! `assert_fuzz_invariants`). +mod counting_fs; + +pub use counting_fs::{CallCounts, CountingFs}; + use crate::{Bash, ControlFlow, ExecResult}; /// Cross-tool banned substrings. Any of these in stderr means a leak — diff --git a/crates/bashkit/src/testing/counting_fs.rs b/crates/bashkit/src/testing/counting_fs.rs new file mode 100644 index 000000000..dd88a878e --- /dev/null +++ b/crates/bashkit/src/testing/counting_fs.rs @@ -0,0 +1,455 @@ +//! Call-counting filesystem for round-trip tests and benches. +//! +//! Decisions: +//! - Models a remote backend (database, object store, network share) where +//! every `FileSystem` call is one round trip. Tests assert call counts; +//! benches add a simulated per-call latency. +//! - Latency is a busy-wait, not `sleep`: tokio timers round up to 1 ms, which +//! would hide sub-millisecond settings and skew bench numbers. +//! - The optional content index is a private map kept next to the inner +//! filesystem (writes go to both). Searching it is not a round trip of the +//! walked tree, the same way a server-side index answers in one query. +//! - `native_walk` makes `walk` one counted call (a prefix query) instead of +//! the `read_dir`-per-directory default, to show what a bulk backend saves. +//! - `walk_page` caps entries per `walk` answer below the caller's +//! `max_entries` (a backend page size), and `walk_peak_entries` records the +//! largest answer, so tests can prove walkers hold a bounded batch. + +use std::collections::BTreeMap; +use std::fmt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, RwLock}; +use std::time::Duration; + +use async_trait::async_trait; + +use crate::error::Result; +use crate::fs::{ + DirEntry, DirListing, FileSystem, FileSystemExt, FsLimits, FsUsage, InMemoryFs, Metadata, + SearchCapabilities, SearchCapable, SearchMatch, SearchProvider, SearchQuery, SearchResults, + WalkOptions, walk_with_read_dir, +}; +use crate::time_compat::SystemTime; + +/// Path -> content map the simulated index searches. +type Index = Arc>>>; + +/// Per-operation call counters, shared with index providers. +#[derive(Default)] +struct Counters { + /// Simulated per-call latency, in nanoseconds. + latency_ns: AtomicU64, + read_file: AtomicU64, + write: AtomicU64, + stat: AtomicU64, + lstat: AtomicU64, + read_dir: AtomicU64, + exists: AtomicU64, + read_link: AtomicU64, + walk: AtomicU64, + search: AtomicU64, + other: AtomicU64, + /// Most entries returned by one `walk` (a peak, not a call count). + walk_peak_entries: AtomicU64, +} + +/// Snapshot of the call counters. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct CallCounts { + /// `read_file` calls. + pub read_file: u64, + /// `write_file`, `append_file`, `mkdir`, `remove`, `rename`, `copy`, + /// `symlink`, `chmod`, `set_modified_time` calls. + pub write: u64, + /// `stat` calls. + pub stat: u64, + /// `lstat` calls. + pub lstat: u64, + /// `read_dir` calls. + pub read_dir: u64, + /// `exists` calls. + pub exists: u64, + /// `read_link` calls. + pub read_link: u64, + /// Native `walk` calls (only counted with `native_walk`). + pub walk: u64, + /// Index `search` calls. + pub search: u64, + /// Anything else. + pub other: u64, + /// Most entries any single `walk` answer held (not a round trip). + pub walk_peak_entries: u64, +} + +impl CallCounts { + /// Every round trip. + pub fn total(&self) -> u64 { + self.read_file + + self.write + + self.stat + + self.lstat + + self.read_dir + + self.exists + + self.read_link + + self.walk + + self.search + + self.other + } + + /// `stat` + `lstat` + `exists`: per-path metadata lookups. + pub fn metadata(&self) -> u64 { + self.stat + self.lstat + self.exists + } +} + +impl fmt::Display for CallCounts { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "total={} read_file={} read_dir={} stat={} lstat={} exists={} read_link={} walk={} search={} write={} other={}", + self.total(), + self.read_file, + self.read_dir, + self.stat, + self.lstat, + self.exists, + self.read_link, + self.walk, + self.search, + self.write, + self.other + ) + } +} + +impl Counters { + fn hit(&self, counter: &AtomicU64) { + counter.fetch_add(1, Ordering::Relaxed); + let ns = self.latency_ns.load(Ordering::Relaxed); + if ns > 0 { + let latency = Duration::from_nanos(ns); + let start = crate::time_compat::Instant::now(); + while start.elapsed() < latency { + std::hint::spin_loop(); + } + } + } +} + +/// A [`FileSystem`] over [`InMemoryFs`] that counts every call. +pub struct CountingFs { + inner: InMemoryFs, + counters: Arc, + index: Option, + native_walk: bool, + walk_page: Option, +} + +impl CountingFs { + /// Plain counting filesystem: no index, no bulk walk, no latency. + pub fn new() -> Self { + Self { + inner: InMemoryFs::new(), + counters: Arc::new(Counters::default()), + index: None, + native_walk: false, + walk_page: None, + } + } + + /// Answer content searches from an index (`SearchCapable`). + pub fn with_index(mut self) -> Self { + self.index = Some(Arc::new(RwLock::new(BTreeMap::new()))); + self + } + + /// Answer `walk` in one counted call instead of one `read_dir` per directory. + pub fn with_native_walk(mut self) -> Self { + self.native_walk = true; + self + } + + /// Return at most `entries` entries per `walk` answer (a backend page + /// size), even when the caller allows more. + pub fn with_walk_page(mut self, entries: usize) -> Self { + self.walk_page = Some(entries); + self + } + + /// Busy-wait this long on every call. + pub fn with_latency(self, latency: Duration) -> Self { + self.set_latency(latency); + self + } + + /// Change the simulated per-call latency (e.g. after seeding). + pub fn set_latency(&self, latency: Duration) { + let ns = u64::try_from(latency.as_nanos()).unwrap_or(u64::MAX); + self.counters.latency_ns.store(ns, Ordering::Relaxed); + } + + /// Use these limits for the inner in-memory filesystem. + pub fn with_limits(mut self, limits: FsLimits) -> Self { + self.inner = InMemoryFs::with_limits(limits); + self + } + + /// Current counters. + pub fn counts(&self) -> CallCounts { + let c = &self.counters; + let get = |a: &AtomicU64| a.load(Ordering::Relaxed); + CallCounts { + read_file: get(&c.read_file), + write: get(&c.write), + stat: get(&c.stat), + lstat: get(&c.lstat), + read_dir: get(&c.read_dir), + exists: get(&c.exists), + read_link: get(&c.read_link), + walk: get(&c.walk), + search: get(&c.search), + other: get(&c.other), + walk_peak_entries: get(&c.walk_peak_entries), + } + } + + /// Zero every counter. + pub fn reset(&self) { + let c = &self.counters; + for a in [ + &c.read_file, + &c.write, + &c.stat, + &c.lstat, + &c.read_dir, + &c.exists, + &c.read_link, + &c.walk, + &c.search, + &c.other, + &c.walk_peak_entries, + ] { + a.store(0, Ordering::Relaxed); + } + } + + fn hit(&self, counter: &AtomicU64) { + self.counters.hit(counter); + } + + fn index_put(&self, path: &Path, content: Option<&[u8]>, append: bool) { + let Some(index) = &self.index else { return }; + let path = crate::fs::normalize_path(path); + let mut map = index.write().unwrap_or_else(|e| e.into_inner()); + match content { + Some(bytes) if append => map.entry(path).or_default().extend_from_slice(bytes), + Some(bytes) => { + map.insert(path, bytes.to_vec()); + } + None => { + let prefix = path.clone(); + map.retain(|p, _| !p.starts_with(&prefix)); + } + } + } +} + +impl Default for CountingFs { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl FileSystemExt for CountingFs { + fn usage(&self) -> FsUsage { + self.inner.usage() + } + fn limits(&self) -> FsLimits { + self.inner.limits() + } + fn backend_kind(&self) -> &'static str { + "counting" + } +} + +#[async_trait] +impl FileSystem for CountingFs { + async fn read_file(&self, path: &Path) -> Result> { + self.hit(&self.counters.read_file); + self.inner.read_file(path).await + } + async fn write_file(&self, path: &Path, content: &[u8]) -> Result<()> { + self.hit(&self.counters.write); + self.inner.write_file(path, content).await?; + self.index_put(path, Some(content), false); + Ok(()) + } + async fn append_file(&self, path: &Path, content: &[u8]) -> Result<()> { + self.hit(&self.counters.write); + self.inner.append_file(path, content).await?; + self.index_put(path, Some(content), true); + Ok(()) + } + async fn mkdir(&self, path: &Path, recursive: bool) -> Result<()> { + self.hit(&self.counters.write); + self.inner.mkdir(path, recursive).await + } + async fn remove(&self, path: &Path, recursive: bool) -> Result<()> { + self.hit(&self.counters.write); + self.inner.remove(path, recursive).await?; + self.index_put(path, None, false); + Ok(()) + } + async fn stat(&self, path: &Path) -> Result { + self.hit(&self.counters.stat); + self.inner.stat(path).await + } + async fn lstat(&self, path: &Path) -> Result { + self.hit(&self.counters.lstat); + self.inner.lstat(path).await + } + async fn read_dir(&self, path: &Path) -> Result> { + self.hit(&self.counters.read_dir); + self.inner.read_dir(path).await + } + async fn exists(&self, path: &Path) -> Result { + self.hit(&self.counters.exists); + self.inner.exists(path).await + } + async fn rename(&self, from: &Path, to: &Path) -> Result<()> { + self.hit(&self.counters.write); + self.inner.rename(from, to).await?; + if let Some(index) = &self.index { + let mut map = index.write().unwrap_or_else(|e| e.into_inner()); + let from = crate::fs::normalize_path(from); + let to = crate::fs::normalize_path(to); + let moved: Vec = map + .keys() + .filter(|p| p.starts_with(&from)) + .cloned() + .collect(); + for old in moved { + if let (Some(content), Ok(rest)) = (map.remove(&old), old.strip_prefix(&from)) { + map.insert(crate::fs::vfs_join(&to, rest), content); + } + } + } + Ok(()) + } + async fn copy(&self, from: &Path, to: &Path) -> Result<()> { + self.hit(&self.counters.write); + self.inner.copy(from, to).await?; + if let Some(index) = &self.index { + let mut map = index.write().unwrap_or_else(|e| e.into_inner()); + if let Some(content) = map.get(&crate::fs::normalize_path(from)).cloned() { + map.insert(crate::fs::normalize_path(to), content); + } + } + Ok(()) + } + async fn symlink(&self, target: &Path, link: &Path) -> Result<()> { + self.hit(&self.counters.write); + self.inner.symlink(target, link).await + } + async fn read_link(&self, path: &Path) -> Result { + self.hit(&self.counters.read_link); + self.inner.read_link(path).await + } + async fn chmod(&self, path: &Path, mode: u32) -> Result<()> { + self.hit(&self.counters.write); + self.inner.chmod(path, mode).await + } + async fn set_modified_time(&self, path: &Path, time: SystemTime) -> Result<()> { + self.hit(&self.counters.write); + self.inner.set_modified_time(path, time).await + } + async fn walk(&self, root: &Path, mut opts: WalkOptions) -> Result> { + if let Some(page) = self.walk_page { + opts.max_entries = opts.max_entries.min(page); + } + let listings = if self.native_walk { + self.hit(&self.counters.walk); + walk_with_read_dir(&self.inner, root, opts).await? + } else { + walk_with_read_dir(self, root, opts).await? + }; + let held: usize = listings.iter().map(|l| l.entries.len()).sum(); + self.counters + .walk_peak_entries + .fetch_max(u64::try_from(held).unwrap_or(u64::MAX), Ordering::Relaxed); + Ok(listings) + } + fn as_search_capable(&self) -> Option<&dyn SearchCapable> { + self.index.as_ref().map(|_| self as &dyn SearchCapable) + } +} + +impl SearchCapable for CountingFs { + fn search_provider(&self, _path: &Path) -> Option> { + let index = self.index.clone()?; + Some(Box::new(IndexProvider { + index, + counters: Arc::clone(&self.counters), + })) + } +} + +/// Searches the index map; counts against the owning `CountingFs`. +struct IndexProvider { + index: Index, + counters: Arc, +} + +impl SearchProvider for IndexProvider { + fn search(&self, query: &SearchQuery) -> Result { + self.counters.hit(&self.counters.search); + let pattern = if query.is_regex { + query.pattern.clone() + } else { + regex::escape(&query.pattern) + }; + let re = regex::RegexBuilder::new(&pattern) + .case_insensitive(query.case_insensitive) + .build() + .map_err(|e| std::io::Error::other(e.to_string()))?; + let root = crate::fs::normalize_path(&query.root); + let map = self.index.read().unwrap_or_else(|e| e.into_inner()); + let mut matches = Vec::new(); + for (path, content) in map.range(root.clone()..) { + if !path.starts_with(&root) { + break; + } + let text = String::from_utf8_lossy(content); + for (i, line) in text.lines().enumerate() { + if re.is_match(line) { + matches.push(SearchMatch { + path: path.clone(), + line_number: i + 1, + line_content: line.to_string(), + }); + if query.max_results.is_some_and(|m| matches.len() >= m) { + return Ok(SearchResults { + matches, + truncated: true, + }); + } + } + } + } + Ok(SearchResults { + matches, + truncated: false, + }) + } + + fn capabilities(&self) -> SearchCapabilities { + SearchCapabilities { + regex: true, + glob_filter: false, + content_search: true, + filename_search: false, + } + } +} diff --git a/crates/bashkit/tests/integration/execution_capability_tests.rs b/crates/bashkit/tests/integration/execution_capability_tests.rs index 2835ef72f..daadad7db 100644 --- a/crates/bashkit/tests/integration/execution_capability_tests.rs +++ b/crates/bashkit/tests/integration/execution_capability_tests.rs @@ -289,3 +289,61 @@ async fn command_resolver_builtins_receive_execution_scoped_access() { let error = fs.read_file(Path::new("/request.txt")).await.unwrap_err(); assert_eq!(error.to_string(), "io error: execution capability revoked"); } + +/// Search providers handed out during an execution are leased too: a +/// builtin that keeps one cannot query the index after its command returns. +struct RetainSearch { + retained: Arc>>>, +} + +#[bashkit::async_trait] +impl Builtin for RetainSearch { + async fn execute(&self, ctx: BuiltinContext<'_>) -> bashkit::Result { + let provider = ctx + .fs + .as_search_capable() + .and_then(|sc| sc.search_provider(Path::new("/"))) + .expect("index reachable during execution"); + let hits = provider.search(&search_query())?.matches.len(); + self.retained.lock().unwrap().push(provider); + Ok(ExecResult::ok(format!("{hits}\n"))) + } +} + +fn search_query() -> bashkit::SearchQuery { + bashkit::SearchQuery { + pattern: "secret".to_string(), + is_regex: false, + case_insensitive: false, + root: "/".into(), + glob_filter: None, + max_results: None, + } +} + +#[tokio::test] +async fn retained_search_provider_fails_after_completion() { + let retained = Arc::new(Mutex::new(Vec::new())); + let fs = Arc::new(bashkit::testing::CountingFs::new().with_index()); + fs.write_file(Path::new("/data.txt"), b"secret\n") + .await + .unwrap(); + let mut bash = Bash::builder() + .fs(fs) + .builtin( + "retain-search", + Box::new(RetainSearch { + retained: retained.clone(), + }), + ) + .build(); + + let result = bash.exec("retain-search").await.unwrap(); + assert_eq!(result.stdout, "1\n", "{}", result.stderr); + + let provider = retained.lock().unwrap().pop().unwrap(); + let error = provider.search(&search_query()).unwrap_err(); + assert_eq!(error.to_string(), "io error: execution capability revoked"); + // Capabilities carry no answer either once the lease is gone. + assert!(!provider.capabilities().content_search); +} diff --git a/crates/bashkit/tests/integration/main.rs b/crates/bashkit/tests/integration/main.rs index 84c0e8b4a..75d39e43b 100644 --- a/crates/bashkit/tests/integration/main.rs +++ b/crates/bashkit/tests/integration/main.rs @@ -144,6 +144,7 @@ pub mod python_security_tests; pub mod redirect_error_tests; pub mod regex_limit_tests; pub mod release_profile_tests; +pub mod remote_fs_round_trip_tests; pub mod request_lifecycle_contract_tests; pub mod runtime_env_tests; pub mod script_analysis; diff --git a/crates/bashkit/tests/integration/remote_fs_round_trip_tests.rs b/crates/bashkit/tests/integration/remote_fs_round_trip_tests.rs new file mode 100644 index 000000000..d2768433b --- /dev/null +++ b/crates/bashkit/tests/integration/remote_fs_round_trip_tests.rs @@ -0,0 +1,508 @@ +//! Round-trip budgets for tree-walking builtins over a remote-style backend. +//! +//! `CountingFs` counts every `FileSystem` call. On a backend where each call +//! is a network round trip (~10 ms), a builtin that issues a `stat` per +//! listed entry, probes for ignore files that do not exist, or reads every +//! file when an index could answer, times out on a modest checkout. These +//! tests pin each command's call budget on a 1,000-file tree, and check that +//! the cheaper paths (index, listing metadata, bulk `walk`) print exactly what +//! the plain paths print. +//! +//! Set `BASHKIT_ROUND_TRIPS=1` to print every command's counts. + +use bashkit::testing::{CallCounts, CountingFs}; +use bashkit::{Bash, FileSystem}; +use std::path::Path; +use std::sync::Arc; + +/// 10 x 10 directories, 10 files each: 1,000 files, 111 directories under +/// `/workspace/tree` (counting `tree` itself). +const TOP: usize = 10; +const SUB: usize = 10; +const FILES: usize = 10; +const DIRS: u64 = 1 + TOP as u64 + (TOP * SUB) as u64; +const TOTAL_FILES: u64 = (TOP * SUB * FILES) as u64; +/// Every 50th file holds the needle: 20 matches. +const NEEDLE_EVERY: usize = 50; +const MATCHES: u64 = (TOP * SUB * FILES / NEEDLE_EVERY) as u64; +/// Directories an index answer may list to apply ignore rules: the two +/// ancestors above the root, the root, and at most one per match level. +const INDEX_LISTING_BUDGET: u64 = 2 + 1 + TOP as u64 + MATCHES; + +async fn seed(fs: &dyn FileSystem) { + let mut n = 0; + fs.mkdir(Path::new("/workspace/tree"), true).await.unwrap(); + for a in 0..TOP { + for b in 0..SUB { + let dir = format!("/workspace/tree/d{a}/s{b}"); + fs.mkdir(Path::new(&dir), true).await.unwrap(); + for c in 0..FILES { + let body = if n % NEEDLE_EVERY == 0 { + format!("line one\nhas needle here\nfile {n}\n") + } else { + format!("line one\nnothing\nfile {n}\n") + }; + fs.write_file(Path::new(&format!("{dir}/f{c}.txt")), body.as_bytes()) + .await + .unwrap(); + n += 1; + } + } + } +} + +async fn setup(fs: CountingFs) -> (Arc, Bash) { + let fs = Arc::new(fs); + seed(fs.as_ref()).await; + let bash = Bash::builder().fs(fs.clone()).cwd("/workspace").build(); + (fs, bash) +} + +async fn run(fs: &CountingFs, bash: &mut Bash, script: &str) -> (CallCounts, String) { + fs.reset(); + let r = bash.exec(script).await.unwrap(); + let counts = fs.counts(); + if std::env::var("BASHKIT_ROUND_TRIPS").is_ok() { + println!("{script:48} {counts}\n stderr: {}", r.stderr); + } + assert_eq!(r.exit_code, 0, "{script}: {}", r.stderr); + (counts, r.stdout.to_string()) +} + +fn line_count(out: &str) -> u64 { + out.lines().count() as u64 +} + +// ---- rg ---------------------------------------------------------------- + +/// Acceptance: `rg PATTERN dir` with ignore rules on makes one `search` +/// call and reads only the matching files. +#[tokio::test] +async fn rg_with_index_makes_one_search_and_reads_only_matches() { + let (fs, mut bash) = setup(CountingFs::new().with_index()).await; + for script in ["rg -l needle tree", "rg needle tree", "rg -c needle tree"] { + let (c, out) = run(&fs, &mut bash, script).await; + assert_eq!(line_count(&out), MATCHES, "{script}"); + assert_eq!(c.search, 1, "{script}: {c}"); + assert_eq!(c.read_file, MATCHES, "{script}: {c}"); + assert!(c.read_dir <= INDEX_LISTING_BUDGET, "{script}: {c}"); + assert!(c.metadata() <= 2, "{script}: {c}"); + } +} + +#[tokio::test] +async fn rg_with_index_and_no_ignore_lists_nothing() { + let (fs, mut bash) = setup(CountingFs::new().with_index()).await; + let (c, out) = run(&fs, &mut bash, "rg -l --no-ignore needle tree").await; + assert_eq!(line_count(&out), MATCHES); + assert_eq!(c.search, 1, "{c}"); + assert_eq!(c.read_file, MATCHES, "{c}"); + assert_eq!(c.read_dir, 0, "{c}"); +} + +/// Without an index, the walk lists each directory once and learns from the +/// listing which ignore files exist: no `stat` probes per directory. +#[tokio::test] +async fn rg_walk_lists_each_directory_once_without_probing_ignore_files() { + let (fs, mut bash) = setup(CountingFs::new()).await; + let (c, out) = run(&fs, &mut bash, "rg -l needle tree").await; + assert_eq!(line_count(&out), MATCHES); + assert_eq!(c.read_file, TOTAL_FILES, "{c}"); + // Every directory under the root, plus the two ancestors above it. + assert_eq!(c.read_dir, DIRS + 2, "{c}"); + assert!(c.metadata() <= 2, "{c}"); +} + +/// Ignore rules, hidden directories and depth limits applied to index +/// matches must give exactly the walk's answer. +#[tokio::test] +async fn rg_index_results_match_walk_with_ignore_files() { + async fn layout(fs: &dyn FileSystem) { + seed(fs).await; + fs.mkdir(Path::new("/workspace/tree/.git"), true) + .await + .unwrap(); + // Root .gitignore: whole subtree, a file glob, and a re-include. + fs.write_file( + Path::new("/workspace/tree/.gitignore"), + b"d1/\nf0.txt\n!d3/s0/f0.txt\n", + ) + .await + .unwrap(); + // Nested .ignore and .rgignore. + fs.write_file(Path::new("/workspace/tree/d2/.ignore"), b"s5/\n") + .await + .unwrap(); + fs.write_file(Path::new("/workspace/tree/d4/s0/.rgignore"), b"*.txt\n") + .await + .unwrap(); + // .gitignore above the search root. + fs.write_file(Path::new("/workspace/.gitignore"), b"d6/s0/\n") + .await + .unwrap(); + // Hidden directory with matches. + fs.mkdir(Path::new("/workspace/tree/.hidden"), true) + .await + .unwrap(); + fs.write_file(Path::new("/workspace/tree/.hidden/h.txt"), b"needle\n") + .await + .unwrap(); + // Extra matches at shallow depth for --max-depth. + fs.write_file(Path::new("/workspace/tree/top.txt"), b"needle\n") + .await + .unwrap(); + fs.write_file(Path::new("/workspace/tree/d7/mid.txt"), b"needle\n") + .await + .unwrap(); + } + + let walk = Arc::new(CountingFs::new()); + layout(walk.as_ref()).await; + let indexed = Arc::new(CountingFs::new().with_index()); + layout(indexed.as_ref()).await; + let mut walk_bash = Bash::builder().fs(walk.clone()).cwd("/workspace").build(); + let mut index_bash = Bash::builder() + .fs(indexed.clone()) + .cwd("/workspace") + .build(); + + for script in [ + "rg -l needle tree | sort", + "rg -n needle tree | sort", + "rg -l --hidden needle tree | sort", + "rg -l --no-ignore-vcs needle tree | sort", + "rg -l --no-ignore-dot needle tree | sort", + "rg -l --no-ignore-parent needle tree | sort", + "rg -l --max-depth 2 needle tree | sort", + "rg -l -g 'f0.txt' needle tree | sort", + "(cd tree && rg -l needle | sort)", + "rg -l needle tree/d3 | sort", + "rg -l needle tree/d2 tree/d7 | sort", + ] { + let (_, expected) = run(&walk, &mut walk_bash, script).await; + let (c, got) = run(&indexed, &mut index_bash, script).await; + assert_eq!(got, expected, "{script}"); + assert!(!expected.is_empty(), "{script}: vacuous"); + assert!(c.search >= 1, "{script} should use the index: {c}"); + } + + // A missing operand falls back to the walk, which reports it. + let script = "rg -l needle tree/missing tree/d3 2>&1 | sort"; + let (_, expected) = run(&walk, &mut walk_bash, script).await; + let (_, got) = run(&indexed, &mut index_bash, script).await; + assert_eq!(got, expected); + assert!( + expected.contains("tree/missing: No such file"), + "{expected}" + ); +} + +// ---- find -------------------------------------------------------------- + +/// Acceptance: `find dir -type f` makes one `read_dir` per directory and no +/// per-file `lstat`. +#[tokio::test] +async fn find_lists_each_directory_once_and_never_stats_listed_entries() { + let (fs, mut bash) = setup(CountingFs::new()).await; + for (script, lines) in [ + ("find tree -type f", TOTAL_FILES), + ("find tree -path '*s3*'", TOP as u64 * (FILES as u64 + 1)), + ("find tree -name f1.txt", (TOP * SUB) as u64), + ("find tree -ls", DIRS + TOTAL_FILES), + ("find tree -printf '%n %p\\n'", DIRS + TOTAL_FILES), + ("find tree -depth -type d", DIRS), + ] { + let (c, out) = run(&fs, &mut bash, script).await; + assert_eq!(line_count(&out), lines, "{script}"); + assert_eq!(c.read_dir, DIRS, "{script}: {c}"); + // Only the root is looked up. + assert!(c.metadata() <= 1, "{script}: {c}"); + assert_eq!(c.read_file, 0, "{script}: {c}"); + } +} + +#[tokio::test] +async fn find_uses_one_bulk_walk_when_the_backend_has_one() { + let (fs, mut bash) = setup(CountingFs::new().with_native_walk()).await; + let (c, out) = run(&fs, &mut bash, "find tree -type f").await; + assert_eq!(line_count(&out), TOTAL_FILES); + assert_eq!(c.walk, 1, "{c}"); + assert_eq!(c.read_dir, 0, "{c}"); + assert!(c.total() <= 2, "{c}"); + + // -prune cuts the traversal, so find lists lazily instead. + let (c, out) = run(&fs, &mut bash, "find tree -name 'd*' -prune").await; + assert_eq!(line_count(&out), TOP as u64); + assert_eq!(c.walk, 0, "{c}"); + assert_eq!(c.read_dir, 1, "{c}"); +} + +/// The bulk walk and listing metadata must not change find's answers. +#[tokio::test] +async fn find_output_is_the_same_with_and_without_bulk_walk() { + async fn layout(fs: &dyn FileSystem) { + seed(fs).await; + fs.symlink(Path::new("d1"), Path::new("/workspace/tree/link-d1")) + .await + .unwrap(); + fs.symlink(Path::new("missing"), Path::new("/workspace/tree/dangling")) + .await + .unwrap(); + fs.mkdir(Path::new("/workspace/tree/empty"), false) + .await + .unwrap(); + } + let plain = Arc::new(CountingFs::new()); + layout(plain.as_ref()).await; + let bulk = Arc::new(CountingFs::new().with_native_walk()); + layout(bulk.as_ref()).await; + let mut plain_bash = Bash::builder().fs(plain.clone()).cwd("/workspace").build(); + let mut bulk_bash = Bash::builder().fs(bulk.clone()).cwd("/workspace").build(); + for script in [ + "find tree | sort", + "find tree -type l | sort", + "find tree -xtype d | sort", + "find -L tree -type f | wc -l", + "find tree -maxdepth 2 | sort", + "find tree -mindepth 2 -maxdepth 2 -type d | sort", + "find tree -empty", + "find tree -printf '%y %n %s %p\\n' | sort", + "find tree -newer tree/d0/s0/f0.txt | wc -l", + "find tree tree/d2 -name f9.txt | sort", + ] { + let (_, expected) = run(&plain, &mut plain_bash, script).await; + let (_, got) = run(&bulk, &mut bulk_bash, script).await; + assert_eq!(got, expected, "{script}"); + assert!(!expected.is_empty(), "{script}: vacuous"); + } +} + +// ---- memory bounds ------------------------------------------------------- + +/// Entries in the seeded tree: every file plus every directory below the root. +const ENTRIES: u64 = TOTAL_FILES + DIRS - 1; +/// Backend page size for the bounded-walk tests: ~1/11 of the tree. +const PAGE: usize = 100; + +/// A walk answer is a bounded batch: find and du hold at most one page +/// (plus one listing) and refill as they descend, with the same output. +#[tokio::test] +async fn walk_batches_are_bounded_for_find_and_du() { + let (plain, mut plain_bash) = setup(CountingFs::new()).await; + let (fs, mut bash) = setup(CountingFs::new().with_native_walk().with_walk_page(PAGE)).await; + for script in [ + "find tree | sort", + "find tree -type f -name 'f1*' | sort", + "find tree -maxdepth 2 | sort", + "find tree -printf '%n %p\\n' | sort", + "du -a tree | sort", + ] { + let (_, expected) = run(&plain, &mut plain_bash, script).await; + let (c, got) = run(&fs, &mut bash, script).await; + assert_eq!(got, expected, "{script}"); + assert!(!expected.is_empty(), "{script}: vacuous"); + assert!( + c.walk_peak_entries <= (PAGE + SUB.max(FILES)) as u64, + "{script}: {c} peak {}", + c.walk_peak_entries + ); + assert_eq!(c.read_dir, 0, "{script}: {c}"); + assert!(c.walk <= DIRS, "{script}: {c}"); + } + // Whole-tree walks need several pages, not one answer. + let (c, _) = run(&fs, &mut bash, "find tree -type f").await; + assert!(c.walk >= ENTRIES / (PAGE as u64 + 10), "{c}"); +} + +/// The default walk (no bulk backend) still lists each directory once when +/// answers are cut at the bound. +#[tokio::test] +async fn walk_batches_are_bounded_with_the_default_walk() { + let (fs, mut bash) = setup(CountingFs::new().with_walk_page(PAGE)).await; + for script in ["find tree -type f", "du -s tree"] { + let (c, _) = run(&fs, &mut bash, script).await; + assert_eq!(c.read_dir, DIRS, "{script}: {c}"); + assert!( + c.walk_peak_entries <= (PAGE + SUB.max(FILES)) as u64, + "{script}: {c}" + ); + } +} + +/// rg charges each file to the aggregate-input budget as it reads, so a tree +/// bigger than the budget stops early instead of loading every file first. +#[tokio::test] +async fn rg_stops_reading_at_input_budget() { + use bashkit::{Error, ExecutionLimits, LimitExceeded}; + for backend in [CountingFs::new(), CountingFs::new().with_index()] { + let fs = Arc::new(backend); + seed(fs.as_ref()).await; + // Files are ~30 bytes: the budget covers about 100 of 1,000. + let limits = ExecutionLimits::new().max_aggregate_input_bytes(3_000); + let mut bash = Bash::builder() + .fs(fs.clone()) + .cwd("/workspace") + .limits(limits) + .build(); + fs.reset(); + let result = bash.exec("rg -c line tree").await; + assert!( + matches!( + result, + Err(Error::ResourceLimit(LimitExceeded::ExecutionBudget(_))) + ), + "expected input budget exhaustion, got {result:?}" + ); + let c = fs.counts(); + assert!(c.read_file < 200, "read past the budget: {c}"); + } +} + +// ---- other tree walkers ------------------------------------------------- + +#[tokio::test] +async fn grep_recursive_reads_files_without_a_stat_each() { + let (fs, mut bash) = setup(CountingFs::new()).await; + let (c, out) = run(&fs, &mut bash, "grep -rl needle tree").await; + assert_eq!(line_count(&out), MATCHES); + assert_eq!(c.read_dir, DIRS, "{c}"); + assert_eq!(c.read_file, TOTAL_FILES, "{c}"); + assert!(c.metadata() <= 3, "{c}"); +} + +#[tokio::test] +async fn grep_recursive_uses_the_index() { + let (fs, mut bash) = setup(CountingFs::new().with_index()).await; + let (c, out) = run(&fs, &mut bash, "grep -rn needle tree").await; + assert_eq!(line_count(&out), MATCHES); + assert_eq!(c.search, 1, "{c}"); + assert_eq!(c.read_file, MATCHES, "{c}"); + assert_eq!(c.read_dir, 0, "{c}"); +} + +#[tokio::test] +async fn du_takes_sizes_from_listings() { + let (fs, mut bash) = setup(CountingFs::new()).await; + for (script, lines) in [("du -s tree", 1), ("du -a tree", DIRS + TOTAL_FILES)] { + let (c, out) = run(&fs, &mut bash, script).await; + assert_eq!(line_count(&out), lines, "{script}"); + assert_eq!(c.read_dir, DIRS, "{script}: {c}"); + assert!(c.metadata() <= 1, "{script}: {c}"); + } + + let (fs, mut bash) = setup(CountingFs::new().with_native_walk()).await; + let (c, _) = run(&fs, &mut bash, "du -s tree").await; + assert_eq!(c.walk, 1, "{c}"); + assert_eq!(c.read_dir, 0, "{c}"); +} + +#[tokio::test] +async fn ls_recursive_and_tree_list_each_directory_once() { + let (fs, mut bash) = setup(CountingFs::new()).await; + for script in ["ls -R tree", "ls -lR tree", "tree tree"] { + let (c, _) = run(&fs, &mut bash, script).await; + assert_eq!(c.read_dir, DIRS, "{script}: {c}"); + assert!(c.metadata() <= 2, "{script}: {c}"); + } +} + +#[tokio::test] +async fn tar_reads_member_metadata_from_listings() { + let (fs, mut bash) = setup(CountingFs::new()).await; + let (c, _) = run(&fs, &mut bash, "tar -cf /tmp/t.tar tree").await; + assert_eq!(c.read_dir, DIRS, "{c}"); + assert_eq!(c.read_file, TOTAL_FILES, "{c}"); + assert!(c.metadata() <= 3, "{c}"); +} + +#[tokio::test] +async fn cp_recursive_stays_linear_in_entries() { + let (fs, mut bash) = setup(CountingFs::new()).await; + let (c, _) = run(&fs, &mut bash, "cp -r tree /tmp/copy").await; + assert_eq!(c.read_dir, DIRS, "{c}"); + // One source + one target lookup per file copy; a handful per directory. + assert!(c.metadata() <= 2 * TOTAL_FILES + 8 * DIRS, "{c}"); + let (_, out) = run(&fs, &mut bash, "find /tmp/copy -type f | wc -l").await; + assert_eq!(out.trim(), TOTAL_FILES.to_string()); +} + +#[tokio::test] +async fn globs_list_each_directory_at_most_twice() { + let (fs, mut bash) = setup(CountingFs::new()).await; + let (c, out) = run(&fs, &mut bash, "echo tree/*/*/*.txt | wc -w").await; + assert_eq!(out.trim(), TOTAL_FILES.to_string()); + assert_eq!(c.read_dir, DIRS, "{c}"); + assert_eq!(c.metadata(), 0, "{c}"); + // TODO: `**` lists every directory in the walk and again for the next + // component; a per-expansion listing cache would halve it. + let (c, out) = run( + &fs, + &mut bash, + "shopt -s globstar; echo tree/**/*.txt | wc -w", + ) + .await; + assert_eq!(out.trim(), TOTAL_FILES.to_string()); + assert!(c.read_dir <= 2 * DIRS, "{c}"); + assert_eq!(c.metadata(), 0, "{c}"); +} + +// ---- mounts ------------------------------------------------------------ + +/// A live mount with its own index answers `rg`/`grep` through the mount +/// table, with paths reported under the mount point. +#[tokio::test] +async fn index_of_a_mounted_filesystem_is_used_through_the_mount() { + let data = Arc::new(CountingFs::new().with_index()); + data.mkdir(Path::new("/src"), true).await.unwrap(); + data.write_file(Path::new("/src/a.rs"), b"fn needle() {}\n") + .await + .unwrap(); + data.write_file(Path::new("/src/b.rs"), b"fn other() {}\n") + .await + .unwrap(); + let mut bash = Bash::new(); + bash.mount("/mnt/data", data.clone()).unwrap(); + + let (c, out) = run(&data, &mut bash, "rg -l needle /mnt/data").await; + assert_eq!(out, "/mnt/data/src/a.rs\n"); + assert_eq!(c.search, 1, "{c}"); + let (c, out) = run(&data, &mut bash, "grep -rl needle /mnt/data/src").await; + assert_eq!(out, "/mnt/data/src/a.rs\n"); + assert_eq!(c.search, 1, "{c}"); + + // The mount's root is listed with its own metadata, as `stat` reports. + let mut bash2 = Bash::new(); + bash2.mount("/mnt/data", data.clone()).unwrap(); + let r = bash2 + .exec("mkdir -p /mnt; [ \"$(stat -c %a /mnt/data)\" = \"$(find /mnt -mindepth 1 -maxdepth 1 -printf '%m')\" ] && echo same") + .await + .unwrap(); + assert_eq!(r.stdout.to_string(), "same\n", "{}", r.stderr); +} + +/// A subtree with another mount below it has no single index: fall back to +/// walking so the nested mount's files are still searched. +#[tokio::test] +async fn nested_mount_below_search_root_disables_the_index() { + let outer = Arc::new(CountingFs::new().with_index()); + outer.mkdir(Path::new("/x"), true).await.unwrap(); + outer + .write_file(Path::new("/x/outer.txt"), b"needle\n") + .await + .unwrap(); + let inner = Arc::new(CountingFs::new()); + inner + .write_file(Path::new("/inner.txt"), b"needle\n") + .await + .unwrap(); + let mut bash = Bash::new(); + bash.mount("/mnt/outer", outer.clone()).unwrap(); + bash.mount("/mnt/outer/x/nested", inner).unwrap(); + + let (c, out) = run(&outer, &mut bash, "rg -l needle /mnt/outer | sort").await; + assert_eq!( + out, + "/mnt/outer/x/nested/inner.txt\n/mnt/outer/x/outer.txt\n" + ); + assert_eq!(c.search, 0, "{c}"); +} diff --git a/docs/filesystem.md b/docs/filesystem.md index d70ec263c..a25ab8c37 100644 --- a/docs/filesystem.md +++ b/docs/filesystem.md @@ -87,6 +87,30 @@ and path normalization. It does not mutate the filesystem or certify failure atomicity, symlink behavior, quota accounting, or error normalization; custom adapter tests remain responsible for those invariants. +### Remote backends + +When every `FileSystem` call is a network round trip, the call count decides +how fast `find`, `rg`, `grep -r` and `du` are. Bashkit's walkers list each +directory once and trust the metadata in `read_dir` entries, so a backend +should return the same metadata there that `lstat` returns. Two optional +methods cut the count further: + +- `as_search_capable()`: a content index (`SearchCapable`). `rg` and `grep -r` + then make one `search` call and read only the matching files, with + `.gitignore`/`.ignore` rules still applied. An answer is taken as + complete for the queried root, and an empty answer means no match: a + provider whose index may be stale or partial must return `None` from + `search_provider` or an error from `search`, and the command walks the + tree instead. +- `walk(root, WalkOptions)`: the directory listings below `root` in one call + (for example one path-prefix query), up to `max_entries` entries per answer + (10,000 by default) so huge trees are fetched in bounded batches. The + default issues one `read_dir` per directory. `find` and `du` use it and ask + for the next batch as they descend. + +`cargo bench --bench remote_fs -- --counts` prints the calls each command makes +over a copy of the bashkit repository. + ## Built-in implementations | Implementation | Purpose | diff --git a/knowledge/foundations/vfs.md b/knowledge/foundations/vfs.md index d6fc7cb54..0361eb2e7 100644 --- a/knowledge/foundations/vfs.md +++ b/knowledge/foundations/vfs.md @@ -400,6 +400,67 @@ their `stat` is an `lstat`, and they refuse to read or write through a link. containment errors stay as they are) or when `stat` reports a link. Writes cost one extra `lstat` on the final component. +### Remote Backends: Round Trips + +Decision (2026-11-02): a custom `FileSystem` may be remote (a database, an +object store); each call is then a network round trip (~10 ms in Everruns). +Builtins are judged by call count, not CPU. Rules, pinned by +`tests/integration/remote_fs_round_trip_tests.rs` (`bashkit::testing::CountingFs`) +and measured by `benches/remote_fs.rs` over the bashkit checkout: + +- **Listing metadata is authoritative.** `read_dir` entries carry `lstat` + metadata; walkers (`find`, `du`, `grep -r`, `tar`, `cp -r`, `rg`) never + `stat` an entry they just listed. Only roots, symlinks (`read_link` + target + `stat`) and explicit operands get a lookup. `MountableFs` lists a mount point + with the mounted root's `stat`, so the rule also holds at mounts. +- **`FileSystem::walk(root, WalkOptions { max_depth, max_entries })`** returns + the listings of `root` and the directories below it in one call, as a + bounded batch: it stops once the answer holds `max_entries` entries + (default 10k; at most one listing over, never a split listing). The + default is one `read_dir` per directory, depth first, reading a directory + only when popped (`walk_with_read_dir`); bulk backends override it with a + prefix query and a row limit. Wrappers forward it when they map the subtree one-to-one + (`FollowFs` rebases a linked root, `MountableFs` when no mount lies below the + root, `RootFs` outside system trees, `ReadOnlyFs`, per-execution wrappers); + otherwise they fall back to the default. Callers use it only when they would + list every directory anyway: `find` without `-exec*`/`-delete`/`-prune`/ + `-quit`, and `du`, through `builtins/walk_cache.rs`: a directory missing + from the cache triggers another bounded `walk` rooted there, listings are + dropped as they are consumed, and an answer without the requested + directory switches the run to plain `read_dir`. Memory stays at about one + batch for trees of millions of files (TM-DOS-140). Not `rg` (ignore rules prune), `ls -R` (skips dot + directories), `tree` (entry budget stops early), where an eager default + walk would read more than today. +- **`SearchCapable` reaches builtins.** `MountableFs` forwards the provider of + the backend that serves the whole requested subtree (none if another mount + sits below it) and maps result paths back under the mount point. A mount + or unmount after the provider was handed out makes its `search` fail, so + callers walk instead of trusting an index that no longer covers the + subtree. Index answers must be complete for the queried root (an empty + answer is "no match"); a backend with a possibly stale index returns no + provider. + `ExecutionFileSystem` hands out a lease-checked provider (TM-ISO-027): each + `search` fails, and `capabilities` reports none, once the execution's + capability is revoked. Before this both + layers returned `None`, so indexed search never ran in a default session. +- **`rg` keeps the index with ignore rules on.** Index matches are filtered + through the walk's rules, loaded only along the directories leading to a + match (one `read_dir` each, ignored or hidden directories are not entered). + The walk itself reads the listing first and opens only the ignore files the + listing shows, and tracks "inside a git repo" down the walk instead of a + `stat` per ancestor. `--max-depth` is applied to index matches. `-L`, + `--type`, `--max-filesize`, `--sort` other than path still walk. +- `rg` keeps every searched file's text until output is built; it charges + each file to the aggregate-input budget as it reads (default 100 MB) and + stops at the first refusal, so a huge tree fails at the budget instead of + after loading every file (TM-DOS-140). Streaming search is a follow-up. +- An index that answers with no match is an answer: `grep -r` no longer + re-walks the tree on zero matches. +- `FollowFs` `copy`/`chmod`/`touch -d` take a fast path (one inner `stat` per + path) instead of resolving every path component. +- Known remaining cost: `**` globs list each directory twice (walk, then the + next component); `ls -R`/`tree` are one `read_dir` per directory. + ## Host Mount Table (`HostMounts`) `realfs` mounts are recorded as a `HostMounts` table on the `Bash` instance: diff --git a/knowledge/log.md b/knowledge/log.md index 911856e98..9828103d2 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -5,12 +5,13 @@ * **Per-call execution timeout**: `ExecOptions::timeout` caps one `exec_with_options` call at the smaller of it and `ExecutionLimits::timeout`; Python `execute*` methods expose it as `timeout_seconds`. See [Architecture](foundations/architecture.md#per-invocation-exec-state). * **Output file tee**: `Execution::output_file` writes combined output into a VFS file under its own cap and `FsLimits`, and the Deep Agents backend uses it for capture-at-source offload without a subshell. See [Architecture](foundations/architecture.md#byte-native-streams), TM-DOS-058. * **Deep Agents backend refresh**: `BashkitBackend` targets Deep Agents 0.7.32+, accepts the per-command `timeout`, reads binary files as base64, and returns protocol error codes for transfers. See [Python Package](runtimes/python-package.md#deep-agents-backend). +* **Remote VFS round trips**: `rg`/`find` timed out over a remote `FileSystem` (10 ms per call). Root causes: indexed search never reached builtins (`MountableFs` and the per-execution VFS wrapper both hid `SearchCapable`), `rg` bailed out of the index whenever ignore files were on and probed four ignore files with `stat`+`read` per directory, `find`/`du`/`grep -r`/`tar` re-`stat`ed every listed entry, `grep -r` canonicalized every directory component by component, and `FollowFs::copy` resolved each path component. Added `FileSystem::walk` (bulk listing, `read_dir` default), a lease-checked search provider, `CountingFs` test/bench backend, round-trip budget tests and the `remote_fs` bench. `walk` answers are bounded batches (`WalkOptions::max_entries`, 10k) refilled on demand by `find`/`du`, and `rg` charges file text to the input budget as it reads, so trees with millions of files never sit in memory at once (TM-DOS-140). See [Virtual Filesystem](foundations/vfs.md#remote-backends-round-trips). ## 2026-11-01 * **CI nightly pin**: Every nightly job (fuzz compile check, scheduled fuzz, Miri, ASan build-std) is pinned to `nightly-2026-10-30`. Nightly 2026-10-31 (rustc f9ceb4a4d) rejects the `bashkit` lib with E0391, an async-drop query cycle on `parser::ast::CommandList`; reproduced on unmodified `main`, and the 2026-10-30 nightly builds it. Unpin when a later nightly passes `cargo +nightly check -p bashkit --lib`. -* **markkit and fetchkit builtins**: new off-by-default features `markkit` and `fetchkit` register both CLIs with their exact interface by calling the upstream `run(args, host)` libraries. All file access goes to the VFS and all requests through the sandbox `HttpClient`, hop by hop. Host-only flags (`--ocr`, `--allow-env-proxy`, `--render-rakers`, `--bot-auth-*`, `--allow-remote-conversion`) and `mcp` are refused. See [Markkit and Fetchkit Builtins](runtimes/markkit-fetchkit-builtins.md), TM-NET-030, TM-DOS-139, L-DOC-001. +* **markkit and fetchkit builtins**: new off-by-default features `markkit` and `fetchkit` register both CLIs with their exact interface by calling the upstream `run(args, host)` libraries. All file access goes to the VFS and all requests through the sandbox `HttpClient`, hop by hop. Host-only flags (`--ocr`, `--allow-env-proxy`, `--render-rakers`, `--bot-auth-*`, `--allow-remote-conversion`) and `mcp` are refused. See [Markkit and Fetchkit Builtins](runtimes/markkit-fetchkit-builtins.md), TM-NET-030, TM-DOS-140, L-DOC-001. ## 2026-10-30 diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 30d75929e..6604e60b6 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -313,6 +313,7 @@ on default 2 MiB test threads (TM-DOS-044). | TM-DOS-107 | Static-analysis command validation amplification | A large comment before thousands of commands makes a whole-script ordered-subsequence scan per command | Build one source-character position index, then validate each analyzed command-name character with a logarithmic position lookup instead of rescanning source | **MITIGATED** | | TM-DOS-138 | Join matching-row output growth | Equal-key row groups multiply generated output and synchronous work before dispatcher capture | Parsed rows, fields, keys, decoded input and generated output own shared-budget capacity before growth. VFS reads charge aggregate input and admit metadata size; output fields append directly without temporary row storage, and the byte-stream copy is admitted. Parsing, grouping (including suppressed matches), comparisons and emissions charge work and poll cancellation/deadlines. Errors abort the request. Ordinary two-row fixtures cover memory/work rejection, cancellation/deadlines, session recovery, and complete pipeline/redirect output | **MITIGATED** | | TM-DOS-139 | Embedded CLI limit escalation | `markkit extract x --max-input-size 99999999999 --timeout 1e9` buys a larger CPU/memory budget in a shared process | Host clamps every markkit limit to its default after flags (`CliHost::limits`); fetchkit bodies capped by `HttpLimits`; OCR (host binaries) refused | **MITIGATED** | +| TM-DOS-140 | Tree-walk memory on huge trees | A tenant mounts or creates a tree with millions of files and runs `find`, `du` or `rg` over it; a bulk prefetch of every listing, or rg collecting every file's text before charging the input budget, holds the whole tree in memory | `FileSystem::walk` answers are bounded batches (`WalkOptions::max_entries`, default 10k entries; a listing is never split); `find`/`du` refill through `builtins/walk_cache.rs` one batch at a time, consume listings as they descend, and fall back to `read_dir` if an answer does not hold the requested directory; rg charges each file's text to the aggregate-input budget as it is read and stops at the first refusal. Regressions: `walk_cache::tests::batches_stay_bounded_and_cover_every_directory`, `remote_fs_round_trip_tests::walk_batches_are_bounded_*`, `rg_stops_reading_at_input_budget` | **MITIGATED** | **TM-DOS-051** is historical. The custom parser was deleted with the `yaml` helper; the replacement `yq` parser/evaluator boundary is tracked by TM-DOS-101.