diff --git a/crates/bashkit/docs/threat-model.md b/crates/bashkit/docs/threat-model.md index b1aa5598b..a435da16a 100644 --- a/crates/bashkit/docs/threat-model.md +++ b/crates/bashkit/docs/threat-model.md @@ -34,7 +34,7 @@ through configurable limits. | Width-derived allocation (TM-DOS-103) | `printf x \| od -w1152921504606846976` | Reject widths above 65,536 bytes; stack-pad one numeric field; budget output capacity before fallible growth | MITIGATED | | Pagination amplification (TM-DOS-137) | Long `pr -h` headers repeated on many short pages | Check the 16 MiB cap before every append; lease input, rows and output capacity before growth; borrow cells/page slices and retain date timestamps; charge input/work and poll cancellation/deadlines, including skipped pages | MITIGATED | | Embedded CLI limit escalation (TM-DOS-139) | `markkit extract x --max-input-size 99999999999` | `markkit` limits can only be lowered below their defaults; `fetchkit` bodies capped by HTTP limits; `--ocr` refused | MITIGATED | -| Tree-walk memory (TM-DOS-140) | `find`, `du` or `rg` over a tree with millions of files | `FileSystem::walk` answers are bounded batches (10k entries) refilled as `find`/`du` descend; `rg` charges each file to the aggregate-input budget as it reads and stops at the first refusal | MITIGATED | +| Tree-walk memory (TM-DOS-140) | `find`, `du` or `rg` over a tree with millions of files | `FileSystem::walk` answers are bounded batches (10k entries) refilled as `find`/`du` descend; `rg` charges each file to the aggregate-input budget as it reads, stops at the first refusal and streams its walk; `ls -R` stops at the stdout limit | MITIGATED | | Variable explosion (TM-DOS-003) | `x=$(cat /dev/urandom)` | /dev/urandom returns bounded 8KB | MITIGATED | | Array growth (TM-DOS-004) | `arr+=(element)` in loop | Command limit | MITIGATED | diff --git a/crates/bashkit/src/builtins/ls/list.rs b/crates/bashkit/src/builtins/ls/list.rs index 99c1a67e7..8898c4e22 100644 --- a/crates/bashkit/src/builtins/ls/list.rs +++ b/crates/bashkit/src/builtins/ls/list.rs @@ -61,6 +61,10 @@ pub(super) struct LsOptions { pub(super) columns: bool, pub(super) directory: bool, pub(super) quoting: QuotingStyle, + /// Stop generating once output passes this (`max_stdout_bytes` + 1): + /// the rest would be cut anyway, so `ls -R` over millions of files holds + /// at most this much (TM-DOS-140). + pub(super) output_limit: usize, } /// The ls builtin - list directory contents. @@ -175,6 +179,11 @@ impl Builtin for Ls { columns: matches.get_flag("C"), directory: matches.get_flag("directory"), quoting, + output_limit: ctx + .execution_extension::() + .and_then(|limits| limits.try_with(|limits| limits.max_stdout_bytes).ok()) + .unwrap_or_else(|| crate::limits::ExecutionLimits::default().max_stdout_bytes) + .saturating_add(1), }; // PATHS holds OsString values; convert to owned strings for the @@ -245,6 +254,9 @@ impl Builtin for Ls { // Output file arguments first (preserving path as given by user) if opts.long { for (path_str, path, metadata) in &file_args { + if output.len() > opts.output_limit { + break; + } let mut entry = format_long_entry("e_name(path_str, opts.quoting), metadata, opts.human); push_link_target(&ctx, path, metadata, &mut entry).await; @@ -272,6 +284,9 @@ impl Builtin for Ls { output.push_str(&format_columns(&names, 80)); } else { for name in &names { + if output.len() > opts.output_limit { + break; + } output.push_str(name); output.push('\n'); } @@ -297,6 +312,13 @@ impl Builtin for Ls { } } + if output.len() > opts.output_limit { + let mut end = opts.output_limit; + while !output.is_char_boundary(end) { + end -= 1; + } + output.truncate(end); + } Ok(ExecResult::ok(output)) } } @@ -312,6 +334,9 @@ async fn list_directory( show_header: bool, add_newline: bool, ) -> std::result::Result<(), String> { + if output.len() > opts.output_limit { + return Ok(()); + } if add_newline { output.push('\n'); } @@ -352,6 +377,11 @@ async fn list_directory( if opts.long { for entry in &filtered { + // Past the stdout limit nothing more is shown; stop before + // formatting (and link lookups) for the rest of a wide dir. + if output.len() > opts.output_limit { + break; + } let mut line = format_long_entry( "e_name(&entry.name, opts.quoting), &entry.metadata, @@ -400,6 +430,9 @@ async fn list_directory( output.push_str(&format_columns(&names, 80)); } else { for name in &names { + if output.len() > opts.output_limit { + break; + } output.push_str(name); output.push('\n'); } @@ -409,6 +442,9 @@ async fn list_directory( // Recursive listing if opts.recursive { for (subpath, display) in subdirs { + if output.len() > opts.output_limit { + break; + } // Box the future to avoid infinite recursion type size Box::pin(list_directory( ctx, listings, &subpath, &display, output, opts, true, true, diff --git a/crates/bashkit/src/builtins/rg/mod.rs b/crates/bashkit/src/builtins/rg/mod.rs index 5d7dabcd4..35c2aa81a 100644 --- a/crates/bashkit/src/builtins/rg/mod.rs +++ b/crates/bashkit/src/builtins/rg/mod.rs @@ -3035,8 +3035,7 @@ async fn collect_rg_inputs( actual: ctx.cwd.to_path_buf(), display_hint: RgDisplayHint::None, }; - let files = collect_rg_files_recursive(&*ctx.fs, &[root], opts, ctx.cwd).await; - return Ok(read_rg_files(&*ctx.fs, files, ctx.cwd, opts, budget).await); + return Ok(stream_rg_root(&*ctx.fs, root, opts, ctx.cwd, budget).await); } if let Some(inputs) = try_indexed_search(&*ctx.fs, opts, ctx.cwd, budget).await { @@ -3088,10 +3087,27 @@ async fn collect_rg_inputs( } } } + let path_order = opts.sort == RgSort::Path && !opts.sort_reverse; + if path_order + && roots.len() == 1 + && let Some(root) = roots.pop() + { + // Explicit files come first, then the one root's walk in path order: + // read them as found instead of holding every candidate. + let read = read_rg_files(&*ctx.fs, candidates, ctx.cwd, opts, budget).await; + collected.had_errors |= read.had_errors; + collected.stderr.push_str(&read.stderr); + collected.inputs.extend(read.inputs); + let read = stream_rg_root(&*ctx.fs, root, opts, ctx.cwd, budget).await; + collected.had_errors |= read.had_errors; + collected.stderr.push_str(&read.stderr); + collected.inputs.extend(read.inputs); + return Ok(collected); + } if !roots.is_empty() { candidates.extend(collect_rg_files_recursive(&*ctx.fs, &roots, opts, ctx.cwd).await); } - if opts.sort != RgSort::Path || opts.sort_reverse { + if !path_order { sort_rg_candidates(&mut candidates, opts); } let read = read_rg_files(&*ctx.fs, candidates, ctx.cwd, opts, budget).await; @@ -3760,30 +3776,74 @@ async fn collect_rg_files_recursive( opts: &RgOptions, cwd: &Path, ) -> Vec { - let mut result = Vec::new(); - let mut stack = Vec::new(); - for root in roots { - let mut rules = Vec::new(); - append_global_ignore_rules_for_root(fs, &root.actual, opts, &mut rules).await; - rules.extend(opts.explicit_ignore_rules.clone()); - let _ = load_parent_ignore_rules(fs, &root.actual, opts, &mut rules).await; - stack.push(RgWalkItem { - logical: root.logical.clone(), - actual: root.actual.clone(), - actual_root: root.actual.clone(), - depth: 0, - rules: Arc::new(RgIgnoreRuleSet::root(rules)), - ancestors: vec![root.actual.clone()], - display_hint: root.display_hint, - in_git: false, - }); + let mut sink = RgSink::Collect(Vec::new()); + walk_rg_files(fs, roots, opts, cwd, &mut sink).await; + let mut result = match sink { + RgSink::Collect(files) => files, + RgSink::Read { .. } => Vec::new(), + }; + sort_rg_candidates(&mut result, opts); + result +} + +/// Where the walk hands each file it accepts. +enum RgSink<'a> { + /// Keep candidates for the caller to sort or list. + Collect(Vec), + /// Read each file as it is found. THREAT[TM-DOS-140]: on a huge tree the + /// candidate list alone (two paths and metadata per file) costs hundreds + /// of bytes per file before any budget check; reading as found holds only + /// what the budget admits and stops the walk once it is spent. + Read { + cwd: &'a Path, + budget: &'a mut RgReadBudget, + out: RgCollectedInputs, + }, +} + +impl RgSink<'_> { + async fn push( + &mut self, + fs: &dyn crate::fs::FileSystem, + opts: &RgOptions, + file: RgFileCandidate, + ) { + match self { + Self::Collect(files) => files.push(file), + Self::Read { cwd, budget, out } => { + read_rg_candidate(fs, file, cwd, opts, budget, out).await; + } + } + } + + fn stopped(&self) -> bool { + matches!(self, Self::Read { budget, .. } if budget.exhausted()) } +} + +/// One listed directory of the walk, its entries still to visit. +struct RgWalkFrame { + item: RgWalkItem, + /// The directory's effective rules (its own ignore files applied). + rules: Arc, + in_git: bool, + /// Sorted by name, descending: `pop` yields them in ascending order. + entries: Vec, +} - while let Some(item) = stack.pop() { - // List first: the listing says which ignore files exist, so missing - // ones cost no lookup (one `read_dir` per directory on remote VFSes). - let listing = fs.read_dir(&item.actual).await; - if let Ok(entries) = listing { +/// List `item` and push its frame, or hand it to the sink if it is a file. +async fn enter_rg_walk_item( + fs: &dyn crate::fs::FileSystem, + item: RgWalkItem, + opts: &RgOptions, + cwd: &Path, + stack: &mut Vec, + sink: &mut RgSink<'_>, +) { + // List first: the listing says which ignore files exist, so missing + // ones cost no lookup (one `read_dir` per directory on remote VFSes). + match fs.read_dir(&item.actual).await { + Ok(mut entries) => { let in_git = item.in_git || listing_has_git_dir(&entries); let mut local_rules = Vec::new(); let _ = load_local_ignore_rules( @@ -3802,84 +3862,144 @@ async fn collect_rg_files_recursive( } else { Arc::new(RgIgnoreRuleSet::child(item.rules.clone(), local_rules)) }; - for entry in entries { - let path = vfs_join(&item.logical, &entry.name); - let actual_path = vfs_join(&item.actual, &entry.name); - let hidden_entry = !opts.hidden && is_hidden_name(&entry.name); - let entry_depth = item.depth + 1; - let (entry_actual_path, entry_metadata) = - if entry.metadata.file_type.is_symlink() && opts.follow_symlinks { - let Some((target, target_meta)) = - resolve_rg_symlink_target(fs, &actual_path, &item.actual_root).await - else { - continue; - }; - (target, target_meta) - } else { - (actual_path, entry.metadata) - }; + entries.sort_by(|a, b| b.name.cmp(&a.name)); + stack.push(RgWalkFrame { + item, + rules, + in_git, + entries, + }); + } + Err(_) => { + if let Ok(meta) = fs.stat(&item.actual).await + && meta.file_type.is_file() + && opts.matches_max_filesize(meta.size) + && opts.matches_globs(&item.logical, cwd) + && opts.matches_type_filters(&item.logical) + { + sink.push( + fs, + opts, + RgFileCandidate { + logical: item.logical, + actual: item.actual, + metadata: meta, + display_hint: item.display_hint, + display_override: None, + explicit: false, + }, + ) + .await; + } + } + } +} + +/// Depth-first walk in path order: each directory's entries by name, a +/// subdirectory's files before the next sibling. That is the order +/// `sort_rg_candidates` gives by path, so a single root can stream. +async fn walk_rg_files( + fs: &dyn crate::fs::FileSystem, + roots: &[RgSearchRoot], + opts: &RgOptions, + cwd: &Path, + sink: &mut RgSink<'_>, +) { + let mut stack: Vec = Vec::new(); + for root in roots { + let mut rules = Vec::new(); + append_global_ignore_rules_for_root(fs, &root.actual, opts, &mut rules).await; + rules.extend(opts.explicit_ignore_rules.clone()); + let _ = load_parent_ignore_rules(fs, &root.actual, opts, &mut rules).await; + let item = RgWalkItem { + logical: root.logical.clone(), + actual: root.actual.clone(), + actual_root: root.actual.clone(), + depth: 0, + rules: Arc::new(RgIgnoreRuleSet::root(rules)), + ancestors: vec![root.actual.clone()], + display_hint: root.display_hint, + in_git: false, + }; + enter_rg_walk_item(fs, item, opts, cwd, &mut stack, sink).await; - if entry_metadata.file_type.is_dir() { - if hidden_entry || opts.is_ignored_by_rules(&path, true, &rules) { + while let Some(frame) = stack.last_mut() { + if sink.stopped() { + return; + } + let Some(entry) = frame.entries.pop() else { + stack.pop(); + continue; + }; + let item = &frame.item; + let rules = frame.rules.clone(); + let in_git = frame.in_git; + let path = vfs_join(&item.logical, &entry.name); + let actual_path = vfs_join(&item.actual, &entry.name); + let hidden_entry = !opts.hidden && is_hidden_name(&entry.name); + let entry_depth = item.depth + 1; + let (entry_actual_path, entry_metadata) = + if entry.metadata.file_type.is_symlink() && opts.follow_symlinks { + let Some((target, target_meta)) = + resolve_rg_symlink_target(fs, &actual_path, &item.actual_root).await + else { continue; - } - if opts - .max_depth - .is_none_or(|max_depth| entry_depth < max_depth) - && !item.ancestors.contains(&entry_actual_path) - { - let mut child_ancestors = item.ancestors.clone(); - child_ancestors.push(entry_actual_path.clone()); - stack.push(RgWalkItem { - logical: path, - actual: entry_actual_path, - actual_root: item.actual_root.clone(), - depth: entry_depth, - rules: rules.clone(), - ancestors: child_ancestors, - display_hint: item.display_hint, - in_git, - }); - } - } else if entry_metadata.file_type.is_file() - && opts - .max_depth - .is_none_or(|max_depth| entry_depth <= max_depth) - && opts.matches_max_filesize(entry_metadata.size) - && !opts.is_ignored_by_rules(&path, false, &rules) - && opts.matches_globs(&path, cwd) - && opts.matches_type_filters(&path) - && (!hidden_entry || opts.explicitly_includes_hidden_file(&path, cwd)) + }; + (target, target_meta) + } else { + (actual_path, entry.metadata) + }; + + if entry_metadata.file_type.is_dir() { + if hidden_entry || opts.is_ignored_by_rules(&path, true, &rules) { + continue; + } + if opts + .max_depth + .is_none_or(|max_depth| entry_depth < max_depth) + && !item.ancestors.contains(&entry_actual_path) { - result.push(RgFileCandidate { + let mut child_ancestors = item.ancestors.clone(); + child_ancestors.push(entry_actual_path.clone()); + let child = RgWalkItem { logical: path, actual: entry_actual_path, - metadata: entry_metadata, + actual_root: item.actual_root.clone(), + depth: entry_depth, + rules, + ancestors: child_ancestors, display_hint: item.display_hint, + in_git, + }; + enter_rg_walk_item(fs, child, opts, cwd, &mut stack, sink).await; + } + } else if entry_metadata.file_type.is_file() + && opts + .max_depth + .is_none_or(|max_depth| entry_depth <= max_depth) + && opts.matches_max_filesize(entry_metadata.size) + && !opts.is_ignored_by_rules(&path, false, &rules) + && opts.matches_globs(&path, cwd) + && opts.matches_type_filters(&path) + && (!hidden_entry || opts.explicitly_includes_hidden_file(&path, cwd)) + { + let display_hint = item.display_hint; + sink.push( + fs, + opts, + RgFileCandidate { + logical: path, + actual: entry_actual_path, + metadata: entry_metadata, + display_hint, display_override: None, explicit: false, - }); - } + }, + ) + .await; } - } else if let Ok(meta) = fs.stat(&item.actual).await - && meta.file_type.is_file() - && opts.matches_max_filesize(meta.size) - && opts.matches_globs(&item.logical, cwd) - && opts.matches_type_filters(&item.logical) - { - result.push(RgFileCandidate { - logical: item.logical, - actual: item.actual, - metadata: meta, - display_hint: item.display_hint, - display_override: None, - explicit: false, - }); } } - - sort_rg_candidates(&mut result, opts); - result } fn sort_rg_candidates(files: &mut [RgFileCandidate], opts: &RgOptions) { @@ -3993,20 +4113,59 @@ async fn read_rg_files( if budget.exhausted() { break; } - let display = candidate_display_path(&file, cwd, opts); - match read_rg_text_file(fs, &file.actual, cwd, &display, opts).await { - Ok(text) if !budget.charge(text.len()) => break, - Ok(text) if file.explicit => collected.inputs.push(RgInput::explicit(display, text)), - Ok(text) => collected.inputs.push(RgInput::discovered(display, text)), - Err(e) => { - collected.had_errors = true; - if opts.messages { - collected.stderr.push_str(&e.stderr); - } + read_rg_candidate(fs, file, cwd, opts, budget, &mut collected).await; + } + collected +} + +/// Walk one root in path order, reading each file as it is found. +async fn stream_rg_root( + fs: &dyn crate::fs::FileSystem, + root: RgSearchRoot, + opts: &RgOptions, + cwd: &Path, + budget: &mut RgReadBudget, +) -> RgCollectedInputs { + if opts.sort != RgSort::Path || opts.sort_reverse { + let files = collect_rg_files_recursive(fs, &[root], opts, cwd).await; + return read_rg_files(fs, files, cwd, opts, budget).await; + } + let mut sink = RgSink::Read { + cwd, + budget, + out: RgCollectedInputs::default(), + }; + walk_rg_files(fs, &[root], opts, cwd, &mut sink).await; + match sink { + RgSink::Read { out, .. } => out, + RgSink::Collect(_) => RgCollectedInputs::default(), + } +} + +/// Read one candidate into `out`, charging `budget` (nothing once spent). +async fn read_rg_candidate( + fs: &dyn crate::fs::FileSystem, + file: RgFileCandidate, + cwd: &Path, + opts: &RgOptions, + budget: &mut RgReadBudget, + out: &mut RgCollectedInputs, +) { + if budget.exhausted() { + return; + } + let display = candidate_display_path(&file, cwd, opts); + match read_rg_text_file(fs, &file.actual, cwd, &display, opts).await { + Ok(text) if !budget.charge(text.len()) => {} + Ok(text) if file.explicit => out.inputs.push(RgInput::explicit(display, text)), + Ok(text) => out.inputs.push(RgInput::discovered(display, text)), + Err(e) => { + out.had_errors = true; + if opts.messages { + out.stderr.push_str(&e.stderr); } } } - collected } fn candidate_display_path(file: &RgFileCandidate, cwd: &Path, opts: &RgOptions) -> String { diff --git a/crates/bashkit/tests/integration/remote_fs_round_trip_tests.rs b/crates/bashkit/tests/integration/remote_fs_round_trip_tests.rs index 00fa6cf5c..71330e12e 100644 --- a/crates/bashkit/tests/integration/remote_fs_round_trip_tests.rs +++ b/crates/bashkit/tests/integration/remote_fs_round_trip_tests.rs @@ -484,6 +484,52 @@ async fn ls_recursive_and_tree_output_is_the_same_with_bulk_walk() { } } +/// `ls -R` stops listing once its output passes the stdout limit, and the +/// visible output is the same prefix (memory: `large_tree_stress_tests`). +#[tokio::test] +async fn ls_recursive_stops_at_the_stdout_limit() { + let (fs, mut bash) = setup(CountingFs::new()).await; + let (_, full) = run(&fs, &mut bash, "ls -R tree").await; + let limit = 2_000; + let mut capped = Bash::builder() + .fs(fs.clone()) + .cwd("/workspace") + .limits(bashkit::ExecutionLimits::new().max_stdout_bytes(limit)) + .build(); + fs.reset(); + let r = capped.exec("ls -R tree").await.unwrap(); + let out = r.stdout.to_string(); + assert!(out.len() <= limit, "{} bytes", out.len()); + assert!(full.starts_with(&out), "capped output is not a prefix"); + assert!(out.len() > limit / 2, "{} bytes", out.len()); +} + +/// One wide directory: `ls -l` and `ls -1` stop appending lines once the +/// stdout limit is passed, and the capped output is still a prefix. +#[tokio::test] +async fn ls_wide_directory_stops_at_the_stdout_limit() { + let (fs, mut bash) = setup(CountingFs::new()).await; + bash.exec("mkdir wide && for i in $(seq 1 500); do : > wide/f$i; done") + .await + .unwrap(); + for cmd in ["ls -l wide", "ls -1 wide", "ls -l wide/f1 wide/f2 wide/f3"] { + let (_, full) = run(&fs, &mut bash, cmd).await; + let limit = 200; + let mut capped = Bash::builder() + .fs(fs.clone()) + .cwd("/workspace") + .limits(bashkit::ExecutionLimits::new().max_stdout_bytes(limit)) + .build(); + let out = capped.exec(cmd).await.unwrap().stdout.to_string(); + assert!(out.len() <= limit, "{cmd}: {} bytes", out.len()); + assert!( + full.starts_with(&out), + "{cmd}: capped output is not a prefix" + ); + assert!(out.len() > limit / 2, "{cmd}: {} bytes", out.len()); + } +} + /// tree stops at its entry budget; prefetch must not read far past it. #[tokio::test] async fn tree_prefetch_stays_within_entry_budget() { diff --git a/crates/bashkit/tests/large_tree_stress_tests.rs b/crates/bashkit/tests/large_tree_stress_tests.rs new file mode 100644 index 000000000..55b508660 --- /dev/null +++ b/crates/bashkit/tests/large_tree_stress_tests.rs @@ -0,0 +1,442 @@ +//! Large-tree stress tests: tree walkers over hundreds of thousands of files +//! must hold bounded memory, list each directory once, and finish. +//! +//! Decisions: +//! - Own test binary: the counting `#[global_allocator]` is process-wide, so +//! unrelated tests in a shared binary would land in the measurement. +//! - The tree is procedural (`SyntheticFs`): names, sizes and contents are +//! computed from the path, so the tree itself costs no memory and a peak +//! measured during a command is the command's own. A real in-memory tree of +//! this size would dwarf the walkers' allocations. +//! - Default scale (~200k files) keeps the debug-build run in CI to seconds; +//! `BASHKIT_STRESS_FILES=2000000 cargo test --release --test +//! large_tree_stress_tests` (`just stress`) runs the same assertions at millions of files. +//! - Memory bound: loading every listing at once costs ~90 B per entry, so +//! 200k files would take ~18 MB and 2M ~180 MB. Walkers must stay under a +//! fixed `BOUNDED_PEAK` whatever the scale; commands whose output is the +//! tree (`ls -R`) may add their output size and nothing per entry beyond it. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use bashkit::{ + Bash, DirEntry, ExecutionLimits, FileSystem, FileSystemExt, FileType, Metadata, Result, +}; + +/// Tests share the process-wide allocator counters, so they run one at a time. +static SERIAL: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +static LIVE: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); + +/// Tracks live heap bytes and their high-water mark. +struct Counting; + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let ptr = unsafe { System.alloc(layout) }; + if !ptr.is_null() { + let live = LIVE.fetch_add(layout.size(), Ordering::Relaxed) + layout.size(); + PEAK.fetch_max(live, Ordering::Relaxed); + } + ptr + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + LIVE.fetch_sub(layout.size(), Ordering::Relaxed); + unsafe { System.dealloc(ptr, layout) }; + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let new = unsafe { System.realloc(ptr, layout, new_size) }; + if !new.is_null() { + if new_size >= layout.size() { + let grow = new_size - layout.size(); + let live = LIVE.fetch_add(grow, Ordering::Relaxed) + grow; + PEAK.fetch_max(live, Ordering::Relaxed); + } else { + LIVE.fetch_sub(layout.size() - new_size, Ordering::Relaxed); + } + } + new + } +} + +#[global_allocator] +static ALLOCATOR: Counting = Counting; + +/// Peak heap growth any walker may reach, at any scale. +const BOUNDED_PEAK: usize = 8 << 20; + +/// Files per leaf directory, leaf directories per top directory. +const FILES: usize = 100; +const SUB: usize = 20; +/// Subdirectories inside the hidden `.git` at the root (each with `FILES`). +const HIDDEN_DIRS: usize = 200; + +/// Procedural tree: `/dA/sB/fC.txt` plus `/.git/objects/oH/fC`. +struct SyntheticFs { + top: usize, + read_dir: AtomicU64, + read_file: AtomicU64, + metadata: AtomicU64, +} + +#[derive(Clone, Copy)] +enum Node { + Root, + Top(usize), + Leaf(usize, usize), + File(usize, usize, usize), + Git, + Objects, + HiddenLeaf, + HiddenFile, +} + +fn index(name: &str, prefix: char, max: usize) -> Option { + let n: usize = name.strip_prefix(prefix)?.parse().ok()?; + (n < max).then_some(n) +} + +impl SyntheticFs { + fn new(files: usize) -> Self { + Self { + top: files.div_ceil(SUB * FILES).max(1), + read_dir: AtomicU64::new(0), + read_file: AtomicU64::new(0), + metadata: AtomicU64::new(0), + } + } + + fn files(&self) -> usize { + self.top * SUB * FILES + } + + /// Directories under the root (not counting it), outside `.git`. + fn dirs(&self) -> usize { + self.top + self.top * SUB + } + + fn reset(&self) { + for c in [&self.read_dir, &self.read_file, &self.metadata] { + c.store(0, Ordering::Relaxed); + } + } + + fn node(&self, path: &Path) -> Option { + let mut node = Node::Root; + for part in path.components() { + let name = match part { + std::path::Component::RootDir => continue, + std::path::Component::Normal(n) => n.to_str()?, + _ => return None, + }; + node = match node { + Node::Root if name == ".git" => Node::Git, + Node::Root => Node::Top(index(name, 'd', self.top)?), + Node::Top(a) => Node::Leaf(a, index(name, 's', SUB)?), + Node::Leaf(a, b) => { + Node::File(a, b, index(name.strip_suffix(".txt")?, 'f', FILES)?) + } + Node::Git if name == "objects" => Node::Objects, + Node::Objects => { + index(name, 'o', HIDDEN_DIRS)?; + Node::HiddenLeaf + } + Node::HiddenLeaf => { + index(name, 'f', FILES)?; + Node::HiddenFile + } + _ => return None, + }; + } + Some(node) + } + + fn content(node: Node) -> Vec { + match node { + Node::File(a, b, c) => format!("line one\nfile {a} {b} {c}\n").into_bytes(), + _ => b"blob\n".to_vec(), + } + } + + fn meta(node: Node) -> Metadata { + match node { + Node::File(..) | Node::HiddenFile => Metadata { + file_type: FileType::File, + size: Self::content(node).len() as u64, + mode: 0o644, + ..Metadata::default() + }, + _ => Metadata { + file_type: FileType::Directory, + mode: 0o755, + ..Metadata::default() + }, + } + } + + fn children(&self, node: Node) -> Option> { + Some(match node { + Node::Root => std::iter::once((".git".to_string(), Node::Git)) + .chain((0..self.top).map(|a| (format!("d{a}"), Node::Top(a)))) + .collect(), + Node::Top(a) => (0..SUB) + .map(|b| (format!("s{b}"), Node::Leaf(a, b))) + .collect(), + Node::Leaf(a, b) => (0..FILES) + .map(|c| (format!("f{c}.txt"), Node::File(a, b, c))) + .collect(), + Node::Git => vec![("objects".to_string(), Node::Objects)], + Node::Objects => (0..HIDDEN_DIRS) + .map(|h| (format!("o{h}"), Node::HiddenLeaf)) + .collect(), + Node::HiddenLeaf => (0..FILES) + .map(|c| (format!("f{c}"), Node::HiddenFile)) + .collect(), + Node::File(..) | Node::HiddenFile => return None, + }) + } +} + +fn not_found() -> bashkit::Error { + std::io::Error::from(std::io::ErrorKind::NotFound).into() +} + +fn read_only() -> bashkit::Error { + std::io::Error::from(std::io::ErrorKind::PermissionDenied).into() +} + +#[async_trait] +impl FileSystemExt for SyntheticFs {} + +#[async_trait] +impl FileSystem for SyntheticFs { + async fn read_file(&self, path: &Path) -> Result> { + self.read_file.fetch_add(1, Ordering::Relaxed); + match self.node(path) { + Some(n @ (Node::File(..) | Node::HiddenFile)) => Ok(Self::content(n)), + Some(_) => Err(std::io::Error::from(std::io::ErrorKind::IsADirectory).into()), + None => Err(not_found()), + } + } + async fn write_file(&self, _: &Path, _: &[u8]) -> Result<()> { + Err(read_only()) + } + async fn append_file(&self, _: &Path, _: &[u8]) -> Result<()> { + Err(read_only()) + } + async fn mkdir(&self, _: &Path, _: bool) -> Result<()> { + Err(read_only()) + } + async fn remove(&self, _: &Path, _: bool) -> Result<()> { + Err(read_only()) + } + async fn stat(&self, path: &Path) -> Result { + self.metadata.fetch_add(1, Ordering::Relaxed); + self.node(path).map(Self::meta).ok_or_else(not_found) + } + async fn read_dir(&self, path: &Path) -> Result> { + self.read_dir.fetch_add(1, Ordering::Relaxed); + let node = self.node(path).ok_or_else(not_found)?; + let kids = self.children(node).ok_or_else(|| { + bashkit::Error::from(std::io::Error::from(std::io::ErrorKind::NotADirectory)) + })?; + Ok(kids + .into_iter() + .map(|(name, n)| DirEntry { + name, + metadata: Self::meta(n), + }) + .collect()) + } + async fn exists(&self, path: &Path) -> Result { + self.metadata.fetch_add(1, Ordering::Relaxed); + Ok(self.node(path).is_some()) + } + async fn rename(&self, _: &Path, _: &Path) -> Result<()> { + Err(read_only()) + } + async fn copy(&self, _: &Path, _: &Path) -> Result<()> { + Err(read_only()) + } + async fn symlink(&self, _: &Path, _: &Path) -> Result<()> { + Err(read_only()) + } + async fn read_link(&self, _: &Path) -> Result { + Err(std::io::Error::from(std::io::ErrorKind::InvalidInput).into()) + } + async fn chmod(&self, _: &Path, _: u32) -> Result<()> { + Err(read_only()) + } +} + +fn scale() -> usize { + std::env::var("BASHKIT_STRESS_FILES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(200_000) +} + +struct Run { + peak: usize, + elapsed: Duration, + stdout: String, + exit_code: i32, + read_dir: u64, + read_file: u64, + metadata: u64, +} + +async fn setup() -> (Arc, Bash) { + let fs = Arc::new(SyntheticFs::new(scale())); + let limits = ExecutionLimits::new() + .timeout(Duration::from_secs(600)) + .max_commands(usize::MAX / 2) + .max_work_units(u64::MAX / 2) + .max_aggregate_input_bytes(u64::MAX / 2); + let bash = Bash::builder().limits(limits).cwd("/").build(); + bash.mount("/big", fs.clone()).unwrap(); + (fs, bash) +} + +async fn measure(fs: &SyntheticFs, bash: &mut Bash, script: &str) -> Run { + // Warm up lazy statics and caches on a tiny command. + bash.exec("ls /big > /dev/null").await.unwrap(); + fs.reset(); + let before = LIVE.load(Ordering::Relaxed); + PEAK.store(before, Ordering::Relaxed); + let start = Instant::now(); + let r = bash.exec(script).await.unwrap(); + let elapsed = start.elapsed(); + let peak = PEAK.load(Ordering::Relaxed).saturating_sub(before); + let run = Run { + peak, + elapsed, + stdout: r.stdout.to_string(), + exit_code: r.exit_code, + read_dir: fs.read_dir.load(Ordering::Relaxed), + read_file: fs.read_file.load(Ordering::Relaxed), + metadata: fs.metadata.load(Ordering::Relaxed), + }; + eprintln!( + "{script:44} files={} peak={:.1} MiB time={:.2?} read_dir={} read_file={} metadata={} exit={}", + fs.files(), + run.peak as f64 / (1 << 20) as f64, + run.elapsed, + run.read_dir, + run.read_file, + run.metadata, + run.exit_code + ); + run +} + +/// Directory walkers with tiny output: bounded memory, one listing per +/// directory, hidden `.git` skipped where the command skips it. +#[tokio::test(flavor = "current_thread")] +async fn walkers_hold_bounded_memory_over_a_huge_tree() { + let _serial = SERIAL.lock().await; + let (fs, mut bash) = setup().await; + let all_dirs = (fs.dirs() + 2 + HIDDEN_DIRS + 1) as u64; + for (script, out, read_dirs) in [ + ("find /big -name nomatch", "", all_dirs), + ("find /big -type f -name f99.txt | wc -l", "", all_dirs), + ("du -s /big", "", all_dirs), + ("tree -L 2 /big | tail -n 1", "", 1 + fs.top as u64), + ("grep -rl nomatch /big", "", all_dirs), + ] { + let r = measure(&fs, &mut bash, script).await; + assert!(r.exit_code <= 1, "{script}: exit {}", r.exit_code); + assert!(r.stdout.contains(out), "{script}"); + assert!( + r.peak < BOUNDED_PEAK, + "{script}: peak {} B over {} files (bound {BOUNDED_PEAK} B)", + r.peak, + fs.files() + ); + assert!(r.read_dir <= read_dirs, "{script}: {} read_dir", r.read_dir); + assert!(r.metadata <= 4, "{script}: {} metadata lookups", r.metadata); + } +} + +/// `ls -R` prints the tree, so its memory is its output, which stops at the +/// stdout limit (1 MiB by default); beyond that it must not hold the tree, +/// and it never enters `.git`. +#[tokio::test(flavor = "current_thread")] +async fn ls_recursive_memory_tracks_output_only() { + let _serial = SERIAL.lock().await; + let (fs, mut bash) = setup().await; + let r = measure(&fs, &mut bash, "ls -R /big").await; + assert_eq!(r.exit_code, 0); + // Generation stops at the output limit; prefetch reads at most one + // walk batch past it, and never `.git`. + assert!(r.read_dir <= (fs.dirs() + 1) as u64, "entered .git"); + let out = r.stdout.len(); + assert!(out <= 1 << 20, "{out} B of output"); + assert!( + r.peak < BOUNDED_PEAK + 3 * out, + "peak {} B for {out} B of output", + r.peak + ); +} + +/// rg must stop at the aggregate-input budget instead of reading the whole +/// tree into memory first. +#[tokio::test(flavor = "current_thread")] +async fn rg_stops_at_the_input_budget_on_a_huge_tree() { + let _serial = SERIAL.lock().await; + let fs = Arc::new(SyntheticFs::new(scale())); + let budget = 1 << 20; + let limits = ExecutionLimits::new() + .timeout(Duration::from_secs(600)) + .max_aggregate_input_bytes(budget); + let mut bash = Bash::builder().limits(limits).cwd("/").build(); + bash.mount("/big", fs.clone()).unwrap(); + fs.reset(); + let before = LIVE.load(Ordering::Relaxed); + PEAK.store(before, Ordering::Relaxed); + let r = bash.exec("rg -l nomatch /big").await; + let peak = PEAK.load(Ordering::Relaxed).saturating_sub(before); + let reads = fs.read_file.load(Ordering::Relaxed); + eprintln!( + "rg over budget: peak={peak} B reads={reads} result_ok={}", + r.is_ok() + ); + assert!(r.is_err(), "expected the input budget to stop rg"); + // ~25 B per file: the 1 MiB budget covers ~40k files of the tree. + assert!((reads as usize) < fs.files() / 2, "read {reads} files"); + // rg keeps every file it read until the budget trips, so the bound + // scales with reads: ~290 B per read locally, ~360 B under tarpaulin's + // instrumented build. Before the walk streamed, the whole-tree candidate + // list pushed this past 1.5 KB per read (~70 MB at 200k files). + let per_read = peak.saturating_sub(BOUNDED_PEAK) / (reads as usize).max(1); + assert!( + peak < BOUNDED_PEAK + 512 * reads as usize, + "peak {peak} B for {reads} reads ({per_read} B/read over the fixed bound)" + ); +} + +/// rg within budget over the whole tree: memory scales with the text it +/// searches (rg holds every searched file until output is built), at +/// ~180 B per file here. TODO(TM-DOS-140): search each file as it is read +/// so non-matching files are dropped; until then the aggregate-input budget +/// is the bound. +#[tokio::test(flavor = "current_thread")] +async fn rg_within_budget_holds_only_file_text() { + let _serial = SERIAL.lock().await; + let (fs, mut bash) = setup().await; + let r = measure(&fs, &mut bash, "rg -l nomatch /big").await; + assert_eq!(r.exit_code, 1); + // Every file, plus any ignore file the listing shows. + assert!(r.read_file as usize >= fs.files()); + // Per searched file: its ~25 B text plus its display path and bookkeeping. + let per_file = r.peak / fs.files(); + eprintln!("rg: {per_file} B per searched file"); + assert!(per_file < 250, "rg holds {per_file} B per file"); +} diff --git a/justfile b/justfile index 5e8ec5e58..7fc27d551 100644 --- a/justfile +++ b/justfile @@ -277,6 +277,10 @@ bench-parallel: ./scripts/bench-parallel.sh pnpm --dir site run data:performance +# Large-tree stress tests at millions of files (release; TM-DOS-140) +stress files="2000000": + BASHKIT_STRESS_FILES={{files}} cargo test --release -p bashkit --test large_tree_stress_tests -- --nocapture + # Run Criterion sqlite builtin benchmark and save results bench-sqlite: ./scripts/bench-sqlite.sh diff --git a/knowledge/foundations/vfs.md b/knowledge/foundations/vfs.md index 200e59182..c626cff08 100644 --- a/knowledge/foundations/vfs.md +++ b/knowledge/foundations/vfs.md @@ -455,7 +455,15 @@ and measured by `benches/remote_fs.rs` over the bashkit checkout: - `rg` keeps every searched file's text until output is built; it charges each file to the aggregate-input budget as it reads (default 100 MB) and stops at the first refusal, so a huge tree fails at the budget instead of - after loading every file (TM-DOS-140). Streaming search is a follow-up. + after loading every file (TM-DOS-140). With one root in path order, the + walk visits entries in sorted order and reads each file as it finds it, + so no candidate list is held and the walk stops at the budget. Several + roots, a non-path `--sort` and `--files` still collect every candidate + (paths + metadata) before reading. Follow-ups: stream each root in turn, + and search each file as it is read (dropping non-matching text). +- `ls` (with `-R` or on one wide directory) stops appending lines once its + output passes `max_stdout_bytes` + (the rest would be cut), so its memory is bounded by the stdout limit. - An index that answers with no match is an answer: `grep -r` no longer re-walks the tree on zero matches. - `FollowFs` `copy`/`chmod`/`touch -d` take a fast path (one inner `stat` per diff --git a/knowledge/log.md b/knowledge/log.md index d02ac5a61..8fd61c21e 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-11-03 +* **Large-tree stress tests**: `tests/large_tree_stress_tests.rs` measures heap peaks of tree walkers over a 200k-file procedural tree (2M with `BASHKIT_STRESS_FILES`). It found `rg` collecting every candidate path before reading (~70 MB for 200k files); its walk now streams in path order. See [Testing](operations/testing.md). * **ls -R / tree bulk walk**: `ls -R` and `tree` list through bounded `walk` batches (`WalkCache`) like `find`/`du`. New `WalkOptions::skip_hidden` keeps the default walk out of dot directories they do not print; `tree` caps batches at its remaining entry budget and keeps `read_dir` under `-I`. See [Virtual Filesystem](foundations/vfs.md#remote-backends-round-trips). ## 2026-11-02 diff --git a/knowledge/operations/testing.md b/knowledge/operations/testing.md index 2c94bfc44..657eb5166 100644 --- a/knowledge/operations/testing.md +++ b/knowledge/operations/testing.md @@ -101,6 +101,17 @@ collector peak while asserting bounded rejection and subsequent session use. The consolidated `tar_files_from_budget_tests` covers live/input/work limits, stdin, repeated lists, blank/CRLF/lossy names, and GNU tar differential behavior. +`large_tree_stress_tests.rs` also needs its own binary for its counting +allocator (TM-DOS-140). It mounts a procedural `SyntheticFs` (names, sizes +and contents computed from the path, so the tree costs no memory) with ~200k +files plus a hidden `.git`, and asserts that `find`, `du`, `tree`, `grep -r` +stay under an 8 MiB heap peak and list each directory once, that `ls -R` +adds only its output, and that `rg` stops at the aggregate-input budget +(~14 MB peak at a 1 MiB budget; it held a ~70 MB candidate list before) and +holds under 250 B per searched file within budget. Scale it with +`BASHKIT_STRESS_FILES=2000000 cargo test --release -p bashkit --test +large_tree_stress_tests -- --nocapture` (or `just stress`). + `pr_allocation_tests.rs` also needs its own binary: its counting allocator proves repeated headers stay within three times a 128 KiB live budget, including transient reallocation overlap and interpreter dispatch storage. diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 6604e60b6..fe921478f 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -313,7 +313,7 @@ on default 2 MiB test threads (TM-DOS-044). | TM-DOS-107 | Static-analysis command validation amplification | A large comment before thousands of commands makes a whole-script ordered-subsequence scan per command | Build one source-character position index, then validate each analyzed command-name character with a logarithmic position lookup instead of rescanning source | **MITIGATED** | | TM-DOS-138 | Join matching-row output growth | Equal-key row groups multiply generated output and synchronous work before dispatcher capture | Parsed rows, fields, keys, decoded input and generated output own shared-budget capacity before growth. VFS reads charge aggregate input and admit metadata size; output fields append directly without temporary row storage, and the byte-stream copy is admitted. Parsing, grouping (including suppressed matches), comparisons and emissions charge work and poll cancellation/deadlines. Errors abort the request. Ordinary two-row fixtures cover memory/work rejection, cancellation/deadlines, session recovery, and complete pipeline/redirect output | **MITIGATED** | | TM-DOS-139 | Embedded CLI limit escalation | `markkit extract x --max-input-size 99999999999 --timeout 1e9` buys a larger CPU/memory budget in a shared process | Host clamps every markkit limit to its default after flags (`CliHost::limits`); fetchkit bodies capped by `HttpLimits`; OCR (host binaries) refused | **MITIGATED** | -| TM-DOS-140 | Tree-walk memory on huge trees | A tenant mounts or creates a tree with millions of files and runs `find`, `du` or `rg` over it; a bulk prefetch of every listing, or rg collecting every file's text before charging the input budget, holds the whole tree in memory | `FileSystem::walk` answers are bounded batches (`WalkOptions::max_entries`, default 10k entries; a listing is never split); `find`/`du` refill through `builtins/walk_cache.rs` one batch at a time, consume listings as they descend, and fall back to `read_dir` if an answer does not hold the requested directory; rg charges each file's text to the aggregate-input budget as it is read and stops at the first refusal. Regressions: `walk_cache::tests::batches_stay_bounded_and_cover_every_directory`, `remote_fs_round_trip_tests::walk_batches_are_bounded_*`, `rg_stops_reading_at_input_budget` | **MITIGATED** | +| TM-DOS-140 | Tree-walk memory on huge trees | A tenant mounts or creates a tree with millions of files and runs `find`, `du` or `rg` over it; a bulk prefetch of every listing, or rg collecting every file's text before charging the input budget, holds the whole tree in memory | `FileSystem::walk` answers are bounded batches (`WalkOptions::max_entries`, default 10k entries; a listing is never split); `find`/`du` refill through `builtins/walk_cache.rs` one batch at a time, consume listings as they descend, and fall back to `read_dir` if an answer does not hold the requested directory; rg charges each file's text to the aggregate-input budget as it is read and stops at the first refusal, and with one root its walk reads files as it finds them (no candidate list); `ls` (recursive or one wide directory) stops appending lines at `max_stdout_bytes`. Heap peaks over a 200k/2M-file procedural tree are pinned by `large_tree_stress_tests` (walkers under 8 MiB). Residuals: within budget rg holds every searched file's text (~160 B per file) until output is built; several roots, a non-path `--sort` or `--files` still build the full candidate list (paths + metadata per file, no budget charge) before reading; one wide directory is held whole by `read_dir`. Regressions: `walk_cache::tests::batches_stay_bounded_and_cover_every_directory`, `remote_fs_round_trip_tests::walk_batches_are_bounded_*`, `rg_stops_reading_at_input_budget` | **MITIGATED** | **TM-DOS-051** is historical. The custom parser was deleted with the `yaml` helper; the replacement `yq` parser/evaluator boundary is tracked by TM-DOS-101.