From bfea3c7c739232432d9ab6c4d3ea53f65a85de6b Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Tue, 3 Nov 2026 05:25:29 +0000 Subject: [PATCH 1/3] test(fs): large-tree stress tests; stream rg's walk, cap ls -R at the stdout limit --- crates/bashkit/docs/threat-model.md | 2 +- crates/bashkit/src/builtins/ls/list.rs | 22 + crates/bashkit/src/builtins/rg/mod.rs | 365 ++++++++++----- .../integration/remote_fs_round_trip_tests.rs | 20 + .../bashkit/tests/large_tree_stress_tests.rs | 436 ++++++++++++++++++ justfile | 4 + knowledge/foundations/vfs.md | 7 +- knowledge/log.md | 1 + knowledge/operations/testing.md | 11 + knowledge/security/threat-model.md | 2 +- 10 files changed, 764 insertions(+), 106 deletions(-) create mode 100644 crates/bashkit/tests/large_tree_stress_tests.rs diff --git a/crates/bashkit/docs/threat-model.md b/crates/bashkit/docs/threat-model.md index b1aa5598b..a435da16a 100644 --- a/crates/bashkit/docs/threat-model.md +++ b/crates/bashkit/docs/threat-model.md @@ -34,7 +34,7 @@ through configurable limits. | Width-derived allocation (TM-DOS-103) | `printf x \| od -w1152921504606846976` | Reject widths above 65,536 bytes; stack-pad one numeric field; budget output capacity before fallible growth | MITIGATED | | Pagination amplification (TM-DOS-137) | Long `pr -h` headers repeated on many short pages | Check the 16 MiB cap before every append; lease input, rows and output capacity before growth; borrow cells/page slices and retain date timestamps; charge input/work and poll cancellation/deadlines, including skipped pages | MITIGATED | | Embedded CLI limit escalation (TM-DOS-139) | `markkit extract x --max-input-size 99999999999` | `markkit` limits can only be lowered below their defaults; `fetchkit` bodies capped by HTTP limits; `--ocr` refused | MITIGATED | -| Tree-walk memory (TM-DOS-140) | `find`, `du` or `rg` over a tree with millions of files | `FileSystem::walk` answers are bounded batches (10k entries) refilled as `find`/`du` descend; `rg` charges each file to the aggregate-input budget as it reads and stops at the first refusal | MITIGATED | +| Tree-walk memory (TM-DOS-140) | `find`, `du` or `rg` over a tree with millions of files | `FileSystem::walk` answers are bounded batches (10k entries) refilled as `find`/`du` descend; `rg` charges each file to the aggregate-input budget as it reads, stops at the first refusal and streams its walk; `ls -R` stops at the stdout limit | MITIGATED | | Variable explosion (TM-DOS-003) | `x=$(cat /dev/urandom)` | /dev/urandom returns bounded 8KB | MITIGATED | | Array growth (TM-DOS-004) | `arr+=(element)` in loop | Command limit | MITIGATED | diff --git a/crates/bashkit/src/builtins/ls/list.rs b/crates/bashkit/src/builtins/ls/list.rs index 99c1a67e7..89d7f60f6 100644 --- a/crates/bashkit/src/builtins/ls/list.rs +++ b/crates/bashkit/src/builtins/ls/list.rs @@ -61,6 +61,10 @@ pub(super) struct LsOptions { pub(super) columns: bool, pub(super) directory: bool, pub(super) quoting: QuotingStyle, + /// Stop generating once output passes this (`max_stdout_bytes` + 1): + /// the rest would be cut anyway, so `ls -R` over millions of files holds + /// at most this much (TM-DOS-140). + pub(super) output_limit: usize, } /// The ls builtin - list directory contents. @@ -175,6 +179,11 @@ impl Builtin for Ls { columns: matches.get_flag("C"), directory: matches.get_flag("directory"), quoting, + output_limit: ctx + .execution_extension::() + .and_then(|limits| limits.try_with(|limits| limits.max_stdout_bytes).ok()) + .unwrap_or_else(|| crate::limits::ExecutionLimits::default().max_stdout_bytes) + .saturating_add(1), }; // PATHS holds OsString values; convert to owned strings for the @@ -297,6 +306,13 @@ impl Builtin for Ls { } } + if output.len() > opts.output_limit { + let mut end = opts.output_limit; + while !output.is_char_boundary(end) { + end -= 1; + } + output.truncate(end); + } Ok(ExecResult::ok(output)) } } @@ -312,6 +328,9 @@ async fn list_directory( show_header: bool, add_newline: bool, ) -> std::result::Result<(), String> { + if output.len() > opts.output_limit { + return Ok(()); + } if add_newline { output.push('\n'); } @@ -409,6 +428,9 @@ async fn list_directory( // Recursive listing if opts.recursive { for (subpath, display) in subdirs { + if output.len() > opts.output_limit { + break; + } // Box the future to avoid infinite recursion type size Box::pin(list_directory( ctx, listings, &subpath, &display, output, opts, true, true, diff --git a/crates/bashkit/src/builtins/rg/mod.rs b/crates/bashkit/src/builtins/rg/mod.rs index 5d7dabcd4..35c2aa81a 100644 --- a/crates/bashkit/src/builtins/rg/mod.rs +++ b/crates/bashkit/src/builtins/rg/mod.rs @@ -3035,8 +3035,7 @@ async fn collect_rg_inputs( actual: ctx.cwd.to_path_buf(), display_hint: RgDisplayHint::None, }; - let files = collect_rg_files_recursive(&*ctx.fs, &[root], opts, ctx.cwd).await; - return Ok(read_rg_files(&*ctx.fs, files, ctx.cwd, opts, budget).await); + return Ok(stream_rg_root(&*ctx.fs, root, opts, ctx.cwd, budget).await); } if let Some(inputs) = try_indexed_search(&*ctx.fs, opts, ctx.cwd, budget).await { @@ -3088,10 +3087,27 @@ async fn collect_rg_inputs( } } } + let path_order = opts.sort == RgSort::Path && !opts.sort_reverse; + if path_order + && roots.len() == 1 + && let Some(root) = roots.pop() + { + // Explicit files come first, then the one root's walk in path order: + // read them as found instead of holding every candidate. + let read = read_rg_files(&*ctx.fs, candidates, ctx.cwd, opts, budget).await; + collected.had_errors |= read.had_errors; + collected.stderr.push_str(&read.stderr); + collected.inputs.extend(read.inputs); + let read = stream_rg_root(&*ctx.fs, root, opts, ctx.cwd, budget).await; + collected.had_errors |= read.had_errors; + collected.stderr.push_str(&read.stderr); + collected.inputs.extend(read.inputs); + return Ok(collected); + } if !roots.is_empty() { candidates.extend(collect_rg_files_recursive(&*ctx.fs, &roots, opts, ctx.cwd).await); } - if opts.sort != RgSort::Path || opts.sort_reverse { + if !path_order { sort_rg_candidates(&mut candidates, opts); } let read = read_rg_files(&*ctx.fs, candidates, ctx.cwd, opts, budget).await; @@ -3760,30 +3776,74 @@ async fn collect_rg_files_recursive( opts: &RgOptions, cwd: &Path, ) -> Vec { - let mut result = Vec::new(); - let mut stack = Vec::new(); - for root in roots { - let mut rules = Vec::new(); - append_global_ignore_rules_for_root(fs, &root.actual, opts, &mut rules).await; - rules.extend(opts.explicit_ignore_rules.clone()); - let _ = load_parent_ignore_rules(fs, &root.actual, opts, &mut rules).await; - stack.push(RgWalkItem { - logical: root.logical.clone(), - actual: root.actual.clone(), - actual_root: root.actual.clone(), - depth: 0, - rules: Arc::new(RgIgnoreRuleSet::root(rules)), - ancestors: vec![root.actual.clone()], - display_hint: root.display_hint, - in_git: false, - }); + let mut sink = RgSink::Collect(Vec::new()); + walk_rg_files(fs, roots, opts, cwd, &mut sink).await; + let mut result = match sink { + RgSink::Collect(files) => files, + RgSink::Read { .. } => Vec::new(), + }; + sort_rg_candidates(&mut result, opts); + result +} + +/// Where the walk hands each file it accepts. +enum RgSink<'a> { + /// Keep candidates for the caller to sort or list. + Collect(Vec), + /// Read each file as it is found. THREAT[TM-DOS-140]: on a huge tree the + /// candidate list alone (two paths and metadata per file) costs hundreds + /// of bytes per file before any budget check; reading as found holds only + /// what the budget admits and stops the walk once it is spent. + Read { + cwd: &'a Path, + budget: &'a mut RgReadBudget, + out: RgCollectedInputs, + }, +} + +impl RgSink<'_> { + async fn push( + &mut self, + fs: &dyn crate::fs::FileSystem, + opts: &RgOptions, + file: RgFileCandidate, + ) { + match self { + Self::Collect(files) => files.push(file), + Self::Read { cwd, budget, out } => { + read_rg_candidate(fs, file, cwd, opts, budget, out).await; + } + } + } + + fn stopped(&self) -> bool { + matches!(self, Self::Read { budget, .. } if budget.exhausted()) } +} + +/// One listed directory of the walk, its entries still to visit. +struct RgWalkFrame { + item: RgWalkItem, + /// The directory's effective rules (its own ignore files applied). + rules: Arc, + in_git: bool, + /// Sorted by name, descending: `pop` yields them in ascending order. + entries: Vec, +} - while let Some(item) = stack.pop() { - // List first: the listing says which ignore files exist, so missing - // ones cost no lookup (one `read_dir` per directory on remote VFSes). - let listing = fs.read_dir(&item.actual).await; - if let Ok(entries) = listing { +/// List `item` and push its frame, or hand it to the sink if it is a file. +async fn enter_rg_walk_item( + fs: &dyn crate::fs::FileSystem, + item: RgWalkItem, + opts: &RgOptions, + cwd: &Path, + stack: &mut Vec, + sink: &mut RgSink<'_>, +) { + // List first: the listing says which ignore files exist, so missing + // ones cost no lookup (one `read_dir` per directory on remote VFSes). + match fs.read_dir(&item.actual).await { + Ok(mut entries) => { let in_git = item.in_git || listing_has_git_dir(&entries); let mut local_rules = Vec::new(); let _ = load_local_ignore_rules( @@ -3802,84 +3862,144 @@ async fn collect_rg_files_recursive( } else { Arc::new(RgIgnoreRuleSet::child(item.rules.clone(), local_rules)) }; - for entry in entries { - let path = vfs_join(&item.logical, &entry.name); - let actual_path = vfs_join(&item.actual, &entry.name); - let hidden_entry = !opts.hidden && is_hidden_name(&entry.name); - let entry_depth = item.depth + 1; - let (entry_actual_path, entry_metadata) = - if entry.metadata.file_type.is_symlink() && opts.follow_symlinks { - let Some((target, target_meta)) = - resolve_rg_symlink_target(fs, &actual_path, &item.actual_root).await - else { - continue; - }; - (target, target_meta) - } else { - (actual_path, entry.metadata) - }; + entries.sort_by(|a, b| b.name.cmp(&a.name)); + stack.push(RgWalkFrame { + item, + rules, + in_git, + entries, + }); + } + Err(_) => { + if let Ok(meta) = fs.stat(&item.actual).await + && meta.file_type.is_file() + && opts.matches_max_filesize(meta.size) + && opts.matches_globs(&item.logical, cwd) + && opts.matches_type_filters(&item.logical) + { + sink.push( + fs, + opts, + RgFileCandidate { + logical: item.logical, + actual: item.actual, + metadata: meta, + display_hint: item.display_hint, + display_override: None, + explicit: false, + }, + ) + .await; + } + } + } +} + +/// Depth-first walk in path order: each directory's entries by name, a +/// subdirectory's files before the next sibling. That is the order +/// `sort_rg_candidates` gives by path, so a single root can stream. +async fn walk_rg_files( + fs: &dyn crate::fs::FileSystem, + roots: &[RgSearchRoot], + opts: &RgOptions, + cwd: &Path, + sink: &mut RgSink<'_>, +) { + let mut stack: Vec = Vec::new(); + for root in roots { + let mut rules = Vec::new(); + append_global_ignore_rules_for_root(fs, &root.actual, opts, &mut rules).await; + rules.extend(opts.explicit_ignore_rules.clone()); + let _ = load_parent_ignore_rules(fs, &root.actual, opts, &mut rules).await; + let item = RgWalkItem { + logical: root.logical.clone(), + actual: root.actual.clone(), + actual_root: root.actual.clone(), + depth: 0, + rules: Arc::new(RgIgnoreRuleSet::root(rules)), + ancestors: vec![root.actual.clone()], + display_hint: root.display_hint, + in_git: false, + }; + enter_rg_walk_item(fs, item, opts, cwd, &mut stack, sink).await; - if entry_metadata.file_type.is_dir() { - if hidden_entry || opts.is_ignored_by_rules(&path, true, &rules) { + while let Some(frame) = stack.last_mut() { + if sink.stopped() { + return; + } + let Some(entry) = frame.entries.pop() else { + stack.pop(); + continue; + }; + let item = &frame.item; + let rules = frame.rules.clone(); + let in_git = frame.in_git; + let path = vfs_join(&item.logical, &entry.name); + let actual_path = vfs_join(&item.actual, &entry.name); + let hidden_entry = !opts.hidden && is_hidden_name(&entry.name); + let entry_depth = item.depth + 1; + let (entry_actual_path, entry_metadata) = + if entry.metadata.file_type.is_symlink() && opts.follow_symlinks { + let Some((target, target_meta)) = + resolve_rg_symlink_target(fs, &actual_path, &item.actual_root).await + else { continue; - } - if opts - .max_depth - .is_none_or(|max_depth| entry_depth < max_depth) - && !item.ancestors.contains(&entry_actual_path) - { - let mut child_ancestors = item.ancestors.clone(); - child_ancestors.push(entry_actual_path.clone()); - stack.push(RgWalkItem { - logical: path, - actual: entry_actual_path, - actual_root: item.actual_root.clone(), - depth: entry_depth, - rules: rules.clone(), - ancestors: child_ancestors, - display_hint: item.display_hint, - in_git, - }); - } - } else if entry_metadata.file_type.is_file() - && opts - .max_depth - .is_none_or(|max_depth| entry_depth <= max_depth) - && opts.matches_max_filesize(entry_metadata.size) - && !opts.is_ignored_by_rules(&path, false, &rules) - && opts.matches_globs(&path, cwd) - && opts.matches_type_filters(&path) - && (!hidden_entry || opts.explicitly_includes_hidden_file(&path, cwd)) + }; + (target, target_meta) + } else { + (actual_path, entry.metadata) + }; + + if entry_metadata.file_type.is_dir() { + if hidden_entry || opts.is_ignored_by_rules(&path, true, &rules) { + continue; + } + if opts + .max_depth + .is_none_or(|max_depth| entry_depth < max_depth) + && !item.ancestors.contains(&entry_actual_path) { - result.push(RgFileCandidate { + let mut child_ancestors = item.ancestors.clone(); + child_ancestors.push(entry_actual_path.clone()); + let child = RgWalkItem { logical: path, actual: entry_actual_path, - metadata: entry_metadata, + actual_root: item.actual_root.clone(), + depth: entry_depth, + rules, + ancestors: child_ancestors, display_hint: item.display_hint, + in_git, + }; + enter_rg_walk_item(fs, child, opts, cwd, &mut stack, sink).await; + } + } else if entry_metadata.file_type.is_file() + && opts + .max_depth + .is_none_or(|max_depth| entry_depth <= max_depth) + && opts.matches_max_filesize(entry_metadata.size) + && !opts.is_ignored_by_rules(&path, false, &rules) + && opts.matches_globs(&path, cwd) + && opts.matches_type_filters(&path) + && (!hidden_entry || opts.explicitly_includes_hidden_file(&path, cwd)) + { + let display_hint = item.display_hint; + sink.push( + fs, + opts, + RgFileCandidate { + logical: path, + actual: entry_actual_path, + metadata: entry_metadata, + display_hint, display_override: None, explicit: false, - }); - } + }, + ) + .await; } - } else if let Ok(meta) = fs.stat(&item.actual).await - && meta.file_type.is_file() - && opts.matches_max_filesize(meta.size) - && opts.matches_globs(&item.logical, cwd) - && opts.matches_type_filters(&item.logical) - { - result.push(RgFileCandidate { - logical: item.logical, - actual: item.actual, - metadata: meta, - display_hint: item.display_hint, - display_override: None, - explicit: false, - }); } } - - sort_rg_candidates(&mut result, opts); - result } fn sort_rg_candidates(files: &mut [RgFileCandidate], opts: &RgOptions) { @@ -3993,20 +4113,59 @@ async fn read_rg_files( if budget.exhausted() { break; } - let display = candidate_display_path(&file, cwd, opts); - match read_rg_text_file(fs, &file.actual, cwd, &display, opts).await { - Ok(text) if !budget.charge(text.len()) => break, - Ok(text) if file.explicit => collected.inputs.push(RgInput::explicit(display, text)), - Ok(text) => collected.inputs.push(RgInput::discovered(display, text)), - Err(e) => { - collected.had_errors = true; - if opts.messages { - collected.stderr.push_str(&e.stderr); - } + read_rg_candidate(fs, file, cwd, opts, budget, &mut collected).await; + } + collected +} + +/// Walk one root in path order, reading each file as it is found. +async fn stream_rg_root( + fs: &dyn crate::fs::FileSystem, + root: RgSearchRoot, + opts: &RgOptions, + cwd: &Path, + budget: &mut RgReadBudget, +) -> RgCollectedInputs { + if opts.sort != RgSort::Path || opts.sort_reverse { + let files = collect_rg_files_recursive(fs, &[root], opts, cwd).await; + return read_rg_files(fs, files, cwd, opts, budget).await; + } + let mut sink = RgSink::Read { + cwd, + budget, + out: RgCollectedInputs::default(), + }; + walk_rg_files(fs, &[root], opts, cwd, &mut sink).await; + match sink { + RgSink::Read { out, .. } => out, + RgSink::Collect(_) => RgCollectedInputs::default(), + } +} + +/// Read one candidate into `out`, charging `budget` (nothing once spent). +async fn read_rg_candidate( + fs: &dyn crate::fs::FileSystem, + file: RgFileCandidate, + cwd: &Path, + opts: &RgOptions, + budget: &mut RgReadBudget, + out: &mut RgCollectedInputs, +) { + if budget.exhausted() { + return; + } + let display = candidate_display_path(&file, cwd, opts); + match read_rg_text_file(fs, &file.actual, cwd, &display, opts).await { + Ok(text) if !budget.charge(text.len()) => {} + Ok(text) if file.explicit => out.inputs.push(RgInput::explicit(display, text)), + Ok(text) => out.inputs.push(RgInput::discovered(display, text)), + Err(e) => { + out.had_errors = true; + if opts.messages { + out.stderr.push_str(&e.stderr); } } } - collected } fn candidate_display_path(file: &RgFileCandidate, cwd: &Path, opts: &RgOptions) -> String { diff --git a/crates/bashkit/tests/integration/remote_fs_round_trip_tests.rs b/crates/bashkit/tests/integration/remote_fs_round_trip_tests.rs index 00fa6cf5c..ca778e67b 100644 --- a/crates/bashkit/tests/integration/remote_fs_round_trip_tests.rs +++ b/crates/bashkit/tests/integration/remote_fs_round_trip_tests.rs @@ -484,6 +484,26 @@ async fn ls_recursive_and_tree_output_is_the_same_with_bulk_walk() { } } +/// `ls -R` stops listing once its output passes the stdout limit, and the +/// visible output is the same prefix (memory: `large_tree_stress_tests`). +#[tokio::test] +async fn ls_recursive_stops_at_the_stdout_limit() { + let (fs, mut bash) = setup(CountingFs::new()).await; + let (_, full) = run(&fs, &mut bash, "ls -R tree").await; + let limit = 2_000; + let mut capped = Bash::builder() + .fs(fs.clone()) + .cwd("/workspace") + .limits(bashkit::ExecutionLimits::new().max_stdout_bytes(limit)) + .build(); + fs.reset(); + let r = capped.exec("ls -R tree").await.unwrap(); + let out = r.stdout.to_string(); + assert!(out.len() <= limit, "{} bytes", out.len()); + assert!(full.starts_with(&out), "capped output is not a prefix"); + assert!(out.len() > limit / 2, "{} bytes", out.len()); +} + /// tree stops at its entry budget; prefetch must not read far past it. #[tokio::test] async fn tree_prefetch_stays_within_entry_budget() { diff --git a/crates/bashkit/tests/large_tree_stress_tests.rs b/crates/bashkit/tests/large_tree_stress_tests.rs new file mode 100644 index 000000000..d56b28fa9 --- /dev/null +++ b/crates/bashkit/tests/large_tree_stress_tests.rs @@ -0,0 +1,436 @@ +//! Large-tree stress tests: tree walkers over hundreds of thousands of files +//! must hold bounded memory, list each directory once, and finish. +//! +//! Decisions: +//! - Own test binary: the counting `#[global_allocator]` is process-wide, so +//! unrelated tests in a shared binary would land in the measurement. +//! - The tree is procedural (`SyntheticFs`): names, sizes and contents are +//! computed from the path, so the tree itself costs no memory and a peak +//! measured during a command is the command's own. A real in-memory tree of +//! this size would dwarf the walkers' allocations. +//! - Default scale (~200k files) keeps the debug-build run in CI to seconds; +//! `BASHKIT_STRESS_FILES=2000000 cargo test --release --test +//! large_tree_stress_tests` (`just stress`) runs the same assertions at millions of files. +//! - Memory bound: loading every listing at once costs ~90 B per entry, so +//! 200k files would take ~18 MB and 2M ~180 MB. Walkers must stay under a +//! fixed `BOUNDED_PEAK` whatever the scale; commands whose output is the +//! tree (`ls -R`) may add their output size and nothing per entry beyond it. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use bashkit::{ + Bash, DirEntry, ExecutionLimits, FileSystem, FileSystemExt, FileType, Metadata, Result, +}; + +/// Tests share the process-wide allocator counters, so they run one at a time. +static SERIAL: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +static LIVE: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); + +/// Tracks live heap bytes and their high-water mark. +struct Counting; + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let ptr = unsafe { System.alloc(layout) }; + if !ptr.is_null() { + let live = LIVE.fetch_add(layout.size(), Ordering::Relaxed) + layout.size(); + PEAK.fetch_max(live, Ordering::Relaxed); + } + ptr + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + LIVE.fetch_sub(layout.size(), Ordering::Relaxed); + unsafe { System.dealloc(ptr, layout) }; + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let new = unsafe { System.realloc(ptr, layout, new_size) }; + if !new.is_null() { + if new_size >= layout.size() { + let grow = new_size - layout.size(); + let live = LIVE.fetch_add(grow, Ordering::Relaxed) + grow; + PEAK.fetch_max(live, Ordering::Relaxed); + } else { + LIVE.fetch_sub(layout.size() - new_size, Ordering::Relaxed); + } + } + new + } +} + +#[global_allocator] +static ALLOCATOR: Counting = Counting; + +/// Peak heap growth any walker may reach, at any scale. +const BOUNDED_PEAK: usize = 8 << 20; + +/// Files per leaf directory, leaf directories per top directory. +const FILES: usize = 100; +const SUB: usize = 20; +/// Subdirectories inside the hidden `.git` at the root (each with `FILES`). +const HIDDEN_DIRS: usize = 200; + +/// Procedural tree: `/dA/sB/fC.txt` plus `/.git/objects/oH/fC`. +struct SyntheticFs { + top: usize, + read_dir: AtomicU64, + read_file: AtomicU64, + metadata: AtomicU64, +} + +#[derive(Clone, Copy)] +enum Node { + Root, + Top(usize), + Leaf(usize, usize), + File(usize, usize, usize), + Git, + Objects, + HiddenLeaf, + HiddenFile, +} + +fn index(name: &str, prefix: char, max: usize) -> Option { + let n: usize = name.strip_prefix(prefix)?.parse().ok()?; + (n < max).then_some(n) +} + +impl SyntheticFs { + fn new(files: usize) -> Self { + Self { + top: files.div_ceil(SUB * FILES).max(1), + read_dir: AtomicU64::new(0), + read_file: AtomicU64::new(0), + metadata: AtomicU64::new(0), + } + } + + fn files(&self) -> usize { + self.top * SUB * FILES + } + + /// Directories under the root (not counting it), outside `.git`. + fn dirs(&self) -> usize { + self.top + self.top * SUB + } + + fn reset(&self) { + for c in [&self.read_dir, &self.read_file, &self.metadata] { + c.store(0, Ordering::Relaxed); + } + } + + fn node(&self, path: &Path) -> Option { + let mut node = Node::Root; + for part in path.components() { + let name = match part { + std::path::Component::RootDir => continue, + std::path::Component::Normal(n) => n.to_str()?, + _ => return None, + }; + node = match node { + Node::Root if name == ".git" => Node::Git, + Node::Root => Node::Top(index(name, 'd', self.top)?), + Node::Top(a) => Node::Leaf(a, index(name, 's', SUB)?), + Node::Leaf(a, b) => { + Node::File(a, b, index(name.strip_suffix(".txt")?, 'f', FILES)?) + } + Node::Git if name == "objects" => Node::Objects, + Node::Objects => { + index(name, 'o', HIDDEN_DIRS)?; + Node::HiddenLeaf + } + Node::HiddenLeaf => { + index(name, 'f', FILES)?; + Node::HiddenFile + } + _ => return None, + }; + } + Some(node) + } + + fn content(node: Node) -> Vec { + match node { + Node::File(a, b, c) => format!("line one\nfile {a} {b} {c}\n").into_bytes(), + _ => b"blob\n".to_vec(), + } + } + + fn meta(node: Node) -> Metadata { + match node { + Node::File(..) | Node::HiddenFile => Metadata { + file_type: FileType::File, + size: Self::content(node).len() as u64, + mode: 0o644, + ..Metadata::default() + }, + _ => Metadata { + file_type: FileType::Directory, + mode: 0o755, + ..Metadata::default() + }, + } + } + + fn children(&self, node: Node) -> Option> { + Some(match node { + Node::Root => std::iter::once((".git".to_string(), Node::Git)) + .chain((0..self.top).map(|a| (format!("d{a}"), Node::Top(a)))) + .collect(), + Node::Top(a) => (0..SUB) + .map(|b| (format!("s{b}"), Node::Leaf(a, b))) + .collect(), + Node::Leaf(a, b) => (0..FILES) + .map(|c| (format!("f{c}.txt"), Node::File(a, b, c))) + .collect(), + Node::Git => vec![("objects".to_string(), Node::Objects)], + Node::Objects => (0..HIDDEN_DIRS) + .map(|h| (format!("o{h}"), Node::HiddenLeaf)) + .collect(), + Node::HiddenLeaf => (0..FILES) + .map(|c| (format!("f{c}"), Node::HiddenFile)) + .collect(), + Node::File(..) | Node::HiddenFile => return None, + }) + } +} + +fn not_found() -> bashkit::Error { + std::io::Error::from(std::io::ErrorKind::NotFound).into() +} + +fn read_only() -> bashkit::Error { + std::io::Error::from(std::io::ErrorKind::PermissionDenied).into() +} + +#[async_trait] +impl FileSystemExt for SyntheticFs {} + +#[async_trait] +impl FileSystem for SyntheticFs { + async fn read_file(&self, path: &Path) -> Result> { + self.read_file.fetch_add(1, Ordering::Relaxed); + match self.node(path) { + Some(n @ (Node::File(..) | Node::HiddenFile)) => Ok(Self::content(n)), + Some(_) => Err(std::io::Error::from(std::io::ErrorKind::IsADirectory).into()), + None => Err(not_found()), + } + } + async fn write_file(&self, _: &Path, _: &[u8]) -> Result<()> { + Err(read_only()) + } + async fn append_file(&self, _: &Path, _: &[u8]) -> Result<()> { + Err(read_only()) + } + async fn mkdir(&self, _: &Path, _: bool) -> Result<()> { + Err(read_only()) + } + async fn remove(&self, _: &Path, _: bool) -> Result<()> { + Err(read_only()) + } + async fn stat(&self, path: &Path) -> Result { + self.metadata.fetch_add(1, Ordering::Relaxed); + self.node(path).map(Self::meta).ok_or_else(not_found) + } + async fn read_dir(&self, path: &Path) -> Result> { + self.read_dir.fetch_add(1, Ordering::Relaxed); + let node = self.node(path).ok_or_else(not_found)?; + let kids = self.children(node).ok_or_else(|| { + bashkit::Error::from(std::io::Error::from(std::io::ErrorKind::NotADirectory)) + })?; + Ok(kids + .into_iter() + .map(|(name, n)| DirEntry { + name, + metadata: Self::meta(n), + }) + .collect()) + } + async fn exists(&self, path: &Path) -> Result { + self.metadata.fetch_add(1, Ordering::Relaxed); + Ok(self.node(path).is_some()) + } + async fn rename(&self, _: &Path, _: &Path) -> Result<()> { + Err(read_only()) + } + async fn copy(&self, _: &Path, _: &Path) -> Result<()> { + Err(read_only()) + } + async fn symlink(&self, _: &Path, _: &Path) -> Result<()> { + Err(read_only()) + } + async fn read_link(&self, _: &Path) -> Result { + Err(std::io::Error::from(std::io::ErrorKind::InvalidInput).into()) + } + async fn chmod(&self, _: &Path, _: u32) -> Result<()> { + Err(read_only()) + } +} + +fn scale() -> usize { + std::env::var("BASHKIT_STRESS_FILES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(200_000) +} + +struct Run { + peak: usize, + elapsed: Duration, + stdout: String, + exit_code: i32, + read_dir: u64, + read_file: u64, + metadata: u64, +} + +async fn setup() -> (Arc, Bash) { + let fs = Arc::new(SyntheticFs::new(scale())); + let limits = ExecutionLimits::new() + .timeout(Duration::from_secs(600)) + .max_commands(usize::MAX / 2) + .max_work_units(u64::MAX / 2) + .max_aggregate_input_bytes(u64::MAX / 2); + let bash = Bash::builder().limits(limits).cwd("/").build(); + bash.mount("/big", fs.clone()).unwrap(); + (fs, bash) +} + +async fn measure(fs: &SyntheticFs, bash: &mut Bash, script: &str) -> Run { + // Warm up lazy statics and caches on a tiny command. + bash.exec("ls /big > /dev/null").await.unwrap(); + fs.reset(); + let before = LIVE.load(Ordering::Relaxed); + PEAK.store(before, Ordering::Relaxed); + let start = Instant::now(); + let r = bash.exec(script).await.unwrap(); + let elapsed = start.elapsed(); + let peak = PEAK.load(Ordering::Relaxed).saturating_sub(before); + let run = Run { + peak, + elapsed, + stdout: r.stdout.to_string(), + exit_code: r.exit_code, + read_dir: fs.read_dir.load(Ordering::Relaxed), + read_file: fs.read_file.load(Ordering::Relaxed), + metadata: fs.metadata.load(Ordering::Relaxed), + }; + eprintln!( + "{script:44} files={} peak={:.1} MiB time={:.2?} read_dir={} read_file={} metadata={} exit={}", + fs.files(), + run.peak as f64 / (1 << 20) as f64, + run.elapsed, + run.read_dir, + run.read_file, + run.metadata, + run.exit_code + ); + run +} + +/// Directory walkers with tiny output: bounded memory, one listing per +/// directory, hidden `.git` skipped where the command skips it. +#[tokio::test(flavor = "current_thread")] +async fn walkers_hold_bounded_memory_over_a_huge_tree() { + let _serial = SERIAL.lock().await; + let (fs, mut bash) = setup().await; + let all_dirs = (fs.dirs() + 2 + HIDDEN_DIRS + 1) as u64; + for (script, out, read_dirs) in [ + ("find /big -name nomatch", "", all_dirs), + ("find /big -type f -name f99.txt | wc -l", "", all_dirs), + ("du -s /big", "", all_dirs), + ("tree -L 2 /big | tail -n 1", "", 1 + fs.top as u64), + ("grep -rl nomatch /big", "", all_dirs), + ] { + let r = measure(&fs, &mut bash, script).await; + assert!(r.exit_code <= 1, "{script}: exit {}", r.exit_code); + assert!(r.stdout.contains(out), "{script}"); + assert!( + r.peak < BOUNDED_PEAK, + "{script}: peak {} B over {} files (bound {BOUNDED_PEAK} B)", + r.peak, + fs.files() + ); + assert!(r.read_dir <= read_dirs, "{script}: {} read_dir", r.read_dir); + assert!(r.metadata <= 4, "{script}: {} metadata lookups", r.metadata); + } +} + +/// `ls -R` prints the tree, so its memory is its output, which stops at the +/// stdout limit (1 MiB by default); beyond that it must not hold the tree, +/// and it never enters `.git`. +#[tokio::test(flavor = "current_thread")] +async fn ls_recursive_memory_tracks_output_only() { + let _serial = SERIAL.lock().await; + let (fs, mut bash) = setup().await; + let r = measure(&fs, &mut bash, "ls -R /big").await; + assert_eq!(r.exit_code, 0); + // Generation stops at the output limit; prefetch reads at most one + // walk batch past it, and never `.git`. + assert!(r.read_dir <= (fs.dirs() + 1) as u64, "entered .git"); + let out = r.stdout.len(); + assert!(out <= 1 << 20, "{out} B of output"); + assert!( + r.peak < BOUNDED_PEAK + 3 * out, + "peak {} B for {out} B of output", + r.peak + ); +} + +/// rg must stop at the aggregate-input budget instead of reading the whole +/// tree into memory first. +#[tokio::test(flavor = "current_thread")] +async fn rg_stops_at_the_input_budget_on_a_huge_tree() { + let _serial = SERIAL.lock().await; + let fs = Arc::new(SyntheticFs::new(scale())); + let budget = 1 << 20; + let limits = ExecutionLimits::new() + .timeout(Duration::from_secs(600)) + .max_aggregate_input_bytes(budget); + let mut bash = Bash::builder().limits(limits).cwd("/").build(); + bash.mount("/big", fs.clone()).unwrap(); + fs.reset(); + let before = LIVE.load(Ordering::Relaxed); + PEAK.store(before, Ordering::Relaxed); + let r = bash.exec("rg -l nomatch /big").await; + let peak = PEAK.load(Ordering::Relaxed).saturating_sub(before); + let reads = fs.read_file.load(Ordering::Relaxed); + eprintln!( + "rg over budget: peak={peak} B reads={reads} result_ok={}", + r.is_ok() + ); + assert!(r.is_err(), "expected the input budget to stop rg"); + // ~25 B per file: the 1 MiB budget covers ~40k files of the tree. + assert!((reads as usize) < fs.files() / 2, "read {reads} files"); + // Held inputs cost ~300 B per file read (text, display path, entry); + // before the walk streamed, the candidate list alone took ~70 MB here. + assert!(peak < 2 * BOUNDED_PEAK, "peak {peak} B"); +} + +/// rg within budget over the whole tree: memory scales with the text it +/// searches (rg holds every searched file until output is built), at +/// ~180 B per file here. TODO(TM-DOS-140): search each file as it is read +/// so non-matching files are dropped; until then the aggregate-input budget +/// is the bound. +#[tokio::test(flavor = "current_thread")] +async fn rg_within_budget_holds_only_file_text() { + let _serial = SERIAL.lock().await; + let (fs, mut bash) = setup().await; + let r = measure(&fs, &mut bash, "rg -l nomatch /big").await; + assert_eq!(r.exit_code, 1); + // Every file, plus any ignore file the listing shows. + assert!(r.read_file as usize >= fs.files()); + // Per searched file: its ~25 B text plus its display path and bookkeeping. + let per_file = r.peak / fs.files(); + eprintln!("rg: {per_file} B per searched file"); + assert!(per_file < 250, "rg holds {per_file} B per file"); +} diff --git a/justfile b/justfile index 5e8ec5e58..7fc27d551 100644 --- a/justfile +++ b/justfile @@ -277,6 +277,10 @@ bench-parallel: ./scripts/bench-parallel.sh pnpm --dir site run data:performance +# Large-tree stress tests at millions of files (release; TM-DOS-140) +stress files="2000000": + BASHKIT_STRESS_FILES={{files}} cargo test --release -p bashkit --test large_tree_stress_tests -- --nocapture + # Run Criterion sqlite builtin benchmark and save results bench-sqlite: ./scripts/bench-sqlite.sh diff --git a/knowledge/foundations/vfs.md b/knowledge/foundations/vfs.md index 200e59182..c8164cce8 100644 --- a/knowledge/foundations/vfs.md +++ b/knowledge/foundations/vfs.md @@ -455,7 +455,12 @@ and measured by `benches/remote_fs.rs` over the bashkit checkout: - `rg` keeps every searched file's text until output is built; it charges each file to the aggregate-input budget as it reads (default 100 MB) and stops at the first refusal, so a huge tree fails at the budget instead of - after loading every file (TM-DOS-140). Streaming search is a follow-up. + after loading every file (TM-DOS-140). With one root in path order, the + walk visits entries in sorted order and reads each file as it finds it, + so no candidate list is held and the walk stops at the budget. Searching + each file as it is read (dropping non-matching text) is a follow-up. +- `ls -R` stops generating once its output passes `max_stdout_bytes` + (the rest would be cut), so its memory is bounded by the stdout limit. - An index that answers with no match is an answer: `grep -r` no longer re-walks the tree on zero matches. - `FollowFs` `copy`/`chmod`/`touch -d` take a fast path (one inner `stat` per diff --git a/knowledge/log.md b/knowledge/log.md index d02ac5a61..8fd61c21e 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-11-03 +* **Large-tree stress tests**: `tests/large_tree_stress_tests.rs` measures heap peaks of tree walkers over a 200k-file procedural tree (2M with `BASHKIT_STRESS_FILES`). It found `rg` collecting every candidate path before reading (~70 MB for 200k files); its walk now streams in path order. See [Testing](operations/testing.md). * **ls -R / tree bulk walk**: `ls -R` and `tree` list through bounded `walk` batches (`WalkCache`) like `find`/`du`. New `WalkOptions::skip_hidden` keeps the default walk out of dot directories they do not print; `tree` caps batches at its remaining entry budget and keeps `read_dir` under `-I`. See [Virtual Filesystem](foundations/vfs.md#remote-backends-round-trips). ## 2026-11-02 diff --git a/knowledge/operations/testing.md b/knowledge/operations/testing.md index 2c94bfc44..657eb5166 100644 --- a/knowledge/operations/testing.md +++ b/knowledge/operations/testing.md @@ -101,6 +101,17 @@ collector peak while asserting bounded rejection and subsequent session use. The consolidated `tar_files_from_budget_tests` covers live/input/work limits, stdin, repeated lists, blank/CRLF/lossy names, and GNU tar differential behavior. +`large_tree_stress_tests.rs` also needs its own binary for its counting +allocator (TM-DOS-140). It mounts a procedural `SyntheticFs` (names, sizes +and contents computed from the path, so the tree costs no memory) with ~200k +files plus a hidden `.git`, and asserts that `find`, `du`, `tree`, `grep -r` +stay under an 8 MiB heap peak and list each directory once, that `ls -R` +adds only its output, and that `rg` stops at the aggregate-input budget +(~14 MB peak at a 1 MiB budget; it held a ~70 MB candidate list before) and +holds under 250 B per searched file within budget. Scale it with +`BASHKIT_STRESS_FILES=2000000 cargo test --release -p bashkit --test +large_tree_stress_tests -- --nocapture` (or `just stress`). + `pr_allocation_tests.rs` also needs its own binary: its counting allocator proves repeated headers stay within three times a 128 KiB live budget, including transient reallocation overlap and interpreter dispatch storage. diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 6604e60b6..521205012 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -313,7 +313,7 @@ on default 2 MiB test threads (TM-DOS-044). | TM-DOS-107 | Static-analysis command validation amplification | A large comment before thousands of commands makes a whole-script ordered-subsequence scan per command | Build one source-character position index, then validate each analyzed command-name character with a logarithmic position lookup instead of rescanning source | **MITIGATED** | | TM-DOS-138 | Join matching-row output growth | Equal-key row groups multiply generated output and synchronous work before dispatcher capture | Parsed rows, fields, keys, decoded input and generated output own shared-budget capacity before growth. VFS reads charge aggregate input and admit metadata size; output fields append directly without temporary row storage, and the byte-stream copy is admitted. Parsing, grouping (including suppressed matches), comparisons and emissions charge work and poll cancellation/deadlines. Errors abort the request. Ordinary two-row fixtures cover memory/work rejection, cancellation/deadlines, session recovery, and complete pipeline/redirect output | **MITIGATED** | | TM-DOS-139 | Embedded CLI limit escalation | `markkit extract x --max-input-size 99999999999 --timeout 1e9` buys a larger CPU/memory budget in a shared process | Host clamps every markkit limit to its default after flags (`CliHost::limits`); fetchkit bodies capped by `HttpLimits`; OCR (host binaries) refused | **MITIGATED** | -| TM-DOS-140 | Tree-walk memory on huge trees | A tenant mounts or creates a tree with millions of files and runs `find`, `du` or `rg` over it; a bulk prefetch of every listing, or rg collecting every file's text before charging the input budget, holds the whole tree in memory | `FileSystem::walk` answers are bounded batches (`WalkOptions::max_entries`, default 10k entries; a listing is never split); `find`/`du` refill through `builtins/walk_cache.rs` one batch at a time, consume listings as they descend, and fall back to `read_dir` if an answer does not hold the requested directory; rg charges each file's text to the aggregate-input budget as it is read and stops at the first refusal. Regressions: `walk_cache::tests::batches_stay_bounded_and_cover_every_directory`, `remote_fs_round_trip_tests::walk_batches_are_bounded_*`, `rg_stops_reading_at_input_budget` | **MITIGATED** | +| TM-DOS-140 | Tree-walk memory on huge trees | A tenant mounts or creates a tree with millions of files and runs `find`, `du` or `rg` over it; a bulk prefetch of every listing, or rg collecting every file's text before charging the input budget, holds the whole tree in memory | `FileSystem::walk` answers are bounded batches (`WalkOptions::max_entries`, default 10k entries; a listing is never split); `find`/`du` refill through `builtins/walk_cache.rs` one batch at a time, consume listings as they descend, and fall back to `read_dir` if an answer does not hold the requested directory; rg charges each file's text to the aggregate-input budget as it is read and stops at the first refusal, and with one root its walk reads files as it finds them (no candidate list); `ls -R` stops generating at `max_stdout_bytes`. Heap peaks over a 200k/2M-file procedural tree are pinned by `large_tree_stress_tests` (walkers under 8 MiB). Residual: within budget rg holds every searched file's text (~160 B per file) until output is built. Regressions: `walk_cache::tests::batches_stay_bounded_and_cover_every_directory`, `remote_fs_round_trip_tests::walk_batches_are_bounded_*`, `rg_stops_reading_at_input_budget` | **MITIGATED** | **TM-DOS-051** is historical. The custom parser was deleted with the `yaml` helper; the replacement `yq` parser/evaluator boundary is tracked by TM-DOS-101. From cf491f9aef49555ab239232b7aae17cfd4260972 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Tue, 3 Nov 2026 06:59:59 +0000 Subject: [PATCH 2/3] test(fs): bound rg over-budget stress peak per file read Coverage (tarpaulin) build peaked at 17.1 MB vs 13.7 MB locally, over the fixed 16 MiB bound. rg holds each read file until the budget trips, so the bound now scales with reads (512 B/read), still far below the pre-streaming ~1.5 KB/read. --- crates/bashkit/tests/large_tree_stress_tests.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/bashkit/tests/large_tree_stress_tests.rs b/crates/bashkit/tests/large_tree_stress_tests.rs index d56b28fa9..55b508660 100644 --- a/crates/bashkit/tests/large_tree_stress_tests.rs +++ b/crates/bashkit/tests/large_tree_stress_tests.rs @@ -411,9 +411,15 @@ async fn rg_stops_at_the_input_budget_on_a_huge_tree() { assert!(r.is_err(), "expected the input budget to stop rg"); // ~25 B per file: the 1 MiB budget covers ~40k files of the tree. assert!((reads as usize) < fs.files() / 2, "read {reads} files"); - // Held inputs cost ~300 B per file read (text, display path, entry); - // before the walk streamed, the candidate list alone took ~70 MB here. - assert!(peak < 2 * BOUNDED_PEAK, "peak {peak} B"); + // rg keeps every file it read until the budget trips, so the bound + // scales with reads: ~290 B per read locally, ~360 B under tarpaulin's + // instrumented build. Before the walk streamed, the whole-tree candidate + // list pushed this past 1.5 KB per read (~70 MB at 200k files). + let per_read = peak.saturating_sub(BOUNDED_PEAK) / (reads as usize).max(1); + assert!( + peak < BOUNDED_PEAK + 512 * reads as usize, + "peak {peak} B for {reads} reads ({per_read} B/read over the fixed bound)" + ); } /// rg within budget over the whole tree: memory scales with the text it From bb10f8ad708af8a0606a7cd1bd67d8f930e6a095 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Tue, 3 Nov 2026 07:03:49 +0000 Subject: [PATCH 3/3] fix(ls): stop appending lines past the stdout limit in one wide directory Review follow-up: the cap was only checked between directories, so a single directory with many entries still built its whole listing before truncation. Also document rg's residual candidate list for several roots, non-path sorts and --files (TM-DOS-140). --- crates/bashkit/src/builtins/ls/list.rs | 14 ++++++++++ .../integration/remote_fs_round_trip_tests.rs | 26 +++++++++++++++++++ knowledge/foundations/vfs.md | 9 ++++--- knowledge/security/threat-model.md | 2 +- 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/crates/bashkit/src/builtins/ls/list.rs b/crates/bashkit/src/builtins/ls/list.rs index 89d7f60f6..8898c4e22 100644 --- a/crates/bashkit/src/builtins/ls/list.rs +++ b/crates/bashkit/src/builtins/ls/list.rs @@ -254,6 +254,9 @@ impl Builtin for Ls { // Output file arguments first (preserving path as given by user) if opts.long { for (path_str, path, metadata) in &file_args { + if output.len() > opts.output_limit { + break; + } let mut entry = format_long_entry("e_name(path_str, opts.quoting), metadata, opts.human); push_link_target(&ctx, path, metadata, &mut entry).await; @@ -281,6 +284,9 @@ impl Builtin for Ls { output.push_str(&format_columns(&names, 80)); } else { for name in &names { + if output.len() > opts.output_limit { + break; + } output.push_str(name); output.push('\n'); } @@ -371,6 +377,11 @@ async fn list_directory( if opts.long { for entry in &filtered { + // Past the stdout limit nothing more is shown; stop before + // formatting (and link lookups) for the rest of a wide dir. + if output.len() > opts.output_limit { + break; + } let mut line = format_long_entry( "e_name(&entry.name, opts.quoting), &entry.metadata, @@ -419,6 +430,9 @@ async fn list_directory( output.push_str(&format_columns(&names, 80)); } else { for name in &names { + if output.len() > opts.output_limit { + break; + } output.push_str(name); output.push('\n'); } diff --git a/crates/bashkit/tests/integration/remote_fs_round_trip_tests.rs b/crates/bashkit/tests/integration/remote_fs_round_trip_tests.rs index ca778e67b..71330e12e 100644 --- a/crates/bashkit/tests/integration/remote_fs_round_trip_tests.rs +++ b/crates/bashkit/tests/integration/remote_fs_round_trip_tests.rs @@ -504,6 +504,32 @@ async fn ls_recursive_stops_at_the_stdout_limit() { assert!(out.len() > limit / 2, "{} bytes", out.len()); } +/// One wide directory: `ls -l` and `ls -1` stop appending lines once the +/// stdout limit is passed, and the capped output is still a prefix. +#[tokio::test] +async fn ls_wide_directory_stops_at_the_stdout_limit() { + let (fs, mut bash) = setup(CountingFs::new()).await; + bash.exec("mkdir wide && for i in $(seq 1 500); do : > wide/f$i; done") + .await + .unwrap(); + for cmd in ["ls -l wide", "ls -1 wide", "ls -l wide/f1 wide/f2 wide/f3"] { + let (_, full) = run(&fs, &mut bash, cmd).await; + let limit = 200; + let mut capped = Bash::builder() + .fs(fs.clone()) + .cwd("/workspace") + .limits(bashkit::ExecutionLimits::new().max_stdout_bytes(limit)) + .build(); + let out = capped.exec(cmd).await.unwrap().stdout.to_string(); + assert!(out.len() <= limit, "{cmd}: {} bytes", out.len()); + assert!( + full.starts_with(&out), + "{cmd}: capped output is not a prefix" + ); + assert!(out.len() > limit / 2, "{cmd}: {} bytes", out.len()); + } +} + /// tree stops at its entry budget; prefetch must not read far past it. #[tokio::test] async fn tree_prefetch_stays_within_entry_budget() { diff --git a/knowledge/foundations/vfs.md b/knowledge/foundations/vfs.md index c8164cce8..c626cff08 100644 --- a/knowledge/foundations/vfs.md +++ b/knowledge/foundations/vfs.md @@ -457,9 +457,12 @@ and measured by `benches/remote_fs.rs` over the bashkit checkout: stops at the first refusal, so a huge tree fails at the budget instead of after loading every file (TM-DOS-140). With one root in path order, the walk visits entries in sorted order and reads each file as it finds it, - so no candidate list is held and the walk stops at the budget. Searching - each file as it is read (dropping non-matching text) is a follow-up. -- `ls -R` stops generating once its output passes `max_stdout_bytes` + so no candidate list is held and the walk stops at the budget. Several + roots, a non-path `--sort` and `--files` still collect every candidate + (paths + metadata) before reading. Follow-ups: stream each root in turn, + and search each file as it is read (dropping non-matching text). +- `ls` (with `-R` or on one wide directory) stops appending lines once its + output passes `max_stdout_bytes` (the rest would be cut), so its memory is bounded by the stdout limit. - An index that answers with no match is an answer: `grep -r` no longer re-walks the tree on zero matches. diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 521205012..fe921478f 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -313,7 +313,7 @@ on default 2 MiB test threads (TM-DOS-044). | TM-DOS-107 | Static-analysis command validation amplification | A large comment before thousands of commands makes a whole-script ordered-subsequence scan per command | Build one source-character position index, then validate each analyzed command-name character with a logarithmic position lookup instead of rescanning source | **MITIGATED** | | TM-DOS-138 | Join matching-row output growth | Equal-key row groups multiply generated output and synchronous work before dispatcher capture | Parsed rows, fields, keys, decoded input and generated output own shared-budget capacity before growth. VFS reads charge aggregate input and admit metadata size; output fields append directly without temporary row storage, and the byte-stream copy is admitted. Parsing, grouping (including suppressed matches), comparisons and emissions charge work and poll cancellation/deadlines. Errors abort the request. Ordinary two-row fixtures cover memory/work rejection, cancellation/deadlines, session recovery, and complete pipeline/redirect output | **MITIGATED** | | TM-DOS-139 | Embedded CLI limit escalation | `markkit extract x --max-input-size 99999999999 --timeout 1e9` buys a larger CPU/memory budget in a shared process | Host clamps every markkit limit to its default after flags (`CliHost::limits`); fetchkit bodies capped by `HttpLimits`; OCR (host binaries) refused | **MITIGATED** | -| TM-DOS-140 | Tree-walk memory on huge trees | A tenant mounts or creates a tree with millions of files and runs `find`, `du` or `rg` over it; a bulk prefetch of every listing, or rg collecting every file's text before charging the input budget, holds the whole tree in memory | `FileSystem::walk` answers are bounded batches (`WalkOptions::max_entries`, default 10k entries; a listing is never split); `find`/`du` refill through `builtins/walk_cache.rs` one batch at a time, consume listings as they descend, and fall back to `read_dir` if an answer does not hold the requested directory; rg charges each file's text to the aggregate-input budget as it is read and stops at the first refusal, and with one root its walk reads files as it finds them (no candidate list); `ls -R` stops generating at `max_stdout_bytes`. Heap peaks over a 200k/2M-file procedural tree are pinned by `large_tree_stress_tests` (walkers under 8 MiB). Residual: within budget rg holds every searched file's text (~160 B per file) until output is built. Regressions: `walk_cache::tests::batches_stay_bounded_and_cover_every_directory`, `remote_fs_round_trip_tests::walk_batches_are_bounded_*`, `rg_stops_reading_at_input_budget` | **MITIGATED** | +| TM-DOS-140 | Tree-walk memory on huge trees | A tenant mounts or creates a tree with millions of files and runs `find`, `du` or `rg` over it; a bulk prefetch of every listing, or rg collecting every file's text before charging the input budget, holds the whole tree in memory | `FileSystem::walk` answers are bounded batches (`WalkOptions::max_entries`, default 10k entries; a listing is never split); `find`/`du` refill through `builtins/walk_cache.rs` one batch at a time, consume listings as they descend, and fall back to `read_dir` if an answer does not hold the requested directory; rg charges each file's text to the aggregate-input budget as it is read and stops at the first refusal, and with one root its walk reads files as it finds them (no candidate list); `ls` (recursive or one wide directory) stops appending lines at `max_stdout_bytes`. Heap peaks over a 200k/2M-file procedural tree are pinned by `large_tree_stress_tests` (walkers under 8 MiB). Residuals: within budget rg holds every searched file's text (~160 B per file) until output is built; several roots, a non-path `--sort` or `--files` still build the full candidate list (paths + metadata per file, no budget charge) before reading; one wide directory is held whole by `read_dir`. Regressions: `walk_cache::tests::batches_stay_bounded_and_cover_every_directory`, `remote_fs_round_trip_tests::walk_batches_are_bounded_*`, `rg_stops_reading_at_input_budget` | **MITIGATED** | **TM-DOS-051** is historical. The custom parser was deleted with the `yaml` helper; the replacement `yq` parser/evaluator boundary is tracked by TM-DOS-101.