diff --git a/.alfonso/release-notes/next/inspect-graph-source-limits.md b/.alfonso/release-notes/next/inspect-graph-source-limits.md new file mode 100644 index 000000000..f2d1e9af4 --- /dev/null +++ b/.alfonso/release-notes/next/inspect-graph-source-limits.md @@ -0,0 +1 @@ +- aft_inspect's dead-code and unused-export checks skip generated and minified JavaScript and TypeScript files and read source files only up to 4 MiB, so one huge file can no longer exhaust memory (#1337). diff --git a/benchmarks/aft-search/slice-descriptors/land-inspect-graph-source-limits.json b/benchmarks/aft-search/slice-descriptors/land-inspect-graph-source-limits.json new file mode 100644 index 000000000..0660584a6 --- /dev/null +++ b/benchmarks/aft-search/slice-descriptors/land-inspect-graph-source-limits.json @@ -0,0 +1,8 @@ +{ + "slice_class": "non_ranking", + "targeted_mechanism": "none", + "kind": "harness", + "fixtures": [ + "harness-goldens" + ] +} diff --git a/crates/aft/src/inspect/cache.rs b/crates/aft/src/inspect/cache.rs index 3f99e91f2..4e48d1f1c 100644 --- a/crates/aft/src/inspect/cache.rs +++ b/crates/aft/src/inspect/cache.rs @@ -202,7 +202,10 @@ impl From for InspectCacheError { /// undecodable dead-code rows remain explicit coverage gaps. /// v43: source-size admission excludes oversized complexity/duplicate inputs /// and reports their coverage gaps instead of reusing unbounded scan results. -pub(crate) const TIER2_CONTRIBUTION_CACHE_VERSION: u32 = 43; +/// v44: dead-code and unused-export readers admit at most 4 MiB of source, +/// omit generated/minified JS/TS, and invalidate graph-wide deadness claims +/// when a source file exceeds the limit. +pub(crate) const TIER2_CONTRIBUTION_CACHE_VERSION: u32 = 44; // Reserved metadata category: its integer is an epoch, never a scan timestamp. const CACHE_VERSION_CATEGORY: &str = "__cache_version__"; @@ -3998,7 +4001,7 @@ pub(crate) mod tests { decoded.contribution["exports"][0]["is_type_like"].as_bool(), Some(true) ); - assert_eq!(TIER2_CONTRIBUTION_CACHE_VERSION, 43); + assert_eq!(TIER2_CONTRIBUTION_CACHE_VERSION, 44); } #[test] diff --git a/crates/aft/src/inspect/manager.rs b/crates/aft/src/inspect/manager.rs index c4f1c61aa..39ea71b74 100644 --- a/crates/aft/src/inspect/manager.rs +++ b/crates/aft/src/inspect/manager.rs @@ -6807,7 +6807,15 @@ fn roll_up_unused_exports_contributions( let mut uncertain_count = 0usize; let mut uncertain_items = Vec::new(); let mut excluded = ExcludedTestTally::default(); + let has_source_gaps = parsed.iter().any(|scan| { + scan.skipped_files + .iter() + .any(|skip| skip["reason"] == super::scanners::source_text::GRAPH_TOO_LARGE) + }); for scan in &parsed { + if has_source_gaps || scan.excluded_from_graph { + continue; + } if public_api_files.contains(&scan.file) { continue; } @@ -6940,6 +6948,7 @@ fn roll_up_unused_exports_contributions( aggregate["parse_errors"] = Value::Array(parse_errors); } if !skipped_files.is_empty() { + aggregate["skipped_files_count"] = json!(skipped_files.len()); aggregate["skipped_files"] = Value::Array(skipped_files); } if !package_warnings.is_empty() { @@ -6957,6 +6966,9 @@ fn roll_up_unused_exports_oxc_contributions( let facts = parsed .iter() .filter_map(|scan| { + if scan.excluded_from_graph { + return None; + } let oxc_facts = scan.oxc_facts.as_ref()?; let path = job.project_root.join(&scan.file); Some(FileFacts { @@ -6982,11 +6994,12 @@ fn roll_up_unused_exports_oxc_contributions( .map(|scan| { ( scan.file.clone(), - super::generated::is_generated_file_with_cached_hint( - &job.project_root, - &scan.file, - scan.generated, - ), + !scan.excluded_from_graph + && super::generated::is_generated_file_with_cached_hint( + &job.project_root, + &scan.file, + scan.generated, + ), ) }) .collect::>(); @@ -7014,8 +7027,13 @@ fn roll_up_unused_exports_oxc_contributions( let mut uncertain_count = 0usize; let mut uncertain_items = Vec::new(); let mut excluded = ExcludedTestTally::default(); + let has_source_gaps = parsed.iter().any(|scan| { + scan.skipped_files + .iter() + .any(|skip| skip["reason"] == super::scanners::source_text::GRAPH_TOO_LARGE) + }); for file in &oxc_result.files { - if public_api_files.contains(&file.relative_file) { + if has_source_gaps || public_api_files.contains(&file.relative_file) { continue; } let test_tree = is_test_tree_file(&file.relative_file); @@ -7199,6 +7217,7 @@ fn roll_up_unused_exports_oxc_contributions( aggregate["parse_errors"] = Value::Array(parse_errors); } if !skipped_files.is_empty() { + aggregate["skipped_files_count"] = json!(skipped_files.len()); aggregate["skipped_files"] = Value::Array(skipped_files); } if !package_warnings.is_empty() { @@ -7306,6 +7325,8 @@ struct DeadCodeRefreshContribution { #[derive(Debug, Clone, Deserialize)] struct UnusedExportsContribution { + #[serde(default)] + excluded_from_graph: bool, file: String, #[serde(default)] generated: Option, @@ -9434,7 +9455,7 @@ export function bannerUnused() {} } #[test] - fn unused_exports_oxc_cached_rollup_preserves_generated_split() { + fn unused_exports_oxc_cached_rollup_preserves_generated_exclusions() { let (_dir, root, paths) = generated_unused_exports_fixture(); let job = unused_exports_job(&root, paths.clone()); let entry_points = crate::inspect::entry_points::resolve_entry_points(&root); @@ -9468,8 +9489,248 @@ export function bannerUnused() {} "cached rollup must match fresh scan" ); assert_eq!(rolled_up["count"], 1, "{rolled_up:#}"); - assert_eq!(rolled_up["generated_count"], 2, "{rolled_up:#}"); - assert_eq!(rolled_up["total_count"], 3, "{rolled_up:#}"); + assert_eq!(rolled_up["generated_count"], 0, "{rolled_up:#}"); + assert_eq!(rolled_up["total_count"], 1, "{rolled_up:#}"); + } + + #[test] + fn unused_exports_oversized_js_ts_sources_are_unopened_and_cached_as_graph_wide_gaps() { + use std::fs; + + use super::super::scanners::unused_exports::{ + run_unused_exports_scan, run_unused_exports_scan_with_oxc, + }; + for extension in ["js", "ts"] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + let large = root.join(format!("importer.{extension}")); + let small = root.join(format!("target.{extension}")); + fs::write( + &small, + super::super::scanners::source_text::handwritten_graph_source_for_test( + 1_572_864, + "export function used() {}\n", + ), + ) + .unwrap(); + fs::write(&large, super::super::scanners::source_text::handwritten_graph_source_for_test(5_242_880, &format!("import {{ used }} from './target.{extension}'; used();\nexport function inLarge() {{}}\n"))).unwrap(); + let paths = vec![large.clone(), small]; + let job = unused_exports_job(root, paths.clone()); + let probe = crate::project_content::Probe::new(root); + let mut cache = super::super::oxc_engine::OxcFactsCache::new(); + for oxc in [false, true] { + let success = if oxc { + let result = super::super::oxc_engine::analyze_files_with_cache( + root, + &paths, + AnalyzeOptions::default(), + &mut cache, + ) + .unwrap(); + assert!(!result.facts.iter().any(|facts| facts.path == large)); + assert!(result + .files + .iter() + .flat_map(|file| &file.exports) + .all(|export| export.verdict == LivenessVerdict::Uncertain)); + run_unused_exports_scan_with_oxc(&job, Some(&result)) + } else { + run_unused_exports_scan(&job) + } + .outcome + .unwrap(); + let aggregate = &success.aggregate; + assert_eq!(aggregate["count"], 0, "{aggregate}"); + assert_eq!(aggregate["complete"], false); + assert_eq!(aggregate["skipped_files_count"], 1); + assert_eq!( + aggregate["skipped_files"], + json!([{"file": format!("importer.{extension}"), "reason": "over the 4 MiB size limit"}]) + ); + let cached = + roll_up_unused_exports_contributions(&job, &success.contributions, None); + for key in [ + "count", + "items", + "skipped_files", + "skipped_files_count", + "complete", + "uncertain_count", + ] { + assert_eq!(cached[key], aggregate[key], "{key}, oxc={oxc}"); + } + } + assert_eq!( + super::super::scanners::source_text::source_read_count_for_debug(&large), + 0 + ); + assert!( + crate::file_kind::read_probe::bytes_read(&large) + <= crate::file_kind::HEAD_BYTES * 4 + ); + drop(probe); + fs::write(&large, "export function nowSmall() {}\n").unwrap(); + let result = super::super::oxc_engine::analyze_files_with_cache( + root, + &paths, + AnalyzeOptions::default(), + &mut cache, + ) + .unwrap(); + assert!(result.errors.is_empty()); + let repaired = run_unused_exports_scan_with_oxc(&job, Some(&result)) + .outcome + .unwrap(); + assert_eq!(repaired.aggregate["count"], 2); + assert_eq!(repaired.aggregate["complete"], true); + assert!(repaired.aggregate.get("skipped_files").is_none()); + } + } + + #[test] + fn unused_exports_generated_and_minified_inputs_are_unopened_deliberate_exclusions() { + use super::super::scanners::unused_exports::{ + run_unused_exports_scan, run_unused_exports_scan_with_oxc, + }; + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + let generated = root.join("api.generated.ts"); + let minified = root.join("bundle.min.js"); + let target = root.join("target.ts"); + std::fs::write( + &generated, + "import { used } from './target'; used(); export function generatedUnused() {}\n", + ) + .unwrap(); + std::fs::write(&minified, "export function minifiedUnused() {}\n").unwrap(); + std::fs::write(&target, "export function used() {}\n").unwrap(); + let paths = vec![generated.clone(), minified.clone(), target]; + let job = unused_exports_job(root, paths.clone()); + let probe = crate::project_content::Probe::new(root); + for oxc in [false, true] { + let success = if oxc { + let result = super::super::oxc_engine::analyze_files( + root, + &paths, + AnalyzeOptions::default(), + ) + .unwrap(); + run_unused_exports_scan_with_oxc(&job, Some(&result)) + } else { + run_unused_exports_scan(&job) + } + .outcome + .unwrap(); + assert_eq!(success.aggregate["count"], 1, "{}", success.aggregate); + assert_eq!(success.aggregate["generated_count"], 0); + assert_eq!(success.aggregate["complete"], true); + assert!(success.aggregate.get("skipped_files").is_none()); + let cached = roll_up_unused_exports_contributions(&job, &success.contributions, None); + assert_eq!(cached["count"], 1); + assert_eq!(cached["complete"], true); + } + assert!(!probe.paths().contains(&generated)); + assert!(!probe.paths().contains(&minified)); + } + + /// Capture uncapped scanner findings and importer evidence for a frozen + /// real-repository corpus. Storage and output are supplied by the caller. + #[test] + #[ignore = "real-repository graph admission measurement; requires AFT_GRAPH_MEASURE_SPEC and AFT_GRAPH_MEASURE_OUTPUT"] + fn measure_graph_source_admission_on_real_repositories() { + let specs: Value = serde_json::from_str( + &std::env::var("AFT_GRAPH_MEASURE_SPEC").expect("measurement spec"), + ) + .unwrap(); + let output = + PathBuf::from(std::env::var("AFT_GRAPH_MEASURE_OUTPUT").expect("measurement output")); + let storage = output.with_extension("storage"); + let mut results = Vec::new(); + for spec in specs.as_array().unwrap() { + let name = spec["name"].as_str().unwrap(); + let root = std::fs::canonicalize(spec["root"].as_str().unwrap()).unwrap(); + let prefix = spec["prefix"].as_str().unwrap_or(""); + let files = crate::callgraph::walk_project_files(&root) + .filter(|path| { + path.strip_prefix(&root) + .is_ok_and(|path| path.starts_with(prefix)) + && crate::parser::detect_language(path).is_some_and(is_js_ts_language) + }) + .collect::>(); + assert!(!files.is_empty(), "empty corpus: {name}"); + let excluded = files + .iter() + .filter_map(|path| { + let kind = crate::file_kind::classify(&root, path, None).content; + matches!( + kind, + crate::file_kind::ContentKind::Generated + | crate::file_kind::ContentKind::Minified + ) + .then(|| json!({"file": relative_cache_key(&root, path), "kind": kind.label()})) + }) + .collect::>(); + if name != "aft-packages" { + assert!(!excluded.is_empty(), "no generated/minified inputs: {name}"); + } + let project_key = crate::search_index::artifact_cache_key(&root); + crate::root_cache::configure_artifact_access(&root, &project_key, false); + let (store, _) = CallGraphStore::cold_build_with_lease( + storage.join("callgraph").join(&project_key), + root.clone(), + &files, + ) + .unwrap(); + let snapshot = Arc::new( + crate::callgraph_store::project_dead_code_snapshot(store.sqlite_path()).unwrap(), + ); + let mut job = snapshot_job(&root, &storage.join("inspect").join(&project_key), true); + job.scope_files = files.clone(); + job.callgraph_snapshot = Some(snapshot.clone()); + let entries = super::super::entry_points::resolve_entry_points(&root); + let options = |entry_reachability| AnalyzeOptions { + entry_points: snapshot.entry_points.iter().cloned().collect(), + public_api_files: entries.public_api_files(), + executable_root_exports: entries.executable_root_exports(), + force_reparse_files: Vec::new(), + entry_reachability, + }; + let dead_oxc = + super::super::oxc_engine::analyze_files(&root, &files, options(true)).unwrap(); + let dead = super::super::scanners::dead_code::run_dead_code_scan_with_oxc( + &job, + Some(&dead_oxc), + ) + .outcome + .unwrap() + .aggregate; + job.category = InspectCategory::UnusedExports; + job.key = JobKey::for_project_category(job.category); + let unused_oxc = + super::super::oxc_engine::analyze_files(&root, &files, options(false)).unwrap(); + let unused = super::super::scanners::unused_exports::run_unused_exports_scan_with_oxc( + &job, + Some(&unused_oxc), + ) + .outcome + .unwrap() + .aggregate; + let calls = snapshot.outbound_calls.iter().map(|call| json!({ + "caller_file": relative_cache_key(&root, &call.caller_file), "caller_symbol": call.caller_symbol, + "target": call.target, "line": call.line, + })).collect::>(); + println!( + "GRAPH_MEASURE {name}: files={} excluded={} dead={} unused={}", + files.len(), + excluded.len(), + dead["count"], + unused["count"] + ); + results.push(json!({"name": name, "root": root, "files": files.len(), "excluded": excluded, + "dead_code": dead, "unused_exports": unused, "oxc": unused_oxc, "snapshot_calls": calls})); + } + std::fs::create_dir_all(output.parent().unwrap()).unwrap(); + std::fs::write(output, serde_json::to_vec(&results).unwrap()).unwrap(); } #[test] @@ -9525,10 +9786,27 @@ export function bannerUnused() {} let legacy_cached = roll_up_unused_exports_contributions(&job, &contributions, Some(MAX_DRILL_DOWN_ITEMS)); assert_eq!(legacy_cached, fresh.aggregate); + assert_eq!( + crate::inspect::generated::file_probe_count_for_debug(&root), + 0, + "deliberate exclusions must not be reprobed when a legacy generated hint is absent" + ); + contributions + .iter_mut() + .find(|contribution| contribution.file_path.ends_with("src/hand.ts")) + .unwrap() + .contribution + .as_object_mut() + .unwrap() + .remove("generated"); + crate::inspect::generated::reset_file_probe_count_for_debug(&root); + let legacy_handwritten = + roll_up_unused_exports_contributions(&job, &contributions, Some(MAX_DRILL_DOWN_ITEMS)); + assert_eq!(legacy_handwritten, fresh.aggregate); assert_eq!( crate::inspect::generated::file_probe_count_for_debug(&root), 1, - "a legacy contribution without generated must probe and recover its classification" + "legacy handwritten contributions still recover a missing classification hint" ); } diff --git a/crates/aft/src/inspect/oxc_engine/mod.rs b/crates/aft/src/inspect/oxc_engine/mod.rs index 42e35355b..afbc80298 100644 --- a/crates/aft/src/inspect/oxc_engine/mod.rs +++ b/crates/aft/src/inspect/oxc_engine/mod.rs @@ -16,6 +16,9 @@ use std::fs; use std::path::{Path, PathBuf}; use std::time::SystemTime; +use crate::inspect::scanners::source_text::{ + graph_source_permit, read_graph_source_text, SourceText, GRAPH_TOO_LARGE, +}; use oxc_span::SourceType; use facts::parse_file_facts; @@ -84,14 +87,28 @@ impl OxcFactsCache { path: &Path, force_reparse: bool, stats: &mut OxcFactsCacheStats, - ) -> std::io::Result { + ) -> Result { let source_type = SourceType::from_path(path).unwrap_or_default(); let source_type_key = source_type_cache_key(source_type); - let metadata = fs::metadata(path)?; + let metadata = fs::metadata(path).map_err(|error| format!("read: {error}"))?; let mtime = metadata.modified().unwrap_or(std::time::UNIX_EPOCH); let size = metadata.len(); let path_key = path.to_path_buf(); - let source = crate::project_content::read_to_string(path)?; + let _permit = graph_source_permit(size); + let source = match read_graph_source_text(path) { + Ok(SourceText::Text(source)) => source, + Ok(SourceText::TooLarge) => return Err(GRAPH_TOO_LARGE.to_string()), + Ok(SourceText::NotUtf8) => { + return Err("read: stream did not contain valid UTF-8".to_string()) + } + Ok(SourceText::Vanished) => { + return Err(format!( + "read: {}", + std::io::Error::from(std::io::ErrorKind::NotFound) + )) + } + Err(error) => return Err(format!("read: {error}")), + }; if !force_reparse { if let Some(entry) = self.entries_by_path.get(&path_key) { @@ -255,6 +272,12 @@ pub fn analyze_files_with_cache( let progress = crate::run_tool_call::progress::FileProgress::current(); for (idx, path) in files.iter().enumerate() { + if crate::inspect::scanners::source_text::graph_source_excluded(&project_root, path) { + if let Some(progress) = &progress { + progress.finished(); + } + continue; + } match cache.facts_for_file( FileId(idx), path, @@ -264,7 +287,7 @@ pub fn analyze_files_with_cache( Ok(file_facts) => facts.push(file_facts), Err(error) => errors.push(OxcEngineError { file: path.clone(), - message: format!("read: {error}"), + message: error, }), } if let Some(progress) = &progress { @@ -332,7 +355,7 @@ fn analyze_preparsed_facts( let public_api_files = normalize_option_paths(&options.public_api_files); let executable_root_exports = normalize_executable_root_exports(&options.executable_root_exports); - let file_verdicts = compute_verdicts( + let mut file_verdicts = compute_verdicts( &project_root, &resolved_modules, &entry_points, @@ -340,6 +363,11 @@ fn analyze_preparsed_facts( &executable_root_exports, options.entry_reachability, ); + // An unread importer can reference any module, including through a dynamic + // import. Missing edges cannot establish that any export is unused. + if errors.iter().any(|error| error.message == GRAPH_TOO_LARGE) { + mark_oversized_graph_uncertain(&mut file_verdicts); + } let resolved_edges = edges .iter() .filter(|edge| edge.resolved_file.is_some()) @@ -366,6 +394,15 @@ fn analyze_preparsed_facts( } } +fn mark_oversized_graph_uncertain(files: &mut [OxcFileVerdicts]) { + for file in files { + for export in &mut file.exports { + export.verdict = LivenessVerdict::Uncertain; + export.reason = GRAPH_TOO_LARGE.to_string(); + } + } +} + #[derive(Debug, Default)] struct NormalizedFileSet { files: Vec, diff --git a/crates/aft/src/inspect/scanners/dead_code.rs b/crates/aft/src/inspect/scanners/dead_code.rs index 9b8d54f38..43642edcb 100644 --- a/crates/aft/src/inspect/scanners/dead_code.rs +++ b/crates/aft/src/inspect/scanners/dead_code.rs @@ -28,6 +28,10 @@ use crate::inspect::{ }; use crate::parser::{detect_language, grammar_for, LangId}; +use super::source_text::{ + graph_source_permit, read_graph_source_text, SourceText, GRAPH_TOO_LARGE, + MAX_GRAPH_SOURCE_BYTES, +}; use super::DEFAULT_EXPORT_MARKER_KIND; pub(crate) const DEAD_CODE_FACTS_FORMAT_VERSION: u32 = 11; @@ -239,6 +243,7 @@ struct ImportedExportLiveness { #[derive(Debug, Default)] struct FileAnalysis { + oversized: bool, freshness: Option, rust_api: Option, go_dispatch: Option, @@ -329,14 +334,22 @@ impl DeadCodeFileAnalyzer { } let metadata = fs::metadata(file).ok(); + let _permit = graph_source_permit(metadata.as_ref().map_or(0, fs::Metadata::len)); #[cfg(test)] AFTER_SOURCE_METADATA.with(|hook| { if let Some(hook) = hook.borrow_mut().take() { hook(); } }); - let Ok(source) = crate::project_content::read_to_string(file) else { - return FileAnalysis::default(); + let source = match read_graph_source_text(file) { + Ok(SourceText::Text(source)) => source, + Ok(SourceText::TooLarge) => { + return FileAnalysis { + oversized: true, + ..Default::default() + } + } + _ => return FileAnalysis::default(), }; let needs_tree = needs_type_refs || needs_ts_raw_facts @@ -450,6 +463,7 @@ impl DeadCodeFileAnalyzer { }; let analysis = FileAnalysis { + oversized: false, freshness: Some(crate::cache_freshness::FileFreshness { mtime: metadata .and_then(|metadata| metadata.modified().ok()) @@ -735,6 +749,26 @@ fn gather_file_contribution( file_analyzer: &mut DeadCodeFileAnalyzer, ) -> FileContribution { let file_name = relative_path(&job.project_root, file); + if super::source_text::graph_source_excluded(&job.project_root, file) { + return FileContribution::new( + InspectCategory::DeadCode, + file.to_path_buf(), + super::source_text::metadata_freshness(file).unwrap_or(FileFreshness { + mtime: UNIX_EPOCH, + size: 0, + content_hash: cache_freshness::zero_hash(), + }), + json!({"file": file_name, "facts_format_version": DEAD_CODE_FACTS_FORMAT_VERSION, + "exports": [], "generated": false, "excluded_from_graph": true}), + ); + } + if fs::metadata(file).is_ok_and(|metadata| metadata.len() > MAX_GRAPH_SOURCE_BYTES) + || oxc_parse_errors_by_file + .get(&file_name) + .is_some_and(|errors| errors.iter().any(|error| error == GRAPH_TOO_LARGE)) + { + return oversized_file_contribution(file, &file_name); + } let generated = crate::inspect::generated::is_generated_file(&job.project_root, file); if let Some(language) = dead_code_skipped_language(file) { return FileContribution::new( @@ -761,6 +795,7 @@ fn gather_file_contribution( .unwrap_or_default() }); let FileAnalysis { + oversized, freshness, rust_api, go_dispatch, @@ -775,6 +810,9 @@ fn gather_file_contribution( trait_impl_methods, type_ref_names, } = file_analyzer.analyze_file(file, &file_name, oxc_facts.is_some()); + if oversized { + return oversized_file_contribution(file, &file_name); + } let mut payload = json!({ "file": file_name, @@ -883,6 +921,30 @@ fn gather_file_contribution( .with_type_ref_names(type_ref_names) } +fn oversized_file_contribution(file: &Path, file_name: &str) -> FileContribution { + let freshness = fs::metadata(file) + .ok() + .map(|metadata| FileFreshness { + mtime: metadata.modified().unwrap_or(UNIX_EPOCH), + size: metadata.len(), + content_hash: cache_freshness::zero_hash(), + }) + .unwrap_or(FileFreshness { + mtime: UNIX_EPOCH, + size: 0, + content_hash: cache_freshness::zero_hash(), + }); + FileContribution::new( + InspectCategory::DeadCode, + file.to_path_buf(), + freshness, + json!({ + "file": file_name, "facts_format_version": DEAD_CODE_FACTS_FORMAT_VERSION, + "exports": [], "skipped_files": [{"file": file_name, "reason": GRAPH_TOO_LARGE}] + }), + ) +} + fn oxc_fact_export_contributions(facts: &FileFacts) -> Vec { facts .exports @@ -1076,13 +1138,25 @@ fn aggregate_dead_code_contributions_incremental_body( let parsed = parse_dead_code_contributions(project_root, contributions); // Unchanged-state reuse must not hide a new gap or retain an error after // repair. Rebuild from the current facts while either state has a gap. - let has_decode_gaps = parsed.iter().any(|contribution| contribution.decode_failed) + let has_coverage_gaps = has_oversized_sources(&parsed) + || parsed.iter().any(|contribution| contribution.decode_failed) || previous.is_some_and(|state| { - state - .materialized - .iter() - .any(|contribution| contribution.decode_failed) + has_oversized_sources(&state.materialized) + || state + .materialized + .iter() + .any(|contribution| contribution.decode_failed) }); + let exclusions_changed = previous.is_some_and(|state| { + let exclusions = |facts: &[DeadCodeContribution]| { + facts + .iter() + .filter(|fact| fact.excluded_from_graph) + .map(|fact| fact.file.clone()) + .collect::>() + }; + exclusions(&parsed) != exclusions(&state.materialized) + }); let contribution_hashes = contribution_hashes( previous.map(|state| &state.contribution_hashes), contributions, @@ -1115,7 +1189,8 @@ fn aggregate_dead_code_contributions_incremental_body( && fragment.production_reachable_exports == reachable_symbols_for_file(&previous.production.reachable, file) }); - if !has_decode_gaps + if !has_coverage_gaps + && !exclusions_changed && previous.contribution_hashes == contribution_hashes && previous.public_api_files == *public_api_files && previous.roles_fingerprint == roles_fingerprint @@ -1145,7 +1220,8 @@ fn aggregate_dead_code_contributions_incremental_body( .union(&changed_graph_files) .cloned() .collect::>(); - if has_decode_gaps + if has_coverage_gaps + || exclusions_changed || previous.is_none() || previous.is_some_and(|state| { state.public_api_files != *public_api_files @@ -1664,6 +1740,7 @@ fn fold_dead_code_fragments( aggregate["parse_errors"] = Value::Array(parse_errors); } if !skipped_files.is_empty() { + aggregate["skipped_files_count"] = json!(skipped_files.len()); aggregate["skipped_files"] = Value::Array(skipped_files); } let mut excluded = ExcludedTestTally::default(); @@ -1838,6 +1915,11 @@ fn materialize_dead_code_contributions( parsed .into_iter() .map(|mut contribution| { + // Cached callgraph edges from deliberately excluded files must not + // keep another file's symbols live after their source is omitted. + if contribution.excluded_from_graph { + return contribution; + } if !affected_files.contains(&contribution.file) { if let Some(previous) = previous_by_file.get(contribution.file.as_str()) { return (*previous).clone(); @@ -2325,7 +2407,13 @@ fn aggregate_materialized_dead_code_contributions( let mut uncertain_count = 0usize; let mut uncertain_items: Vec = Vec::new(); let mut excluded_by_file: BTreeMap = BTreeMap::new(); + // A skipped source can be the only caller/importer of a symbol elsewhere. + // With no edges from that file, deadness cannot be established graph-wide. + let has_source_gaps = has_oversized_sources(facts); for contribution in parsed { + if has_source_gaps { + continue; + } let generated_file = crate::inspect::generated::is_generated_file_with_cached_hint( project_root, &contribution.file, @@ -2547,6 +2635,7 @@ fn aggregate_materialized_dead_code_contributions( aggregate["parse_errors"] = Value::Array(parse_errors); } if !skipped_files.is_empty() { + aggregate["skipped_files_count"] = json!(skipped_files.len()); aggregate["skipped_files"] = Value::Array(skipped_files); } let mut excluded = ExcludedTestTally::default(); @@ -2634,6 +2723,15 @@ fn dead_code_honesty_fields( ) } +fn has_oversized_sources(contributions: &[DeadCodeContribution]) -> bool { + contributions.iter().any(|contribution| { + contribution + .skipped_files + .iter() + .any(|skip| skip["reason"] == GRAPH_TOO_LARGE) + }) +} + fn edges_by_source( contributions: &[DeadCodeContribution], exclude_test_origins: bool, @@ -7329,11 +7427,13 @@ fn supports_type_refs(lang: LangId) -> bool { } fn collect_freshness(file: &Path) -> FileFreshness { - cache_freshness::collect(file).unwrap_or_else(|_| FileFreshness { - mtime: UNIX_EPOCH, - size: 0, - content_hash: cache_freshness::zero_hash(), - }) + super::source_text::graph_source_freshness(file) + .map(|(freshness, _permit)| freshness) + .unwrap_or_else(|_| FileFreshness { + mtime: UNIX_EPOCH, + size: 0, + content_hash: cache_freshness::zero_hash(), + }) } fn relative_path(project_root: &Path, path: &Path) -> String { @@ -7398,6 +7498,8 @@ fn normalize_path(path: &Path) -> PathBuf { #[derive(Debug, Clone, Default, Deserialize)] struct DeadCodeContribution { + #[serde(default)] + excluded_from_graph: bool, // This marker belongs only to the in-memory rollup, never cached JSON. #[serde(skip)] decode_failed: bool, @@ -9093,6 +9195,137 @@ mod tests { (temp_dir, root, paths) } + #[test] + fn oversized_js_ts_sources_are_unopened_graph_wide_gaps_including_incremental_rollups() { + for extension in ["js", "ts"] { + let importer = format!("src/importer.{extension}"); + let target = format!("src/target.{extension}"); + let handwritten = super::super::source_text::handwritten_graph_source_for_test( + 1_572_864, + "export function inLarge() {}\n", + ); + let (_temp, root, files) = fixture_project(&[ + (&importer, &handwritten), + (&target, "export function used() {}\n"), + ]); + let snapshot = snapshot( + files.clone(), + vec![ + export(&root, &importer, "inLarge", "function"), + export(&root, &target, "used", "function"), + ], + vec![], + ); + let scan_job = job(&root, files.clone(), snapshot.clone()); + let public = BTreeSet::new(); + let roles = crate::inspect::entry_points::ProjectRoles::default(); + for oxc in [false, true] { + let scan = || { + if oxc { + scan_success_with_oxc(scan_job.clone()) + } else { + run_dead_code_scan(&scan_job).outcome.unwrap() + } + }; + let rollup = |facts: &[FileContribution], + previous: Option<&DeadCodeRollupState>| { + aggregate_dead_code_contributions_incremental( + &root, + &snapshot, + facts, + &public, + &roles, + None, + None, + previous, + &BTreeSet::from([importer.clone()]), + ) + }; + let initial = scan(); + assert_eq!(initial.aggregate["count"], 2); + let (_, initial_state, _) = rollup(&initial.contributions, None); + let source_reads = + super::super::source_text::source_read_count_for_debug(&files[0]); + let head_bytes = crate::file_kind::read_probe::bytes_read(&files[0]); + fs::write(&files[0], super::super::source_text::handwritten_graph_source_for_test(5_242_880, &format!("import {{ used }} from './target.{extension}'; used();\nexport function inLarge() {{}}\n"))).unwrap(); + let probe = crate::project_content::Probe::new(&root); + let skipped = scan(); + let expected = json!([{"file": importer, "reason": "over the 4 MiB size limit"}]); + assert_eq!( + skipped.aggregate["count"], 0, + "{extension}, oxc={oxc}: {}", + skipped.aggregate + ); + assert_eq!(skipped.aggregate["complete"], false); + assert_eq!(skipped.aggregate["skipped_files"], expected); + let (cached, state, _) = rollup(&skipped.contributions, Some(&initial_state)); + assert_eq!(cached, skipped.aggregate); + let (warm, _, _) = rollup(&skipped.contributions, Some(&state)); + assert_eq!(warm, cached); + assert_eq!( + super::super::source_text::source_read_count_for_debug(&files[0]), + source_reads + ); + assert!( + crate::file_kind::read_probe::bytes_read(&files[0]) - head_bytes + <= crate::file_kind::HEAD_BYTES * 2 + ); + drop(probe); + fs::write(&files[0], &handwritten).unwrap(); + let repaired = scan(); + let (cached_repair, _, _) = rollup(&repaired.contributions, Some(&state)); + assert_eq!(cached_repair, initial.aggregate); + } + } + } + + #[test] + fn generated_and_minified_js_ts_are_unopened_deliberate_graph_exclusions() { + let (_temp, root, files) = fixture_project(&[ + ( + "src/api.generated.ts", + "import { used } from './target'; used(); export function generatedUnused() {}\n", + ), + ("src/bundle.min.js", "export function minifiedUnused() {}\n"), + ("src/target.ts", "export function used() {}\n"), + ]); + let snapshot = snapshot_with_entry_points( + files.clone(), + vec![ + export(&root, "src/api.generated.ts", "generatedUnused", "function"), + export(&root, "src/bundle.min.js", "minifiedUnused", "function"), + export(&root, "src/target.ts", "used", "function"), + ], + vec![outbound( + &root, + "src/api.generated.ts", + "", + "used", + )], + BTreeSet::from([files[0].clone()]), + ); + let scan_job = job(&root, files.clone(), snapshot); + let probe = crate::project_content::Probe::new(&root); + for oxc in [false, true] { + let success = if oxc { + scan_success_with_oxc(scan_job.clone()) + } else { + run_dead_code_scan(&scan_job).outcome.unwrap() + }; + assert_eq!(success.aggregate["count"], 1, "{}", success.aggregate); + assert_eq!(success.aggregate["generated_count"], 0); + assert_eq!(success.aggregate["complete"], true); + assert!(success.aggregate.get("skipped_files").is_none()); + assert!(aggregate_has_item( + &success.aggregate, + "src/target.ts", + "used" + )); + } + assert!(!probe.paths().contains(&files[0])); + assert!(!probe.paths().contains(&files[1])); + } + fn job(root: &Path, scope_files: Vec, snapshot: CallgraphSnapshot) -> InspectJob { InspectJob { job_id: 1, @@ -10860,7 +11093,7 @@ throughBarrel(); } #[test] - fn oxc_dead_code_buckets_generated_exports_below_headline() { + fn oxc_dead_code_excludes_generated_exports_from_the_graph() { let (_temp_dir, root, paths) = fixture_project(&[ ("package.json", r#"{"main":"src/main.ts"}"#), ( @@ -10902,20 +11135,20 @@ export function bannerDead() {} assert_eq!(first.aggregate["count"], 1, "{:#}", first.aggregate); assert_eq!( - first.aggregate["generated_count"], 2, + first.aggregate["generated_count"], 0, "{:#}", first.aggregate ); - assert_eq!(first.aggregate["total_count"], 3, "{:#}", first.aggregate); + assert_eq!(first.aggregate["total_count"], 1, "{:#}", first.aggregate); assert!(aggregate_item(&first.aggregate, "src/hand.ts", "handDead").is_some()); assert!(aggregate_generated_item( &first.aggregate, "gen/schema_pb.ts", "generatedPathDead" ) - .is_some()); + .is_none()); assert!( - aggregate_generated_item(&first.aggregate, "src/banner.ts", "bannerDead").is_some() + aggregate_generated_item(&first.aggregate, "src/banner.ts", "bannerDead").is_none() ); let item_files = first.aggregate["items"] diff --git a/crates/aft/src/inspect/scanners/source_text.rs b/crates/aft/src/inspect/scanners/source_text.rs index 6e8883b06..f9e3dc958 100644 --- a/crates/aft/src/inspect/scanners/source_text.rs +++ b/crates/aft/src/inspect/scanners/source_text.rs @@ -22,8 +22,10 @@ use serde_json::{json, Value}; pub(crate) const NOT_VALID_UTF8: &str = "not valid UTF-8"; pub(crate) const MAX_SOURCE_BYTES: u64 = crate::search_index::DEFAULT_MAX_FILE_SIZE; +pub(crate) const MAX_GRAPH_SOURCE_BYTES: u64 = 4 * 1024 * 1024; // Agent-visible reason, approved by Ufuk; change it only with his approval. pub(crate) const TOO_LARGE: &str = "over the 1 MiB size limit"; +pub(crate) const GRAPH_TOO_LARGE: &str = "over the 4 MiB size limit"; // Charge source bytes, not file count, so small files can use the whole Rayon // pool. Reservations stay live through parsing and tree traversal. Source-byte @@ -47,7 +49,11 @@ impl SourceScanBudget { } fn reserve(&self, stat_size: u64) -> SourceScanPermit<'_> { - let bytes = stat_size.min(MAX_SOURCE_BYTES); + self.reserve_with_limit(stat_size, MAX_SOURCE_BYTES) + } + + fn reserve_with_limit(&self, stat_size: u64, limit: u64) -> SourceScanPermit<'_> { + let bytes = stat_size.min(limit); let mut active = self.in_flight.lock().unwrap_or_else(|e| e.into_inner()); // One file must make progress even if the budget is smaller than that // file's reservation; other nonempty reservations wait for its release. @@ -74,6 +80,36 @@ pub(crate) fn source_scan_permit(stat_size: u64) -> SourceScanPermit<'static> { SOURCE_SCANS.reserve(stat_size) } +pub(crate) fn graph_source_permit(stat_size: u64) -> SourceScanPermit<'static> { + SOURCE_SCANS.reserve_with_limit(stat_size, MAX_GRAPH_SOURCE_BYTES) +} + +/// Generated and minified JS/TS inputs are deliberate graph exclusions, not +/// missing coverage. Name-only classifications avoid even a header read. +pub(crate) fn graph_source_excluded(project_root: &Path, path: &Path) -> bool { + use crate::file_kind::ContentKind; + use crate::parser::LangId; + let _permit = source_scan_permit(crate::file_kind::HEAD_BYTES as u64); + matches!( + crate::parser::detect_language(path), + Some(LangId::JavaScript | LangId::TypeScript | LangId::Tsx) + ) && matches!( + crate::file_kind::classify(project_root, path, None).content, + ContentKind::Generated | ContentKind::Minified + ) +} + +pub(crate) fn metadata_freshness( + path: &Path, +) -> std::io::Result { + let metadata = std::fs::metadata(path)?; + Ok(crate::cache_freshness::FileFreshness { + mtime: metadata.modified().unwrap_or(std::time::UNIX_EPOCH), + size: metadata.len(), + content_hash: crate::cache_freshness::zero_hash(), + }) +} + impl Drop for SourceScanPermit<'_> { fn drop(&mut self) { let mut active = self @@ -95,26 +131,30 @@ pub(crate) enum SourceBytes { /// Admit by metadata before opening, then bound the read too: a file can grow /// between stat and read. The extra byte detects overflow, never a partial parse. pub(crate) fn read_source_bytes(path: &Path) -> std::io::Result { + read_source_bytes_with_limit(path, MAX_SOURCE_BYTES) +} + +fn read_source_bytes_with_limit(path: &Path, limit: u64) -> std::io::Result { if crate::project_content::is_excluded(path) { return Ok(SourceBytes::Vanished); } let read = || -> std::io::Result { let metadata = std::fs::metadata(path)?; - if metadata.len() > MAX_SOURCE_BYTES { + if metadata.len() > limit { return Ok(SourceBytes::TooLarge); } let file = crate::project_content::open(path)?; - if file.metadata()?.len() > MAX_SOURCE_BYTES { + if file.metadata()?.len() > limit { return Ok(SourceBytes::TooLarge); } #[cfg(debug_assertions)] bump_source_read_count(path); let mut bytes = Vec::new(); - file.take(MAX_SOURCE_BYTES + 1).read_to_end(&mut bytes)?; + file.take(limit + 1).read_to_end(&mut bytes)?; crate::views::io::WORK .content_read_bytes .fetch_add(bytes.len() as u64, std::sync::atomic::Ordering::Relaxed); - Ok(if bytes.len() as u64 > MAX_SOURCE_BYTES { + Ok(if bytes.len() as u64 > limit { SourceBytes::TooLarge } else { SourceBytes::Bytes(bytes) @@ -134,10 +174,29 @@ pub(crate) fn source_freshness( ) -> std::io::Result<( crate::cache_freshness::FileFreshness, SourceScanPermit<'static>, +)> { + source_freshness_with_limit(path, MAX_SOURCE_BYTES) +} + +pub(crate) fn graph_source_freshness( + path: &Path, +) -> std::io::Result<( + crate::cache_freshness::FileFreshness, + SourceScanPermit<'static>, +)> { + source_freshness_with_limit(path, MAX_GRAPH_SOURCE_BYTES) +} + +fn source_freshness_with_limit( + path: &Path, + limit: u64, +) -> std::io::Result<( + crate::cache_freshness::FileFreshness, + SourceScanPermit<'static>, )> { let metadata = std::fs::metadata(path)?; - let permit = source_scan_permit(metadata.len()); - if metadata.len() > MAX_SOURCE_BYTES { + let permit = SOURCE_SCANS.reserve_with_limit(metadata.len(), limit); + if metadata.len() > limit { return Ok(( crate::cache_freshness::FileFreshness { mtime: metadata.modified().unwrap_or(std::time::UNIX_EPOCH), @@ -173,7 +232,15 @@ pub(crate) enum SourceText { /// example permission denied) are returned unchanged so each scanner keeps /// its existing contract for them. pub(crate) fn read_source_text(path: &Path) -> std::io::Result { - match read_source_bytes(path)? { + read_source_text_with_limit(path, MAX_SOURCE_BYTES) +} + +pub(crate) fn read_graph_source_text(path: &Path) -> std::io::Result { + read_source_text_with_limit(path, MAX_GRAPH_SOURCE_BYTES) +} + +fn read_source_text_with_limit(path: &Path, limit: u64) -> std::io::Result { + match read_source_bytes_with_limit(path, limit)? { SourceBytes::Bytes(bytes) => Ok(match String::from_utf8(bytes) { Ok(text) => SourceText::Text(text), Err(_) => SourceText::NotUtf8, @@ -241,6 +308,18 @@ pub fn source_read_count_for_debug(path: &Path) -> usize { .unwrap_or(0) } +#[cfg(test)] +pub(crate) fn handwritten_graph_source_for_test(bytes: usize, header: &str) -> String { + assert!(header.ends_with('\n') && header.len() <= bytes); + let line = "// handwritten source\n"; + let mut source = format!( + "{header}{}", + line.repeat((bytes - header.len()) / line.len()) + ); + source.push_str(&" ".repeat(bytes - source.len())); + source +} + #[cfg(test)] mod tests { use super::*; @@ -369,6 +448,69 @@ mod tests { assert_eq!(probe.paths(), vec![path]); } + #[test] + fn graph_sources_have_a_separate_four_mib_limit_and_byte_reservations() { + assert_eq!(MAX_SOURCE_BYTES, 1_048_576); + assert_eq!(MAX_GRAPH_SOURCE_BYTES, 4_194_304); + assert_eq!(GRAPH_TOO_LARGE, "over the 4 MiB size limit"); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("handwritten.ts"); + std::fs::write( + &path, + handwritten_graph_source_for_test(1_572_864, "// handwritten source\n"), + ) + .unwrap(); + assert!(matches!( + read_source_text(&path).unwrap(), + SourceText::TooLarge + )); + assert!( + matches!(read_graph_source_text(&path).unwrap(), SourceText::Text(source) if source.len() == 1_572_864) + ); + let budget = SourceScanBudget::new(SOURCE_SCAN_BUDGET_BYTES); + let permit = budget.reserve_with_limit(3 * 1024 * 1024, MAX_GRAPH_SOURCE_BYTES); + assert_eq!(permit.bytes, 3 * 1024 * 1024); + } + + #[test] + fn graph_classifier_reads_only_a_header_for_content_exclusions() { + use crate::inspect::{InspectCategory, InspectJob, JobKey}; + use std::sync::{Arc, RwLock}; + let dir = tempfile::tempdir().unwrap(); + for (name, source) in [ + ( + "header.ts", + format!("// @generated\n{}", "// generated source\n".repeat(10_000)), + ), + ("dense.js", "export const minified = 1;".repeat(10_000)), + ] { + let path = dir.path().join(name); + std::fs::write(&path, source).unwrap(); + let job = InspectJob { + job_id: 1, + key: JobKey::for_project_category(InspectCategory::UnusedExports), + category: InspectCategory::UnusedExports, + scope_files: vec![path.clone()], + project_root: dir.path().to_path_buf(), + inspect_dir: dir.path().join("inspect"), + config: Arc::new(crate::config::Config::default()), + symbol_cache: Arc::new(RwLock::new(crate::parser::SymbolCache::new())), + inspect_writer: true, + callgraph_writer: true, + callgraph_snapshot: None, + }; + let success = super::super::unused_exports::run_unused_exports_scan(&job) + .outcome + .unwrap(); + assert_eq!(success.aggregate["count"], 0); + assert!(success.aggregate.get("skipped_files").is_none()); + assert_eq!(source_read_count_for_debug(&path), 0); + assert!( + crate::file_kind::read_probe::bytes_read(&path) <= crate::file_kind::HEAD_BYTES + ); + } + } + #[test] fn oversized_sources_are_unopened_named_gaps_in_other_scanners() { use crate::inspect::{InspectCategory, InspectJob, JobKey, JobScope}; diff --git a/crates/aft/src/inspect/scanners/unused_exports.rs b/crates/aft/src/inspect/scanners/unused_exports.rs index 9b693f74b..d51a13c7e 100644 --- a/crates/aft/src/inspect/scanners/unused_exports.rs +++ b/crates/aft/src/inspect/scanners/unused_exports.rs @@ -8,8 +8,12 @@ use serde::Serialize; use serde_json::{json, Value}; use tree_sitter::{Node, Tree}; +use super::source_text::{ + graph_source_freshness, read_graph_source_text, SourceText, GRAPH_TOO_LARGE, + MAX_GRAPH_SOURCE_BYTES, +}; use crate::cache_freshness; -use crate::imports::{parse_file_imports, specifier_imported_name, ImportBlock, ImportStatement}; +use crate::imports::{parse_imports, specifier_imported_name, ImportBlock, ImportStatement}; use crate::inspect::job::{is_test_tree_file, ExcludedTestTally}; use crate::inspect::oxc_engine::{ ExportFact, FileFacts, LivenessVerdict, OxcEngineError, OxcEngineResult, FACTS_FORMAT_VERSION, @@ -139,7 +143,18 @@ fn run_unused_exports_legacy_scan(job: &InspectJob, started: Instant) -> Inspect let mut uncertain_count = 0usize; let mut uncertain_items = Vec::new(); let mut excluded = ExcludedTestTally::default(); + // Without the skipped files' imports, even exports in other files may + // have consumers that are absent from the graph. + let has_source_gaps = per_file.iter().any(|scan| { + scan.contribution + .contribution + .get("skipped_files") + .is_some() + }); for scan in &per_file { + if has_source_gaps { + continue; + } if public_api_entries.is_public_api_file(&scan.file_path) { continue; } @@ -239,8 +254,24 @@ fn run_unused_exports_legacy_scan(job: &InspectJob, started: Instant) -> Inspect "languages_skipped": languages_skipped, "uncertain_count": uncertain_count, "uncertain_items": uncertain_items, + "complete": true, }); excluded.write_into(&mut aggregate); + let skipped = per_file + .iter() + .flat_map(|scan| { + scan.contribution.contribution["skipped_files"] + .as_array() + .into_iter() + .flatten() + }) + .cloned() + .collect::>(); + if !skipped.is_empty() { + aggregate["complete"] = json!(false); + aggregate["skipped_files_count"] = json!(skipped.len()); + aggregate["skipped_files"] = json!(skipped); + } if !package_warnings.is_empty() { aggregate["note"] = Value::String(package_warnings.join("; ")); } @@ -284,7 +315,45 @@ fn run_unused_exports_oxc_scan( .iter() .map(|facts| (normalize_path(&facts.path), facts)) .collect::>(); - let mut contributions = Vec::new(); + let mut contributions = oxc_result + .files + .iter() + .filter_map(|file| { + let path = normalize_path(&file.file); + oxc_unused_exports_contribution( + &project_root, + facts_by_file.get(&path)?, + parse_errors_by_file.get(&path), + &skipped_files_payload, + ) + }) + .collect::>(); + let non_js_scans = job + .scope_files + .iter() + .filter(|path| !oxc_paths.contains(&normalize_path(path))) + .filter_map(|path| scan_non_js_empty_file(path, &project_root)) + .collect::>(); + let languages_skipped = non_js_scans + .iter() + .filter_map(|scan| scan.skipped_language) + .collect::>() + .into_iter() + .collect::>(); + contributions.extend(non_js_scans.into_iter().map(|scan| scan.contribution)); + let excluded_paths = contributions + .iter() + .filter(|contribution| contribution.contribution["excluded_from_graph"] == true) + .map(|contribution| normalize_path(&contribution.file_path)) + .collect::>(); + let has_source_gaps = skipped_files_payload + .iter() + .any(|skip| skip["reason"] == GRAPH_TOO_LARGE) + || contributions.iter().any(|contribution| { + contribution.contribution["skipped_files"] + .as_array() + .is_some_and(|skips| skips.iter().any(|skip| skip["reason"] == GRAPH_TOO_LARGE)) + }); let mut count = 0usize; let mut headline_items = Vec::new(); let mut generated_count = 0usize; @@ -296,18 +365,10 @@ fn run_unused_exports_oxc_scan( let mut excluded = ExcludedTestTally::default(); for file in &oxc_result.files { - if let Some(facts) = facts_by_file.get(&normalize_path(&file.file)) { - if let Some(contribution) = oxc_unused_exports_contribution( - &project_root, - facts, - parse_errors_by_file.get(&normalize_path(&file.file)), - &skipped_files_payload, - ) { - contributions.push(contribution); - } - } - - if public_api_entries.is_public_api_file(&file.file) { + if has_source_gaps + || excluded_paths.contains(&normalize_path(&file.file)) + || public_api_entries.is_public_api_file(&file.file) + { continue; } // Test files and fixtures are still analyzed (their imports keep @@ -440,20 +501,6 @@ fn run_unused_exports_oxc_scan( } } - let non_js_scans = job - .scope_files - .iter() - .filter(|path| !oxc_paths.contains(&normalize_path(path))) - .filter_map(|path| scan_non_js_empty_file(path, &project_root)) - .collect::>(); - let languages_skipped = non_js_scans - .iter() - .filter_map(|scan| scan.skipped_language) - .collect::>() - .into_iter() - .collect::>(); - contributions.extend(non_js_scans.into_iter().map(|scan| scan.contribution)); - let headline_items = crate::inspect::entry_points::rank_and_truncate_items(headline_items, &roles, None); let generated_items = @@ -495,6 +542,24 @@ fn run_unused_exports_oxc_scan( aggregate["note"] = Value::String(package_warnings.join("; ")); } add_oxc_honesty_fields(&mut aggregate, &project_root, oxc_result); + let skipped = contributions + .iter() + .flat_map(|contribution| { + contribution.contribution["skipped_files"] + .as_array() + .into_iter() + .flatten() + }) + .filter_map(|skip| { + Some(( + skip["file"].as_str()?.to_string(), + skip["reason"].as_str()?.to_string(), + )) + }) + .collect::>() + .into_iter() + .collect(); + super::source_text::write_skipped_files(&mut aggregate, skipped, None); let success = InspectScanSuccess { scanned_files: contributions @@ -522,7 +587,21 @@ fn oxc_unused_exports_contribution( parse_errors: Option<&Vec>, skipped_files: &[Value], ) -> Option { - let freshness = cache_freshness::collect(&facts.path).ok()?; + if super::source_text::graph_source_excluded(project_root, &facts.path) { + return Some(excluded_file_scan(project_root, facts.path.clone())?.contribution); + } + let (freshness, _permit) = graph_source_freshness(&facts.path).ok()?; + if freshness.size > MAX_GRAPH_SOURCE_BYTES { + return Some( + oversized_file_scan( + project_root, + facts.path.clone(), + relative_string(project_root, &facts.path), + freshness, + ) + .contribution, + ); + } let exports = facts .exports .iter() @@ -600,8 +679,10 @@ fn oxc_read_error_contribution( normalize_path(&project_root.join(&relative_file)) }; let freshness = freshness_for_oxc_error_file(&file_path)?; - let generated = crate::inspect::generated::is_generated_file(project_root, &file_path); - let contribution = json!({ + let oversized = error.message == GRAPH_TOO_LARGE; + let generated = + !oversized && crate::inspect::generated::is_generated_file(project_root, &file_path); + let mut contribution = json!({ "file": relative_file, "generated": generated, "exports": [], @@ -611,6 +692,10 @@ fn oxc_read_error_contribution( "message": error.message, }], }); + if oversized { + contribution.as_object_mut()?.remove("parse_errors"); + contribution["skipped_files"] = json!([{"file": relative_file, "reason": GRAPH_TOO_LARGE}]); + } Some(FileContribution::new( InspectCategory::UnusedExports, file_path, @@ -648,10 +733,21 @@ fn scan_non_js_empty_file(path: &Path, project_root: &Path) -> Option return None; } if is_js_ts_path(&file_path) { + if super::source_text::graph_source_excluded(project_root, &file_path) { + return excluded_file_scan(project_root, file_path); + } return None; } let relative_file = relative_string(project_root, &file_path); - let freshness = cache_freshness::collect(&file_path).ok()?; + let (freshness, _permit) = graph_source_freshness(&file_path).ok()?; + if freshness.size > MAX_GRAPH_SOURCE_BYTES { + return Some(oversized_file_scan( + project_root, + file_path, + relative_file, + freshness, + )); + } let skipped_language = detect_language(&file_path).map(language_name); Some(empty_file_scan( project_root, @@ -672,6 +768,9 @@ fn oxc_skipped_files_payload(project_root: &Path, oxc_result: &OxcEngineResult) "reason": "outside_project_root", }) }) + .chain(oxc_result.errors.iter().filter(|error| error.message == GRAPH_TOO_LARGE).map(|error| { + json!({"file": relative_string(project_root, &error.file), "reason": GRAPH_TOO_LARGE}) + })) .collect::>() } @@ -684,6 +783,7 @@ fn add_oxc_honesty_fields( let parse_errors = oxc_result .errors .iter() + .filter(|error| error.message != GRAPH_TOO_LARGE) .map(|error| { json!({ "file": relative_string(project_root, &error.file), @@ -692,6 +792,8 @@ fn add_oxc_honesty_fields( }) .collect::>(); if !skipped_files.is_empty() { + aggregate["complete"] = json!(false); + aggregate["skipped_files_count"] = json!(skipped_files.len()); aggregate["skipped_files"] = Value::Array(skipped_files); } if !parse_errors.is_empty() { @@ -724,7 +826,18 @@ fn scan_file(path: &Path, project_root: &Path) -> Option { } let relative_file = relative_string(project_root, &file_path); - let freshness = cache_freshness::collect(&file_path).ok()?; + if super::source_text::graph_source_excluded(project_root, &file_path) { + return excluded_file_scan(project_root, file_path); + } + let (freshness, _permit) = graph_source_freshness(&file_path).ok()?; + if freshness.size > MAX_GRAPH_SOURCE_BYTES { + return Some(oversized_file_scan( + project_root, + file_path, + relative_file, + freshness, + )); + } let Some(lang) = detect_language(&file_path) else { return Some(empty_file_scan( project_root, @@ -745,7 +858,27 @@ fn scan_file(path: &Path, project_root: &Path) -> Option { )); } - let Ok((source, tree, import_block)) = parse_file_imports(&file_path, lang) else { + let source = match read_graph_source_text(&file_path) { + Ok(SourceText::Text(source)) => source, + Ok(SourceText::TooLarge) => { + return Some(oversized_file_scan( + project_root, + file_path, + relative_file, + freshness, + )) + } + _ => { + return Some(empty_file_scan( + project_root, + file_path, + relative_file, + freshness, + None, + )) + } + }; + let Ok(tree) = crate::parser::parse_source_with_cached_parser(&file_path, &source, lang) else { return Some(empty_file_scan( project_root, file_path, @@ -754,6 +887,7 @@ fn scan_file(path: &Path, project_root: &Path) -> Option { None, )); }; + let import_block = parse_imports(&source, &tree, lang); let generated = crate::inspect::generated::is_generated_file_from_source(project_root, &file_path, &source); @@ -781,6 +915,48 @@ fn scan_file(path: &Path, project_root: &Path) -> Option { }) } +fn oversized_file_scan( + _project_root: &Path, + file_path: PathBuf, + relative_file: String, + freshness: cache_freshness::FileFreshness, +) -> FileScan { + FileScan { + contribution: FileContribution::new( + InspectCategory::UnusedExports, + file_path.clone(), + freshness, + json!({"file": relative_file, "generated": false, "exports": [], "imports": [], + "skipped_files": [{"file": relative_file, "reason": GRAPH_TOO_LARGE}]}), + ), + file_path, + relative_file, + exports: Vec::new(), + imports: Vec::new(), + skipped_language: None, + generated: false, + } +} + +fn excluded_file_scan(project_root: &Path, file_path: PathBuf) -> Option { + let relative_file = relative_string(project_root, &file_path); + let freshness = super::source_text::metadata_freshness(&file_path).ok()?; + Some(FileScan { + contribution: FileContribution::new( + InspectCategory::UnusedExports, + file_path.clone(), + freshness, + json!({"file": relative_file, "generated": false, "excluded_from_graph": true, "exports": [], "imports": []}), + ), + file_path, + relative_file, + exports: Vec::new(), + imports: Vec::new(), + skipped_language: None, + generated: false, + }) +} + fn empty_file_scan( project_root: &Path, file_path: PathBuf, @@ -1645,7 +1821,7 @@ mod tests { } #[test] - fn unused_exports_buckets_generated_exports_below_headline() { + fn unused_exports_excludes_generated_exports_from_the_graph() { let (_temp_dir, root, paths) = fixture_project(&[ ( "src/hand.ts", @@ -1681,13 +1857,13 @@ export function bannerUnused() {} assert_eq!(first, second, "twice-cold scan must be deterministic"); assert_eq!(first["count"], 1, "{first:#}"); - assert_eq!(first["generated_count"], 2, "{first:#}"); - assert_eq!(first["total_count"], 3, "{first:#}"); + assert_eq!(first["generated_count"], 0, "{first:#}"); + assert_eq!(first["total_count"], 1, "{first:#}"); assert!(aggregate_item(&first, "src/hand.ts", "handUnused").is_some()); assert!( - aggregate_generated_item(&first, "gen/schema_pb.ts", "generatedPathUnused").is_some() + aggregate_generated_item(&first, "gen/schema_pb.ts", "generatedPathUnused").is_none() ); - assert!(aggregate_generated_item(&first, "src/banner.ts", "bannerUnused").is_some()); + assert!(aggregate_generated_item(&first, "src/banner.ts", "bannerUnused").is_none()); let item_files = first["items"] .as_array()