diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2c0e47..f6cfd83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,10 +21,13 @@ jobs: - name: Check formatting - run: cargo fmt -- --check + run: cargo fmt --all -- --check - name: Lint code - run: cargo clippy -- -D warnings + run: cargo clippy --all-targets --all-features -- -D warnings - - name: Run tests - run: cargo test --verbose + - name: Run all-feature tests + run: cargo test --all-features --verbose + + - name: Run minimal-feature tests + run: cargo test --no-default-features --verbose diff --git a/Cargo.lock b/Cargo.lock index d2a95b4..1923b57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,6 +25,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "anes" version = "0.1.6" @@ -67,7 +76,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -78,7 +87,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -131,12 +140,6 @@ version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" -[[package]] -name = "bytes" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" - [[package]] name = "cast" version = "0.3.0" @@ -196,17 +199,14 @@ dependencies = [ "clap", "criterion", "crossbeam-channel", - "http", "ignore", "indicatif", "memmap2", - "num_cpus", "rayon", "regex", "serde", "serde_json", "tempfile", - "tokio", "toml", "tree-sitter", "tree-sitter-c", @@ -270,38 +270,36 @@ checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "console" -version = "0.15.11" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" dependencies = [ "encode_unicode", "libc", - "once_cell", "unicode-width", - "windows-sys 0.59.0", + "windows-sys", ] [[package]] name = "criterion" -version = "0.5.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" dependencies = [ + "alloca", "anes", "cast", "ciborium", "clap", "criterion-plot", - "is-terminal", "itertools", "num-traits", - "once_cell", "oorandom", + "page_size", "plotters", "rayon", "regex", "serde", - "serde_derive", "serde_json", "tinytemplate", "walkdir", @@ -309,9 +307,9 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.5.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" dependencies = [ "cast", "itertools", @@ -388,7 +386,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -441,9 +439,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "heck" @@ -451,22 +449,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" - -[[package]] -name = "http" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" -dependencies = [ - "bytes", - "itoa", -] - [[package]] name = "ignore" version = "0.4.25" @@ -485,9 +467,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown", @@ -495,28 +477,17 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.11" +version = "0.18.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" dependencies = [ "console", - "number_prefix", "portable-atomic", "unicode-width", + "unit-prefix", "web-time", ] -[[package]] -name = "is-terminal" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.61.2", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -525,9 +496,9 @@ checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itertools" -version = "0.10.5" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" dependencies = [ "either", ] @@ -590,22 +561,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "num_cpus" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - [[package]] name = "once_cell" version = "1.21.3" @@ -625,10 +580,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] -name = "pin-project-lite" -version = "0.2.16" +name = "page_size" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] [[package]] name = "plotters" @@ -774,7 +733,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -844,11 +803,11 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.9" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -890,7 +849,7 @@ dependencies = [ "getrandom", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -909,73 +868,50 @@ dependencies = [ "serde_json", ] -[[package]] -name = "tokio" -version = "1.49.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" -dependencies = [ - "pin-project-lite", - "tokio-macros", -] - -[[package]] -name = "tokio-macros" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "toml" -version = "0.8.23" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" dependencies = [ - "serde", + "indexmap", + "serde_core", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_parser", + "toml_writer", + "winnow", ] [[package]] name = "toml_datetime" -version = "0.6.11" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ - "serde", + "serde_core", ] [[package]] -name = "toml_edit" -version = "0.22.27" +name = "toml_parser" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime", - "toml_write", "winnow", ] [[package]] -name = "toml_write" -version = "0.1.2" +name = "toml_writer" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" [[package]] name = "tree-sitter" -version = "0.25.10" +version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78f873475d258561b06f1c595d93308a7ed124d9977cb26b148c2084a4a3cc87" +checksum = "af1c71c1c4cc0920b20d6b0f6572e7682cd07a6a2faec71067a31fa394c586df" dependencies = [ "cc", "regex", @@ -1113,6 +1049,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + [[package]] name = "utf8parse" version = "0.2.2" @@ -1218,29 +1160,42 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys", ] [[package]] -name = "windows-link" -version = "0.2.1" +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows-sys" -version = "0.59.0" +name = "windows-link" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-sys" @@ -1251,78 +1206,11 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - [[package]] name = "winnow" -version = "0.7.14" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" -dependencies = [ - "memchr", -] +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" [[package]] name = "wit-bindgen" diff --git a/Cargo.toml b/Cargo.toml index 56ee4c7..a4af229 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,13 +2,25 @@ name = "cipherscope" version = "0.2.5" edition = "2024" +rust-version = "1.88" description = "A high-performance, language-aware tool for scanning source code to detect cryptographic libraries and algorithms usage." license = "MIT" repository = "https://github.com/script3r/cipherscope" readme = "README.md" keywords = ["crypto", "static-analysis", "security", "inventory", "tree-sitter"] categories = ["development-tools", "command-line-utilities"] -include = ["src/**", "patterns.toml", "Cargo.toml", "README.md", "LICENSE"] +include = [ + "src/**", + "benches/**", + "fixtures/**", + "tests/**", + "patterns.toml", + "Cargo.toml", + "README.md", + "BENCHMARK.md", + "DESIGN.md", + "LICENSE", +] [dependencies] anyhow = "1.0" @@ -20,13 +32,10 @@ rayon = "1.10" regex = { version = "1.11", default-features = false, features = ["std", "unicode-perl"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -toml = "0.8" -indicatif = "0.17" -tree-sitter = "0.25" -num_cpus = "1.17" +toml = "1.1" +indicatif = "0.18" +tree-sitter = "0.26" crossbeam-channel = "0.5" -tokio = { version = "1.41", features = ["macros", "rt-multi-thread"] } -http = "1.1" tree-sitter-c = { version = "0.24", optional = true } tree-sitter-cpp = { version = "0.23", optional = true } tree-sitter-java = { version = "0.23", optional = true } @@ -56,7 +65,7 @@ lang-typescript = ["tree-sitter-typescript"] [dev-dependencies] assert_cmd = "2.0" tempfile = "3.10" -criterion = "0.5" +criterion = "0.8" [[bench]] name = "scan_bench" diff --git a/README.md b/README.md index 9660359..3768c28 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # cipherscope
- CipherScope Logo + CipherScope Logo
[![CI](https://github.com/script3r/cipherscope/actions/workflows/ci.yml/badge.svg)](https://github.com/script3r/cipherscope/actions/workflows/ci.yml) @@ -14,7 +14,7 @@ - **Language-Aware**: Uses Tree-sitter parsers to reduce false positives by understanding code structure. - **Inventory-First**: Focused on assembling a reliable crypto usage inventory across large repos. - **Extensible Patterns**: Easily add new libraries and algorithms via a simple TOML configuration. -- **Broad Language Support**: Currently supports C, C++, Java, Python, Go, Swift, PHP, Objective-C, and Rust. +- **Broad Language Support**: Currently supports C, C++, Java, Python, Go, Swift, PHP, Objective-C, Rust, JavaScript, and TypeScript. - **Developer Friendly**: JSONL output for easy integration with CI/CD pipelines and security tools. - **Cross-Platform**: Native binaries for macOS, Linux, and Windows. @@ -52,11 +52,11 @@ cipherscope [OPTIONS] - `-r, --roots `: One or more root directories to scan (default: `.`). - `-e, --exclude `: Glob patterns to exclude from the scan (e.g., `vendor/**`). -- `-p, --patterns `: Path to the `patterns.toml` file (default: `patterns.toml`). +- `-p, --patterns `: Path to a custom `patterns.toml` file (default: patterns embedded in the binary). - `-o, --output `: Output file path for JSONL results (default: stdout). - `--threads `: Maximum number of parallel threads to use. - `-v, --progress`: Enable progress bars for discovery and scanning. -- `--gitignore`: Respect `.gitignore` files (default: true). +- `--gitignore[=]`: Respect `.gitignore` files (default: true; use `--gitignore=false` to disable). - `--max-file-mb `: Skip files larger than this size in megabytes (default: 1). ### Examples diff --git a/benches/file_size_bench.rs b/benches/file_size_bench.rs index 0402b0f..6f38c54 100644 --- a/benches/file_size_bench.rs +++ b/benches/file_size_bench.rs @@ -51,7 +51,9 @@ fn bench_file_sizes(c: &mut Criterion) { group.sample_size(10); group.measurement_time(Duration::from_secs(5)); - let threads = num_cpus::get(); + let threads = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1); for size_kb in sizes_kb { let config = FixtureConfig { diff --git a/benches/large_fixture_bench.rs b/benches/large_fixture_bench.rs index 559f859..f91741f 100644 --- a/benches/large_fixture_bench.rs +++ b/benches/large_fixture_bench.rs @@ -76,7 +76,9 @@ fn bench_large_fixture(c: &mut Criterion) { group.measurement_time(Duration::from_secs(10)); group.throughput(Throughput::Bytes(total_bytes)); - let max_cpus = num_cpus::get(); + let max_cpus = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1); let thread_counts: Vec = vec![1, max_cpus] .into_iter() .filter(|&t| t > 0 && t <= max_cpus) @@ -151,7 +153,9 @@ fn bench_nested_directories(c: &mut Criterion) { group.measurement_time(Duration::from_secs(10)); group.throughput(Throughput::Elements(total_files as u64)); - let threads = num_cpus::get(); + let threads = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1); group.bench_function("scan", |b| { b.iter_custom(|iters| { let mut total = Duration::ZERO; diff --git a/benches/memory_bench.rs b/benches/memory_bench.rs index 1bf39ab..6dca665 100644 --- a/benches/memory_bench.rs +++ b/benches/memory_bench.rs @@ -117,7 +117,9 @@ fn bench_memory_scaling(c: &mut Criterion) { group.sample_size(10); group.measurement_time(Duration::from_secs(5)); - let threads = num_cpus::get(); + let threads = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1); for count in file_counts { let config = FixtureConfig { diff --git a/benches/scale_bench.rs b/benches/scale_bench.rs index 3df5827..0b2e5ff 100644 --- a/benches/scale_bench.rs +++ b/benches/scale_bench.rs @@ -51,7 +51,9 @@ fn bench_directory_scale(c: &mut Criterion) { group.sample_size(10); group.measurement_time(Duration::from_secs(5)); - let threads = num_cpus::get(); + let threads = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1); for count in file_counts { let config = FixtureConfig { @@ -101,7 +103,9 @@ fn bench_crypto_density(c: &mut Criterion) { group.sample_size(10); group.measurement_time(Duration::from_secs(5)); - let threads = num_cpus::get(); + let threads = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1); for density in densities { let config = FixtureConfig { diff --git a/benches/scan_bench.rs b/benches/scan_bench.rs index 1c277d3..906a815 100644 --- a/benches/scan_bench.rs +++ b/benches/scan_bench.rs @@ -36,7 +36,12 @@ fn bench_scan(c: &mut Criterion) { group.sample_size(10); group.measurement_time(Duration::from_secs(10)); - let thread_counts = [1usize, num_cpus::get()]; + let thread_counts = [ + 1usize, + std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1), + ]; for threads in thread_counts { group.bench_with_input( BenchmarkId::new("fixtures", threads), diff --git a/benches/scan_large_bench.rs b/benches/scan_large_bench.rs index 38a1bf8..7b10e01 100644 --- a/benches/scan_large_bench.rs +++ b/benches/scan_large_bench.rs @@ -61,7 +61,14 @@ fn bench_scan_large(c: &mut Criterion) { Some(parsed) } }) - .unwrap_or_else(|| vec![1usize, num_cpus::get()]); + .unwrap_or_else(|| { + vec![ + 1usize, + std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1), + ] + }); for threads in thread_counts { group.bench_with_input( BenchmarkId::new("fixture", threads), diff --git a/benches/thread_scaling_bench.rs b/benches/thread_scaling_bench.rs index e3bb33d..824da03 100644 --- a/benches/thread_scaling_bench.rs +++ b/benches/thread_scaling_bench.rs @@ -57,7 +57,9 @@ fn bench_thread_scaling(c: &mut Criterion) { let roots = vec![fixture_path.clone()]; // Thread counts to test - let max_cpus = num_cpus::get(); + let max_cpus = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1); let thread_counts: Vec = if is_extended() { vec![1, 2, 4, 8, 16, 32, 64] .into_iter() diff --git a/src/lib.rs b/src/lib.rs index 0cf5017..61915d3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,43 +38,60 @@ pub fn scan_snippet( lang: patterns::Language, source_label: &str, ) -> Result> { - if !scan::has_anchor_hint(lang, content, &PATTERNS) { + scan_with_patterns(content, lang, source_label, &PATTERNS) +} + +/// Scans source text using a caller-supplied pattern set. +/// +/// This is the shared scanning pipeline used by both the library convenience API +/// and the command-line application. +pub fn scan_with_patterns( + content: &str, + lang: patterns::Language, + source_label: &str, + patterns: &patterns::PatternSet, +) -> Result> { + if !scan::has_anchor_hint(lang, content, patterns) { return Ok(Vec::new()); } let tree = scan::parse(lang, content)?; - let lib_hits = scan::find_library_anchors(lang, content, &tree, &PATTERNS); + let lib_hits = scan::find_library_anchors(lang, content, &tree, patterns); if lib_hits.is_empty() { return Ok(Vec::new()); } let mut findings = Vec::new(); - let mut seen: HashSet = HashSet::new(); let mut alg_hits_all = Vec::new(); + let mut scanned_libraries = HashSet::new(); for lib in lib_hits { + if !scanned_libraries.insert(lib.library_name) { + continue; + } + let evidence = Evidence { line: lib.line, column: lib.column, }; - let finding = Finding { + findings.push(Finding { asset_type: "library".to_string(), identifier: lib.library_name.to_string(), path: source_label.to_string(), evidence, metadata: HashMap::new(), - }; - let key = format!("lib|{}", finding.identifier); - if seen.insert(key) { - findings.push(finding); - } + }); - let alg_hits = scan::find_algorithms(lang, content, &tree, &PATTERNS, lib.library_name); + let alg_hits = scan::find_algorithms(lang, content, &tree, patterns, lib.library_name); alg_hits_all.extend(alg_hits); } let alg_hits_all = scan::dedupe_more_specific_hits(alg_hits_all); + let mut seen_algorithms = HashSet::new(); for alg in alg_hits_all { + if !seen_algorithms.insert((alg.algorithm_name, alg.line, alg.column)) { + continue; + } let mut metadata = HashMap::new(); for (k, v) in alg.metadata { metadata.insert(k.to_string(), v); @@ -83,20 +100,13 @@ pub fn scan_snippet( line: alg.line, column: alg.column, }; - let finding = Finding { + findings.push(Finding { asset_type: "algorithm".to_string(), identifier: alg.algorithm_name.to_string(), path: source_label.to_string(), evidence, metadata, - }; - let key = format!( - "alg|{}|{}:{}", - finding.identifier, finding.evidence.line, finding.evidence.column - ); - if seen.insert(key) { - findings.push(finding); - } + }); } Ok(findings) diff --git a/src/main.rs b/src/main.rs index ea7efd2..2e15ead 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,8 +4,8 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use ahash::{AHashMap as HashMap, AHashSet as HashSet}; -use anyhow::{Context, Result}; +use anyhow::{Context, Result, anyhow}; +use cipherscope::{DEFAULT_PATTERNS, Finding, patterns, scan, scan_with_patterns}; use clap::Parser; use crossbeam_channel as channel; use ignore::WalkBuilder; @@ -14,12 +14,6 @@ use ignore::types::TypesBuilder; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use memmap2::Mmap; use rayon::prelude::*; -use serde::Serialize; - -mod patterns; -mod scan; - -const DEFAULT_PATTERNS: &str = include_str!("../patterns.toml"); #[derive(Parser, Debug)] #[command( @@ -45,7 +39,7 @@ struct Cli { output: String, /// Max parallelism - #[arg(long, default_value_t = num_cpus::get())] + #[arg(long, default_value_t = default_thread_count(), value_parser = parse_thread_count)] threads: usize, /// Show progress bars @@ -53,7 +47,14 @@ struct Cli { progress: bool, /// Respect .gitignore files (enabled by default) - #[arg(long, default_value = "true")] + #[arg( + long, + default_value_t = true, + action = clap::ArgAction::Set, + num_args = 0..=1, + default_missing_value = "true", + value_name = "BOOL" + )] gitignore: bool, /// Skip files larger than this many megabytes @@ -61,25 +62,20 @@ struct Cli { max_file_mb: Option, } -#[derive(Serialize, Clone)] -struct Evidence { - line: usize, - column: usize, -} - -#[derive(Serialize, Clone)] -struct Finding { - #[serde(rename = "assetType")] - asset_type: String, - identifier: String, - path: String, - evidence: Evidence, - #[serde(skip_serializing_if = "map_is_empty")] - metadata: HashMap, +fn default_thread_count() -> usize { + std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1) } -fn map_is_empty(m: &HashMap) -> bool { - m.is_empty() +fn parse_thread_count(value: &str) -> std::result::Result { + let threads = value + .parse::() + .map_err(|_| "thread count must be a positive integer".to_string())?; + if threads == 0 { + return Err("thread count must be at least 1".to_string()); + } + Ok(threads) } /// Main entry point for the scanner. @@ -98,7 +94,7 @@ fn main() -> Result<()> { rayon::ThreadPoolBuilder::new() .num_threads(cli.threads) .build_global() - .ok(); + .context("configure scanner thread pool")?; let patterns_text = if let Some(path) = cli.patterns.as_ref() { std::fs::read_to_string(path) @@ -142,20 +138,22 @@ fn main() -> Result<()> { pb }); - // Use an unbounded channel to avoid deadlocks - // We'll rely on the file processing being the bottleneck, not the writer - let (tx, rx) = channel::unbounded::(); - let output_path = cli.output.clone(); + // Bounded queues apply backpressure when discovery or scanning outruns its consumer. + let queue_capacity = cli.threads.saturating_mul(4).max(1); + let (tx, rx) = channel::bounded::(queue_capacity); + let (writer, output_scan_path): (Box, Option) = if cli.output == "-" + { + (Box::new(std::io::stdout()), None) + } else { + let file = File::create(&cli.output).with_context(|| format!("create {}", cli.output))?; + let output_scan_path = std::fs::canonicalize(&cli.output) + .with_context(|| format!("resolve output path: {}", cli.output))?; + (Box::new(BufWriter::new(file)), Some(output_scan_path)) + }; let found_count_writer = found_count.clone(); let scan_bar_writer = scan_bar.clone(); let writer_handle = std::thread::spawn(move || -> Result<()> { - let mut writer: Box = if output_path == "-" { - Box::new(std::io::stdout()) - } else { - let f = - File::create(&output_path).with_context(|| format!("create {}", output_path))?; - Box::new(BufWriter::new(f)) - }; + let mut writer = writer; for finding in rx.iter() { serde_json::to_writer(&mut writer, &finding)?; writer.write_all(b"\n")?; @@ -171,7 +169,7 @@ fn main() -> Result<()> { // Streaming architecture: WalkBuilder sends files to channel, rayon workers process immediately // This eliminates the mutex contention and synchronous barrier of collect-then-process - let (file_tx, file_rx) = channel::unbounded::(); + let (file_tx, file_rx) = channel::bounded::(queue_capacity); let patterns_for_discovery = patterns.clone(); let file_count_discovery = file_count.clone(); let skipped_oversize_discovery = skipped_oversize_count.clone(); @@ -270,9 +268,19 @@ fn main() -> Result<()> { let file_count = file_count_discovery.clone(); let discovery_bar = discovery_bar.clone(); let skipped_oversize = skipped_oversize_discovery.clone(); + let output_scan_path = output_scan_path.clone(); Box::new(move |entry| { match entry { Ok(e) if e.file_type().map(|t| t.is_file()).unwrap_or(false) => { + // Never scan the output while it is being written, even if it has a + // supported source extension and lives below a requested root. + if output_scan_path.as_ref().is_some_and(|output| { + e.path().file_name() == output.file_name() + && e.path().canonicalize().is_ok_and(|path| path == *output) + }) { + return ignore::WalkState::Continue; + } + // Skip files larger than the configured limit (if any) if let (Some(limit), Ok(meta)) = (max_bytes, e.metadata()) && meta.len() > limit @@ -325,11 +333,17 @@ fn main() -> Result<()> { } // Wait for all scanning to complete - scanner_handle.join().expect("scanner thread panicked"); + scanner_handle + .join() + .map_err(|_| anyhow!("scanner thread panicked"))?; // All files have been processed drop(tx); + writer_handle + .join() + .map_err(|_| anyhow!("writer thread panicked"))??; + if let Some(pb) = &scan_bar { pb.finish_with_message(format!( "Scanned {} files, found {} cryptographic items", @@ -338,8 +352,6 @@ fn main() -> Result<()> { )); } - writer_handle.join().unwrap()?; - if !cli.progress && cli.output != "-" { eprintln!( "Scanned {} files, found {} cryptographic items", @@ -369,6 +381,9 @@ fn process_file( tx: &channel::Sender, ) -> Result<()> { let file = File::open(path).with_context(|| format!("open {}", path.display()))?; + if file.metadata()?.len() == 0 { + return Ok(()); + } let mmap = unsafe { Mmap::map(&file)? }; // Decode file contents safely; fall back to lossy if not valid UTF-8 to avoid UB let content_owned; @@ -380,81 +395,12 @@ fn process_file( } }; - // Avoid per-file canonicalize overhead; preserve original path for output - let absolute_path = path.to_path_buf(); - let Some(lang) = scan::language_from_path(path) else { return Ok(()); }; - if !scan::has_anchor_hint(lang, content, patterns) { - return Ok(()); - } - - let tree = scan::parse(lang, content)?; - - // 1) library anchors - let lib_hits = scan::find_library_anchors(lang, content, &tree, patterns); - if lib_hits.is_empty() { - return Ok(()); - } - - let mut seen: HashSet = HashSet::new(); - let mut alg_hits_all = Vec::new(); - - for lib in lib_hits { - let evidence = Evidence { - line: lib.line, - column: lib.column, - }; - let finding = Finding { - asset_type: "library".to_string(), - identifier: lib.library_name.to_string(), - path: absolute_path.to_string_lossy().to_string(), - evidence, - metadata: HashMap::new(), - }; - let key = format!("lib|{}", finding.identifier); - if seen.insert(key) { - // Use blocking send but log if it takes too long - if let Err(e) = tx.send(finding) { - eprintln!("error: writer thread has stopped: {}", e); - return Ok(()); - } - } - - // 2) algorithms for this library - let alg_hits = scan::find_algorithms(lang, content, &tree, patterns, lib.library_name); - alg_hits_all.extend(alg_hits); - } - - let alg_hits_all = scan::dedupe_more_specific_hits(alg_hits_all); - for alg in alg_hits_all { - let mut metadata = HashMap::new(); - for (k, v) in alg.metadata { - metadata.insert(k.to_string(), v); - } - let evidence = Evidence { - line: alg.line, - column: alg.column, - }; - let finding = Finding { - asset_type: "algorithm".to_string(), - identifier: alg.algorithm_name.to_string(), - path: absolute_path.to_string_lossy().to_string(), - evidence, - metadata, - }; - let key = format!( - "alg|{}|{}:{}", - finding.identifier, finding.evidence.line, finding.evidence.column - ); - if seen.insert(key) { - // Use blocking send but log if it takes too long - if let Err(e) = tx.send(finding) { - eprintln!("error: writer thread has stopped: {}", e); - return Ok(()); - } - } + let source_label = path.to_string_lossy(); + for finding in scan_with_patterns(content, lang, &source_label, patterns)? { + tx.send(finding).context("writer thread stopped")?; } Ok(()) diff --git a/src/patterns.rs b/src/patterns.rs index 6bcfc96..9601017 100644 --- a/src/patterns.rs +++ b/src/patterns.rs @@ -1,7 +1,7 @@ -use anyhow::{Context, Result}; +use anyhow::{Context, Result, bail}; use regex::{Regex, RegexSet}; use serde::Deserialize; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Language { @@ -18,6 +18,41 @@ pub enum Language { TypeScript, } +impl Language { + pub const fn is_enabled(self) -> bool { + match self { + Self::C => cfg!(feature = "lang-c"), + Self::Cpp => cfg!(feature = "lang-cpp"), + Self::Java => cfg!(feature = "lang-java"), + Self::Python => cfg!(feature = "lang-python"), + Self::Go => cfg!(feature = "lang-go"), + Self::Swift => cfg!(feature = "lang-swift"), + Self::Php => cfg!(feature = "lang-php"), + Self::Objc => cfg!(feature = "lang-objc"), + Self::Rust => cfg!(feature = "lang-rust"), + Self::JavaScript => cfg!(feature = "lang-javascript"), + Self::TypeScript => cfg!(feature = "lang-typescript"), + } + } + + fn from_pattern_name(name: &str) -> Option { + match name { + "C" => Some(Self::C), + "C++" => Some(Self::Cpp), + "Java" => Some(Self::Java), + "Python" => Some(Self::Python), + "Go" => Some(Self::Go), + "Swift" => Some(Self::Swift), + "PHP" => Some(Self::Php), + "ObjC" => Some(Self::Objc), + "Rust" => Some(Self::Rust), + "JavaScript" => Some(Self::JavaScript), + "TypeScript" => Some(Self::TypeScript), + _ => None, + } + } +} + #[derive(Debug, Deserialize)] struct RawPatternSet { #[allow(dead_code)] @@ -131,28 +166,19 @@ impl PatternSet { pub fn from_toml(text: &str) -> Result { let raw: RawPatternSet = toml::from_str(text).context("parse patterns.toml")?; let mut libraries = Vec::new(); + let mut library_names = HashSet::new(); for lib in raw.library { let languages = lib .languages .into_iter() - .filter_map(|s| match s.as_str() { - "C" => Some(Language::C), - "C++" => Some(Language::Cpp), - "Java" => Some(Language::Java), - "Python" => Some(Language::Python), - "Go" => Some(Language::Go), - "Swift" => Some(Language::Swift), - "PHP" => Some(Language::Php), - "ObjC" => Some(Language::Objc), - "Rust" => Some(Language::Rust), - "JavaScript" => Some(Language::JavaScript), - "TypeScript" => Some(Language::TypeScript), - _ => None, - }) + .filter_map(|name| Language::from_pattern_name(&name)) .collect::>(); if languages.is_empty() { continue; } + if !library_names.insert(lib.name.clone()) { + bail!("duplicate library name {:?}", lib.name); + } let mut include_regexes = Vec::new(); let mut api_regexes = Vec::new(); if let Some(p) = lib.patterns { @@ -330,7 +356,7 @@ impl PatternSet { } pub fn supports_language(&self, lang: Language) -> bool { - self.libraries.iter().any(|l| l.languages.contains(&lang)) + lang.is_enabled() && self.libraries.iter().any(|l| l.languages.contains(&lang)) } } @@ -351,3 +377,30 @@ fn toml_value_to_json(v: toml::Value) -> serde_json::Value { toml::Value::Datetime(dt) => serde_json::Value::String(dt.to_string()), } } + +#[cfg(test)] +mod tests { + use super::PatternSet; + + #[test] + fn rejects_duplicate_library_names() { + let error = PatternSet::from_toml( + r#" +[[library]] +name = "Example" +languages = ["Rust"] + +[[library]] +name = "Example" +languages = ["Go"] +"#, + ) + .unwrap_err(); + + assert!( + error + .to_string() + .contains("duplicate library name \"Example\"") + ); + } +} diff --git a/src/scan.rs b/src/scan.rs index 30efcb6..cfe2b41 100644 --- a/src/scan.rs +++ b/src/scan.rs @@ -1,18 +1,20 @@ use ahash::{AHashMap as HashMap, AHashSet as HashSet}; use anyhow::{Context, Result}; -use tree_sitter::{Language as TsLanguage, Node, Parser, Point, Tree}; +use std::ops::ControlFlow; +use std::time::{Duration, Instant}; +use tree_sitter::{Language as TsLanguage, Node, ParseOptions, Parser, Point, Tree}; use crate::patterns::{Language, ParameterPattern, PatternSet}; macro_rules! define_ts_lang { ($name:ident, $feature:literal, $grammar:expr) => { #[cfg(feature = $feature)] - fn $name() -> TsLanguage { - $grammar.into() + fn $name() -> Result { + Ok($grammar.into()) } #[cfg(not(feature = $feature))] - fn $name() -> TsLanguage { - unreachable!(concat!($feature, " feature disabled")) + fn $name() -> Result { + anyhow::bail!(concat!($feature, " feature is disabled")) } }; } @@ -57,21 +59,24 @@ pub struct AlgorithmHit<'a> { pub fn language_from_path(path: &std::path::Path) -> Option { let ext = path.extension()?.to_str()?.to_ascii_lowercase(); - match ext.as_str() { - "c" => Some(Language::C), - "h" => Some(Language::Cpp), // Parse .h as C++ since it's backwards compatible with C - "cc" | "cpp" | "cxx" | "hpp" | "hh" | "hxx" => Some(Language::Cpp), - "java" => Some(Language::Java), - "py" => Some(Language::Python), - "go" => Some(Language::Go), - "swift" => Some(Language::Swift), - "php" | "hack" => Some(Language::Php), - "m" | "mm" => Some(Language::Objc), - "rs" => Some(Language::Rust), - "js" | "mjs" | "cjs" | "jsx" => Some(Language::JavaScript), - "ts" | "mts" | "cts" | "tsx" => Some(Language::TypeScript), - _ => None, - } + let language = match ext.as_str() { + "c" => Language::C, + // Prefer the C++ grammar for ambiguous headers, but keep C-only builds useful. + "h" if Language::Cpp.is_enabled() => Language::Cpp, + "h" => Language::C, + "cc" | "cpp" | "cxx" | "hpp" | "hh" | "hxx" => Language::Cpp, + "java" => Language::Java, + "py" => Language::Python, + "go" => Language::Go, + "swift" => Language::Swift, + "php" | "hack" => Language::Php, + "m" | "mm" => Language::Objc, + "rs" => Language::Rust, + "js" | "mjs" | "cjs" | "jsx" => Language::JavaScript, + "ts" | "mts" | "cts" | "tsx" => Language::TypeScript, + _ => return None, + }; + language.is_enabled().then_some(language) } /// Parses the given string content into a `tree-sitter` syntax tree. @@ -82,26 +87,37 @@ pub fn language_from_path(path: &std::path::Path) -> Option { pub fn parse(lang: Language, content: &str) -> Result { let mut parser = Parser::new(); let ts_lang = match lang { - Language::C => ts_lang_c(), - Language::Cpp => ts_lang_cpp(), - Language::Java => ts_lang_java(), - Language::Python => ts_lang_python(), - Language::Go => ts_lang_go(), - Language::Swift => ts_lang_swift(), - Language::Php => ts_lang_php(), - Language::Objc => ts_lang_objc(), - Language::Rust => ts_lang_rust(), - Language::JavaScript => ts_lang_javascript(), - Language::TypeScript => ts_lang_typescript(), + Language::C => ts_lang_c()?, + Language::Cpp => ts_lang_cpp()?, + Language::Java => ts_lang_java()?, + Language::Python => ts_lang_python()?, + Language::Go => ts_lang_go()?, + Language::Swift => ts_lang_swift()?, + Language::Php => ts_lang_php()?, + Language::Objc => ts_lang_objc()?, + Language::Rust => ts_lang_rust()?, + Language::JavaScript => ts_lang_javascript()?, + Language::TypeScript => ts_lang_typescript()?, }; parser.set_language(&ts_lang).context("set language")?; - // Allow deprecated method for now - the new API is more complex - // and the current method works fine for our needs - #[allow(deprecated)] - parser.set_timeout_micros(5_000_000); // 5 second timeout - - parser.parse(content, None).context("parse") + let deadline = Instant::now() + Duration::from_secs(5); + let mut progress = |_state: &tree_sitter::ParseState| { + if Instant::now() >= deadline { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + }; + let options = ParseOptions::new().progress_callback(&mut progress); + let bytes = content.as_bytes(); + parser + .parse_with_options( + &mut |offset, _| bytes.get(offset..).unwrap_or_default(), + None, + Some(options), + ) + .context("parse") } /// Scans the AST to find "library anchors" which are top-level import-like @@ -628,11 +644,8 @@ fn import_like_nodes<'a>(lang: Language, root: Node<'a>, content: &[u8]) -> Vec< if is_import { nodes.push(node); } - for i in 0..node.child_count() { - if let Some(child) = node.child(i) { - stack.push(child); - } - } + let mut cursor = node.walk(); + stack.extend(node.children(&mut cursor)); } nodes } @@ -703,11 +716,8 @@ fn code_symbol_nodes<'a>(lang: Language, root: Node<'a>) -> Vec> { if interesting { nodes.push(node); } - for i in 0..node.child_count() { - if let Some(child) = node.child(i) { - stack.push(child); - } - } + let mut cursor = node.walk(); + stack.extend(node.children(&mut cursor)); } nodes } @@ -816,9 +826,9 @@ fn is_more_specific(specific: &str, generic: &str) -> bool { #[cfg(test)] mod tests { - use super::{ - AlgorithmHit, dedupe_more_specific_hits, find_library_anchors, has_anchor_hint, parse, - }; + #[cfg(feature = "lang-python")] + use super::find_library_anchors; + use super::{AlgorithmHit, dedupe_more_specific_hits, has_anchor_hint, parse}; use crate::patterns::{Language, PatternSet}; use ahash::AHashMap as HashMap; use serde_json::Value; @@ -1005,6 +1015,19 @@ apis = ["testlib\\.crypto"] )); } + #[test] + fn c_headers_use_an_enabled_grammar() { + let detected = super::language_from_path(std::path::Path::new("example.h")); + if cfg!(feature = "lang-cpp") { + assert_eq!(detected, Some(Language::Cpp)); + } else if cfg!(feature = "lang-c") { + assert_eq!(detected, Some(Language::C)); + } else { + assert_eq!(detected, None); + } + } + + #[cfg(feature = "lang-python")] #[test] fn find_library_anchors_uses_import_nodes() { let patterns = patterns_from_toml( @@ -1025,6 +1048,7 @@ include = ["testlib"] assert_eq!(hits[0].column, 1); } + #[cfg(feature = "lang-python")] #[test] fn find_library_anchors_falls_back_to_api_regex() { let patterns = patterns_from_toml( @@ -1044,4 +1068,15 @@ apis = ["CryptoLib"] assert_eq!(hits[0].line, 1); assert_eq!(hits[0].column, 1); } + + #[cfg(not(feature = "lang-python"))] + #[test] + fn parsing_a_disabled_language_returns_an_error() { + let error = parse(Language::Python, "print('hello')").unwrap_err(); + assert!( + error + .to_string() + .contains("lang-python feature is disabled") + ); + } } diff --git a/tests/integration.rs b/tests/integration.rs index d1fc8e1..0e2f901 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -3,7 +3,9 @@ use std::{ path::{Path, PathBuf}, process::Command, }; -use tempfile::{NamedTempFile, TempDir}; +use tempfile::NamedTempFile; +#[cfg(feature = "lang-rust")] +use tempfile::TempDir; fn normalize_path_in_value(mut v: serde_json::Value) -> serde_json::Value { if let Some(obj) = v.as_object_mut() @@ -49,6 +51,9 @@ fn should_skip_family(family: &str) -> bool { "swift" => !cfg!(feature = "lang-swift"), "php" => !cfg!(feature = "lang-php"), "objc" | "objective-c" => !cfg!(feature = "lang-objc"), + "openssl_c" => !cfg!(feature = "lang-c") || !cfg!(feature = "lang-cpp"), + "cpp" => !cfg!(feature = "lang-cpp"), + "rust" => !cfg!(feature = "lang-rust"), "javascript" | "js" => !cfg!(feature = "lang-javascript"), "typescript" | "ts" => !cfg!(feature = "lang-typescript"), _ => false, @@ -131,6 +136,7 @@ fn fixtures_match_ground_truth() { } } +#[cfg(feature = "lang-go")] #[test] fn exclude_works() { let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); @@ -170,6 +176,7 @@ fn exclude_works() { assert_eq!(actual, expected); } +#[cfg(feature = "lang-go")] #[test] fn multiple_roots_work() { let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); @@ -208,6 +215,7 @@ fn multiple_roots_work() { assert_eq!(actual, expected1); } +#[cfg(feature = "lang-rust")] #[test] fn max_file_mb_skips_large_files() { let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); @@ -279,16 +287,142 @@ symbol_patterns = ["test_algo"] #[test] fn javascript_extension_detection() { - // This test verifies JavaScript support is available - // by checking that the feature flag is enabled - #[cfg(not(feature = "lang-javascript"))] - compile_error!("lang-javascript feature must be enabled"); + let detected = cipherscope::scan::language_from_path(Path::new("example.js")); + if cfg!(feature = "lang-javascript") { + assert_eq!(detected, Some(cipherscope::patterns::Language::JavaScript)); + } else { + assert_eq!(detected, None); + } } #[test] fn typescript_extension_detection() { - // This test verifies TypeScript support is available - // by checking that the feature flag is enabled - #[cfg(not(feature = "lang-typescript"))] - compile_error!("lang-typescript feature must be enabled"); + let detected = cipherscope::scan::language_from_path(Path::new("example.ts")); + if cfg!(feature = "lang-typescript") { + assert_eq!(detected, Some(cipherscope::patterns::Language::TypeScript)); + } else { + assert_eq!(detected, None); + } +} + +#[test] +fn zero_threads_is_rejected() { + let output = Command::new(env!("CARGO_BIN_EXE_cipherscope")) + .args(["--threads", "0"]) + .output() + .unwrap(); + + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("at least 1")); +} + +#[cfg(feature = "lang-rust")] +#[test] +fn source_shaped_output_is_not_scanned() { + let tmp_dir = TempDir::new().unwrap(); + let root = tmp_dir.path(); + let patterns_path = root.join("patterns.toml"); + fs::write( + &patterns_path, + r#" +[[library]] +name = "TestLib" +languages = ["Rust"] +[library.patterns] +include = ["TestLib"] + +[[library.algorithms]] +name = "TEST-ALG" +primitive = "test" +symbol_patterns = ["test_algo"] +"#, + ) + .unwrap(); + fs::write( + root.join("input.rs"), + "use TestLib;\nfn main() { test_algo(); }\n", + ) + .unwrap(); + let output_path = root.join("findings.rs"); + + let output = Command::new(env!("CARGO_BIN_EXE_cipherscope")) + .args([ + "--roots", + root.to_str().unwrap(), + "--patterns", + patterns_path.to_str().unwrap(), + "--output", + output_path.to_str().unwrap(), + ]) + .output() + .unwrap(); + + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(read_jsonl(&output_path).len(), 2); +} + +#[cfg(feature = "lang-rust")] +#[test] +fn gitignore_can_be_disabled() { + let tmp_dir = TempDir::new().unwrap(); + let root = tmp_dir.path(); + fs::create_dir(root.join(".git")).unwrap(); + fs::write(root.join(".gitignore"), "ignored.rs\n").unwrap(); + fs::write( + root.join("ignored.rs"), + "use TestLib;\nfn main() { test_algo(); }\n", + ) + .unwrap(); + let patterns_path = root.join("patterns.toml"); + fs::write( + &patterns_path, + r#" +[[library]] +name = "TestLib" +languages = ["Rust"] +[library.patterns] +include = ["TestLib"] + +[[library.algorithms]] +name = "TEST-ALG" +primitive = "test" +symbol_patterns = ["test_algo"] +"#, + ) + .unwrap(); + + let default_output = NamedTempFile::new().unwrap(); + let status = Command::new(env!("CARGO_BIN_EXE_cipherscope")) + .args([ + "--roots", + root.to_str().unwrap(), + "--patterns", + patterns_path.to_str().unwrap(), + "--output", + default_output.path().to_str().unwrap(), + ]) + .status() + .unwrap(); + assert!(status.success()); + assert!(read_jsonl(default_output.path()).is_empty()); + + let unignored_output = NamedTempFile::new().unwrap(); + let status = Command::new(env!("CARGO_BIN_EXE_cipherscope")) + .args([ + "--roots", + root.to_str().unwrap(), + "--patterns", + patterns_path.to_str().unwrap(), + "--output", + unignored_output.path().to_str().unwrap(), + "--gitignore=false", + ]) + .status() + .unwrap(); + assert!(status.success()); + assert_eq!(read_jsonl(unignored_output.path()).len(), 2); }