From 0e962a30d9f735acc1b04e238f5d870deb9fbc60 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Sun, 1 Nov 2026 03:23:36 +0000 Subject: [PATCH 1/5] feat(builtins): add markkit and fetchkit builtins with the upstream CLI Opt-in `markkit` and `fetchkit` features (off by default) register both tools as builtins. Each calls the upstream `run(args, host)` library entry point, so flags, help, errors, output and exit codes match the standalone binaries. Files come from the VFS; every HTTP hop goes through the sandbox HttpClient (allowlist, SSRF precheck, credential injection, signing). Host-only flags (`--ocr`, `--render-rakers`, proxies, bot-auth keys, remote conversion) and `mcp` parse but fail closed. markkit limits can only be lowered. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Wy7MByPXsW4jHxfLeTfGEc --- .github/workflows/ci.yml | 8 + AGENTS.md | 1 + Cargo.lock | 902 +++++++++++++++++- crates/bashkit/Cargo.toml | 24 + crates/bashkit/src/builtins/fetchkit.rs | 203 ++++ crates/bashkit/src/builtins/markkit.rs | 352 +++++++ crates/bashkit/src/builtins/mod.rs | 8 + crates/bashkit/src/interpreter/mod.rs | 4 + .../integration/fetchkit_builtin_tests.rs | 244 +++++ crates/bashkit/tests/integration/main.rs | 2 + .../integration/markkit_builtin_tests.rs | 253 +++++ docs/markkit-fetchkit.md | 72 ++ knowledge/log.md | 2 + knowledge/operations/limitations.md | 1 + knowledge/runtimes/index.md | 1 + .../runtimes/markkit-fetchkit-builtins.md | 117 +++ knowledge/security/threat-model.md | 2 + site/src/pages/docs/_meta.ts | 10 + 18 files changed, 2198 insertions(+), 8 deletions(-) create mode 100644 crates/bashkit/src/builtins/fetchkit.rs create mode 100644 crates/bashkit/src/builtins/markkit.rs create mode 100644 crates/bashkit/tests/integration/fetchkit_builtin_tests.rs create mode 100644 crates/bashkit/tests/integration/markkit_builtin_tests.rs create mode 100644 docs/markkit-fetchkit.md create mode 100644 knowledge/runtimes/markkit-fetchkit-builtins.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a54548d6e..4a67d953e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -378,6 +378,14 @@ jobs: cargo test -p bashkit --features terminal --test integration -- terminal_tests l_term_ cargo test -p bashkit --features terminal --doc terminal + # markkit/fetchkit builtins are `cfg(markkit|fetchkit)` (off by default), + # so the main "Run tests" step compiles them out entirely. + - name: Run markkit/fetchkit builtin tests + run: | + cargo test -p bashkit --features markkit,fetchkit --lib -- builtins::markkit:: builtins::fetchkit:: + cargo test -p bashkit --features markkit,fetchkit --test integration -- markkit_builtin_tests fetchkit_builtin_tests + cargo test -p bashkit --features markkit --test integration -- markkit_builtin_tests + - name: Run fail-point tests (single-threaded) run: cargo test --features failpoints --test security_failpoint_tests -- --test-threads=1 diff --git a/AGENTS.md b/AGENTS.md index ee79a7a63..cc5eca532 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,6 +52,7 @@ Fix root cause. Unsure: read more code; if stuck, ask w/ short options. Unrecogn | security/request-signing | Transparent Ed25519 request signing (bot-auth) per RFC 9421 | | integrations/interactive-shell | Interactive REPL mode with rustyline line editing | | integrations/in-process-terminal | In-memory PTY-like `Terminal` (feature `terminal`), screen text, `vi` builtin | +| runtimes/markkit-fetchkit-builtins | `markkit`/`fetchkit` builtins: same CLI via upstream `run(args, host)`, VFS + allowlist | | runtimes/sqlite-builtin | Embedded SQLite via Turso (MemoryIO + VfsIO backends, dot-commands) | | runtimes/coreutils-args-port | Codegen port of uutils clap definitions + uucore modules | | security/credential-injection | Per-host HTTP credential injection without exposing secrets | diff --git a/Cargo.lock b/Cargo.lock index ebc028bb1..d8235f849 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -158,12 +158,56 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "antithesis_sdk" version = "0.3.0" @@ -256,6 +300,16 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "atoi_simd" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cdb3708a128e559a30fb830e8a77a5022ee6902806925c216658652b452a44" +dependencies = [ + "debug_unsafe", + "rustversion", +] + [[package]] name = "atomic-polyfill" version = "1.0.3" @@ -377,6 +431,8 @@ dependencies = [ "ed25519-dalek", "fail", "fancy-regex 0.19.2", + "fetchkit", + "fetchkit-cli", "flate2", "foldhash", "futures-core", @@ -390,7 +446,9 @@ dependencies = [ "insta", "jaq-core", "jaq-std", - "md-5", + "markkit", + "markkit-cli", + "md-5 0.11.0", "monty", "monty-types", "num-bigint 0.4.8", @@ -687,6 +745,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array 0.14.7", +] + [[package]] name = "block-padding" version = "0.4.2" @@ -806,6 +873,23 @@ dependencies = [ "libbz2-rs-sys", ] +[[package]] +name = "calamine" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9ca341b94bc4b29e08970b1a03914fad2dfe177e2ecb67fcd6de9046004aafb" +dependencies = [ + "atoi_simd", + "byteorder", + "codepage", + "encoding_rs", + "fast-float2", + "log", + "quick-xml 0.42.0", + "serde", + "zip", +] + [[package]] name = "cast" version = "0.3.0" @@ -821,6 +905,15 @@ dependencies = [ "rustversion", ] +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "cbc" version = "0.2.1" @@ -842,6 +935,17 @@ dependencies = [ "shlex", ] +[[package]] +name = "cfb" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f13298d9178622b7916ec5d01d470425c04273c8ed07e87b1c38b34c4f4612b" +dependencies = [ + "fnv", + "uuid", + "web-time", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -1017,12 +1121,27 @@ dependencies = [ "thiserror", ] +[[package]] +name = "codepage" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdff162541cd8b79de82e2edcc7eff3a8c2a6dc3d75152636028f96d93de3b26" +dependencies = [ + "encoding_rs", +] + [[package]] name = "collection_literals" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "colored" version = "3.1.1" @@ -1463,6 +1582,29 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "cssparser" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11119743ad110e8c1bdccd930d7f5c30c99e5fc76a7b63ec9807e84eef0c5f59" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf 0.14.0", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d045de693cb712d0b22c6a64be5b953f67b3ce00ab5ad3dd5d8b441886ab8e1a" +dependencies = [ + "quote", + "syn 3.0.9", +] + [[package]] name = "ctor" version = "1.0.13" @@ -1531,6 +1673,12 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" +[[package]] +name = "debug_unsafe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" + [[package]] name = "defmt" version = "1.1.1" @@ -1584,6 +1732,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" + [[package]] name = "derive-where" version = "1.6.1" @@ -1595,6 +1749,27 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", +] + [[package]] name = "des" version = "0.9.0" @@ -1649,6 +1824,21 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd8e701084c37e7ef62d3f9e453b618130cbc0ef3573847785952a3ac3f746bf" +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + [[package]] name = "dunce" version = "1.0.5" @@ -1661,6 +1851,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ecb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "ecdsa" version = "0.17.0" @@ -1702,6 +1901,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ego-tree" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd50d24b1b75aa817bb0339cb9f2e0e664bdfab1e160a69a0fda9145b77fab2" + [[package]] name = "either" version = "1.18.0" @@ -1748,6 +1953,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "encoding_rs" +version = "0.8.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89253175e704ab869b04f9db224d38630e0211a53841a79181deae0889e169f5" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + [[package]] name = "endian-type" version = "0.2.0" @@ -1766,6 +1985,29 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1827,6 +2069,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "fast-float2" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e8948ce679d00a02a94739ea185595dca7118ed04feb991127e443bd3d761f" + [[package]] name = "fastbloom" version = "0.14.1" @@ -1844,6 +2092,46 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "fetchkit" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eef9fadf79a855de81a091f76b8449f8f6d312e7b6622e14a961bad84b350f22" +dependencies = [ + "async-trait", + "bytes", + "futures", + "libc", + "markkit", + "pdf-inspector", + "percent-encoding", + "quick-xml 0.41.0", + "reqwest", + "schemars", + "serde", + "serde_json", + "thiserror", + "tokio", + "tower", + "tracing", + "url", + "zip", +] + +[[package]] +name = "fetchkit-cli" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "903241be14111602101e5615c9cc91c04af4369e7b4c770d1d4dfd9ef087e9d1" +dependencies = [ + "async-trait", + "clap", + "fetchkit", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "ff" version = "0.14.0" @@ -2206,6 +2494,52 @@ dependencies = [ "serde_core", ] +[[package]] +name = "hashify" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd1246c0e5493286aeb2dde35b1f4eb9c4ce00e628641210a5e553fc001a1f26" +dependencies = [ + "indexmap", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "hayro-ccitt" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2866f0d976e76497480b853d5e258dab6e4b89ecfe991124e08f15dfa8fd03ad" + +[[package]] +name = "hayro-jbig2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "000e025850f4592ddcaee85640db70704151f2bafd43b6f431aa5b6a2c6a35a9" +dependencies = [ + "hayro-ccitt", +] + +[[package]] +name = "hayro-jpeg2000" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "122c5f834c37a2260b9c851bb8be685cf539b2a8b94b704a931ef7d2525d5835" + +[[package]] +name = "hayro-syntax" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdc818d8d265dcb42081d488f7599df4f73fb79e9d347ecedcd3c23809d2026" +dependencies = [ + "hayro-jbig2", + "hayro-jpeg2000", + "rustc-hash", + "smallvec", + "zune-jpeg", +] + [[package]] name = "heapless" version = "0.7.17" @@ -2277,6 +2611,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "html5ever" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456a1a377e608e555d22ddab27ac0114bc7a7b4199078108e34c2aeae6c9b130" +dependencies = [ + "log", + "markup5ever", + "memchr", +] + [[package]] name = "http" version = "1.5.0" @@ -2588,6 +2933,25 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfdf4f5d937a025381f5ab13624b1c5f51414bfe5c9885663226eae8d6d39560" +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indexmap" version = "2.14.2" @@ -2606,6 +2970,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ + "block-padding 0.3.3", "generic-array 0.14.7", ] @@ -2615,7 +2980,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" dependencies = [ - "block-padding", + "block-padding 0.4.2", "hybrid-array", ] @@ -2673,6 +3038,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.13.0" @@ -3026,6 +3397,37 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lopdf" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25aab26d99567469098e64a02f42679f8965c6401263eefa31d8f2dcc37a221c" +dependencies = [ + "aes 0.8.4", + "bitflags 2.13.2", + "cbc 0.1.2", + "chrono", + "ecb", + "encoding_rs", + "flate2", + "getrandom 0.4.3", + "indexmap", + "itoa", + "jiff", + "log", + "md-5 0.10.6", + "nom", + "rand 0.10.3", + "rangemap", + "rayon", + "sha2 0.10.9", + "stringprep", + "thiserror", + "time", + "ttf-parser", + "weezl", +] + [[package]] name = "lzma-rs" version = "0.3.0" @@ -3042,6 +3444,15 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" +[[package]] +name = "mail-parser" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec00bda90c6e645a54506c630c2820cd6b1890cfd2b0a169b50f74b2b8c7c86" +dependencies = [ + "hashify", +] + [[package]] name = "manyhow" version = "0.11.4" @@ -3065,6 +3476,57 @@ dependencies = [ "quote", ] +[[package]] +name = "markkit" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688329d19a51cc92a77ae70536ce55e9f5d45d86e542616066134cce8fd03b54" +dependencies = [ + "base64 0.23.1", + "calamine", + "cfb", + "ego-tree", + "futures-core", + "hayro-syntax", + "html5ever", + "mail-parser", + "miniz_oxide", + "pulldown-cmark", + "quick-xml 0.42.0", + "scraper", + "serde", + "serde_json", + "thiserror", + "tokio", + "tower", + "url", + "zip", +] + +[[package]] +name = "markkit-cli" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e88e13ea9f8bb4bb436563d0ec33342f438306810b5053df1336c3493d3479" +dependencies = [ + "anyhow", + "clap", + "markkit", + "serde_json", + "tokio", +] + +[[package]] +name = "markup5ever" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab3dc68ac4a0f5719e560136778c1ee716e296030d75dbd4484e37e39e3a842" +dependencies = [ + "log", + "tendril", + "web_atoms", +] + [[package]] name = "matchers" version = "0.2.0" @@ -3074,6 +3536,16 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + [[package]] name = "md-5" version = "0.11.0" @@ -3269,6 +3741,12 @@ dependencies = [ "web-time", ] +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "napi" version = "3.15.0" @@ -3328,6 +3806,12 @@ dependencies = [ "libloading 0.9.0", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "nibble_vec" version = "0.1.0" @@ -3353,7 +3837,16 @@ dependencies = [ name = "nohash-hasher" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] [[package]] name = "nonmax" @@ -3393,6 +3886,12 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-integer" version = "0.1.47" @@ -3429,6 +3928,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "oorandom" version = "11.1.5" @@ -3816,6 +4321,24 @@ dependencies = [ "hmac", ] +[[package]] +name = "pdf-inspector" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd2f755e49ad38eafbc82ba2bec1c59d57b5bf829b13a8f35e4263bc8913df3a" +dependencies = [ + "env_logger", + "include_dir", + "log", + "lopdf", + "once_cell", + "rayon", + "regex", + "thiserror", + "ttf-parser", + "unicode-normalization", +] + [[package]] name = "pem-rfc7468" version = "1.0.0" @@ -3865,11 +4388,22 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ - "phf_macros", + "phf_macros 0.13.1", "phf_shared 0.13.1", "serde", ] +[[package]] +name = "phf" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6" +dependencies = [ + "phf_macros 0.14.0", + "phf_shared 0.14.0", + "serde", +] + [[package]] name = "phf_codegen" version = "0.11.3" @@ -3880,6 +4414,16 @@ dependencies = [ "phf_shared 0.11.3", ] +[[package]] +name = "phf_codegen" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b585a510fb76fdebead6897982ef2a03a21d8e6cbcca904999742a4afc6ffe" +dependencies = [ + "phf_generator 0.14.0", + "phf_shared 0.14.0", +] + [[package]] name = "phf_generator" version = "0.11.3" @@ -3900,6 +4444,16 @@ dependencies = [ "phf_shared 0.13.1", ] +[[package]] +name = "phf_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100" +dependencies = [ + "fastrand", + "phf_shared 0.14.0", +] + [[package]] name = "phf_macros" version = "0.13.1" @@ -3913,6 +4467,19 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "phf_macros" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085" +dependencies = [ + "phf_generator 0.14.0", + "phf_shared 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "phf_shared" version = "0.11.3" @@ -3940,6 +4507,15 @@ dependencies = [ "siphasher", ] +[[package]] +name = "phf_shared" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -3964,7 +4540,7 @@ checksum = "63d440a804ec8d6fafbb6b84471e013286658d373248927692ab3366686220ca" dependencies = [ "aes 0.9.3", "aes-gcm 0.11.1", - "cbc", + "cbc 0.2.1", "der", "pbkdf2", "rand_core 0.10.1", @@ -4107,6 +4683,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -4116,6 +4698,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "pretty_assertions" version = "1.4.1" @@ -4212,6 +4800,17 @@ dependencies = [ "unarray", ] +[[package]] +name = "pulldown-cmark" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db2be46835f013dd3f4fa7f5f7402c537787b54ebb6fe2b94bbdd7e2669bb7b" +dependencies = [ + "bitflags 2.13.2", + "memchr", + "unicase", +] + [[package]] name = "pulley-interpreter" version = "48.0.8" @@ -4314,6 +4913,25 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" +dependencies = [ + "encoding_rs", + "memchr", +] + [[package]] name = "quote" version = "1.0.47" @@ -4467,6 +5085,12 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rangemap" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a611d15b50743feb4c76b7d03edcb0e64f399c26961e4efe6975bc398be6aa3d" + [[package]] name = "rapidhash" version = "4.5.1" @@ -4505,6 +5129,26 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "regalloc2" version = "0.15.2" @@ -4740,10 +5384,10 @@ dependencies = [ "aes 0.9.3", "aws-lc-rs", "bitflags 2.13.2", - "block-padding", + "block-padding 0.4.2", "byteorder", "bytes", - "cbc", + "cbc 0.2.1", "cipher 0.5.2", "crypto-bigint", "ctr 0.10.1", @@ -5018,6 +5662,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "schemars" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" +dependencies = [ + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1c3a92094fa7d61aa124645844facb6b554dfc797136d0f5fd1f890e2bffc69" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 3.0.9", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -5030,6 +5699,20 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scraper" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6844b5f49a5579f12ba4f89d9f7eec6d71889e4aa0bc7181821040ce66f08fff" +dependencies = [ + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "precomputed-hash", + "selectors", +] + [[package]] name = "scrypt" version = "0.12.0" @@ -5079,6 +5762,23 @@ dependencies = [ "libc", ] +[[package]] +name = "selectors" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8322aebea94a6cc288ff884e8a49bdbceef60206c400c6c46d22e62fc4fd91a6" +dependencies = [ + "bitflags 2.13.2", + "cssparser", + "derive_more", + "log", + "new_debug_unreachable", + "precomputed-hash", + "rustc-hash", + "servo_arc", + "smallvec", +] + [[package]] name = "self_cell" version = "1.3.0" @@ -5151,6 +5851,17 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "serde_derive_internals" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "serde_json" version = "1.0.154" @@ -5221,6 +5932,15 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "servo_arc" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4eceb519ec34d8290aaf6e46f435102f2bfa63f22b05a144fb16ab266d3b198" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "sha1" version = "0.11.0" @@ -5533,6 +6253,41 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "string_cache" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffa8a5dbe8b3f0bbe29d4c3225daafaeead63afdc1b65fc4c01a1384166038e6" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared 0.14.0", + "precomputed-hash", +] + +[[package]] +name = "string_cache_codegen" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928dcdf75e47626b3617a976ec205d9f057584c371c1f23b782129268d0e6edc" +dependencies = [ + "phf_generator 0.14.0", + "phf_shared 0.14.0", + "proc-macro2", + "quote", +] + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strum" version = "0.26.3" @@ -5672,6 +6427,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tendril" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fed54709c5b3a53d09bb1c113ea4f5ceafd1e772ddcb0030a82e1d56c087b08" +dependencies = [ + "new_debug_unreachable", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -5746,6 +6510,36 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.4" @@ -6017,6 +6811,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + [[package]] name = "turso_core" version = "0.8.3" @@ -6141,6 +6941,12 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" +[[package]] +name = "typed-path" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce898823d8cd0f028662d2f349694b1af0a5cc9be83f1d213a00d3147445e1e" + [[package]] name = "typenum" version = "1.20.1" @@ -6162,6 +6968,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "unicase" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-general-category" version = "1.1.0" @@ -6195,6 +7013,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993ad287eb20e8ba755312801cfe1413e0f25d604075863acaddca10b04c1833" + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -6231,7 +7055,7 @@ checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e" dependencies = [ "getopts", "log", - "phf_codegen", + "phf_codegen 0.11.3", "rand 0.8.8", ] @@ -6737,6 +7561,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web_atoms" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7572660c8890448ba236b7376f27e389c6a7e1c70195622faced601f855c0ada" +dependencies = [ + "phf 0.14.0", + "phf_codegen 0.14.0", + "string_cache", + "string_cache_codegen", +] + [[package]] name = "webpki-root-certs" version = "1.0.10" @@ -6746,6 +7582,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "winapi" version = "0.3.9" @@ -7183,6 +8025,23 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "zip" +version = "9.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f5b58784007ff6e5f280ec82c8596686a3b9b5dd5836c63288d4479f84728b9" +dependencies = [ + "crc32fast", + "flate2", + "getrandom 0.4.3", + "indexmap", + "memchr", + "nix", + "typed-path", + "windows-sys 0.61.2", + "zopfli", +] + [[package]] name = "zlib-rs" version = "0.6.7" @@ -7194,3 +8053,30 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/crates/bashkit/Cargo.toml b/crates/bashkit/Cargo.toml index c61ce4ea3..2cf04b054 100644 --- a/crates/bashkit/Cargo.toml +++ b/crates/bashkit/Cargo.toml @@ -284,6 +284,23 @@ typescript = ["dep:zapcode-core", "dep:oxc_parser", "dep:oxc_ast", "dep:oxc_allo # back to the async VFS via `block_in_place`. # Usage: cargo build --features sqlite sqlite = ["dep:turso_core", "dep:branches", "tokio/rt-multi-thread"] +# `markkit` builtin: the markkit CLI (document -> Markdown), identical +# interface, reading the VFS. URL sources go through `http_client` when that +# feature and a network allowlist are configured. Off by default. +# +# Important decision: depend on `markkit-cli`'s library (`markkit_cli::run`) +# rather than re-declaring its flags, so the interface cannot drift. markkit is +# built without `http_client` (no second HTTP/TLS stack) and without OCR (host +# Tesseract / model files). See knowledge/runtimes/markkit-fetchkit-builtins.md. +# Usage: cargo build --features markkit +markkit = ["dep:markkit", "dep:markkit-cli"] +# `fetchkit` builtin: the fetchkit CLI (URL -> Markdown with frontmatter), +# identical interface. Every request goes through the `http_client` sandbox +# client (allowlist, credentials, signing, host transport). Off by default. +# Same library-reuse decision as `markkit`; see +# knowledge/runtimes/markkit-fetchkit-builtins.md. +# Usage: cargo build --features fetchkit +fetchkit = ["http_client", "dep:fetchkit", "dep:fetchkit-cli", "dep:bytes"] # Enable RealFs backend for accessing host filesystem directories # WARNING: This intentionally breaks the sandbox boundary. # Usage: cargo build --features realfs @@ -421,6 +438,13 @@ required-features = ["sqlite"] # Additional tokio features needed only on native (not WASM) [target.'cfg(not(target_arch = "wasm32"))'.dependencies] tokio = { version = "1", features = ["rt-multi-thread", "fs"] } +# markkit builtin (native only: markkit-cli needs tokio's multi-thread rt). +markkit-cli = { version = "0.3", default-features = false, features = ["all_formats"], optional = true } +markkit = { version = "0.3", default-features = false, features = ["all_formats"], optional = true } +# fetchkit builtin (native only). No `reqwest-tls`: all traffic goes through +# bashkit's HttpClient, so fetchkit adds no TLS stack. +fetchkit-cli = { version = "0.7", default-features = false, features = ["docs-pdf", "docs-markkit"], optional = true } +fetchkit = { version = "0.7", default-features = false, features = ["docs-pdf", "docs-markkit"], optional = true } # std::time::{Instant,SystemTime} panic on wasm32-unknown-unknown; see # time_compat/mod.rs. Only pulled in for wasm32 — everywhere else time_compat diff --git a/crates/bashkit/src/builtins/fetchkit.rs b/crates/bashkit/src/builtins/fetchkit.rs new file mode 100644 index 000000000..da7e1af12 --- /dev/null +++ b/crates/bashkit/src/builtins/fetchkit.rs @@ -0,0 +1,203 @@ +//! `fetchkit` builtin: the fetchkit CLI, unchanged, on the network allowlist. +//! +//! # Decisions +//! +//! - **Same CLI, same code.** Arguments go to `fetchkit_cli::run`, the function +//! the standalone `fetchkit` binary calls, so flags, help, version, errors, +//! output and exit codes are identical. Nothing here parses arguments. See +//! `knowledge/runtimes/markkit-fetchkit-builtins.md`. +//! - **One egress path.** Every request fetchkit makes (the page, redirect +//! hops, specialized-fetcher API calls, agent-discovery probes) goes through +//! [`SandboxTransport`], a fetchkit `HttpTransport` over this `Bash`'s +//! [`HttpClient`]: allowlist, SSRF precheck, credential injection, bot-auth +//! signing, hooks and the host `HttpTransport` all apply. fetchkit is built +//! without `reqwest-tls`, so it adds no TLS stack. +//! - **fetchkit DNS policy is off** (`block_private_ips(false)`): bashkit's +//! client already resolves and checks before dispatch, and fetchkit's +//! resolver is blocking `to_socket_addrs`. +//! - **Refused in the sandbox (fail closed):** `--allow-env-proxy` (host proxy +//! env), `--render-rakers` (embedded JS engine), `--bot-auth-key`/`-agent` +//! (bashkit signs in `HttpClient`; a key in argv would double-sign and land in +//! history), `--allow-remote-conversion` (uploads documents to a third +//! party), and the `mcp` subcommand. All still parse and show in `--help`. +//! - **Stateless.** No globals; every run builds its own fetchkit `Tool`. + +use std::io; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use fetchkit::{ + HttpTransport, ToolBuilder, TransportError, TransportMethod, TransportRequest, + TransportResponse, +}; +use fetchkit_cli::{CliHost, HostFeature}; + +use super::{Builtin, Context}; +use crate::error::{Error, Result}; +use crate::interpreter::ExecResult; +use crate::network::{HttpClient, Method}; + +/// The `fetchkit` command (feature `fetchkit`). +pub struct Fetchkit; + +#[async_trait] +impl Builtin for Fetchkit { + async fn execute(&self, ctx: Context<'_>) -> Result { + let host = Host { + stdout: Mutex::new(String::new()), + stderr: Mutex::new(String::new()), + env: ctx.env.clone(), + http: ctx.http_client.cloned(), + }; + let args = std::iter::once("fetchkit".to_string()).chain(ctx.args.iter().cloned()); + let exit_code = fetchkit_cli::run(args, &host).await; + Ok(ExecResult { + stdout: take(&host.stdout).into(), + stderr: take(&host.stderr).into(), + exit_code, + ..Default::default() + }) + } + + fn llm_hint(&self) -> Option<&'static str> { + Some( + "fetchkit: Fetch a URL as LLM-ready Markdown with YAML frontmatter \ + (`fetchkit fetch URL`, `-o json`, `--crawl`, `--content-focus agent`). Same CLI as \ + the standalone fetchkit. Needs network allowlist; no mcp, proxies, rakers or bot-auth flags.", + ) + } +} + +fn take(buf: &Mutex) -> String { + std::mem::take(&mut *buf.lock().unwrap_or_else(|p| p.into_inner())) +} + +/// [`CliHost`] over one builtin invocation. +struct Host { + stdout: Mutex, + stderr: Mutex, + env: std::collections::HashMap, + http: Option>, +} + +#[async_trait] +impl CliHost for Host { + fn write_stdout(&self, text: &str) -> io::Result<()> { + self.stdout + .lock() + .unwrap_or_else(|p| p.into_inner()) + .push_str(text); + Ok(()) + } + + fn write_stderr(&self, text: &str) { + self.stderr + .lock() + .unwrap_or_else(|p| p.into_inner()) + .push_str(text); + } + + /// Shell environment, never the host process environment. + fn env_var(&self, name: &str) -> Option { + self.env.get(name).cloned() + } + + fn configure_tool(&self, builder: ToolBuilder) -> ToolBuilder { + builder + .transport(Arc::new(SandboxTransport { + http: self.http.clone(), + })) + .block_private_ips(false) + .use_env_proxy(false) + } + + fn refuse(&self, feature: HostFeature) -> Option { + let _ = feature; + Some("is not available in bashkit".to_string()) + } +} + +/// fetchkit `HttpTransport` over the sandbox [`HttpClient`]. One call per hop: +/// fetchkit follows redirects itself, so every hop is re-checked here. +struct SandboxTransport { + http: Option>, +} + +#[async_trait] +impl HttpTransport for SandboxTransport { + async fn execute( + &self, + req: TransportRequest, + ) -> std::result::Result { + let Some(http) = &self.http else { + return Err(TransportError::Request( + "network access not configured (see BashBuilder::network)".to_string(), + )); + }; + let method = match req.method { + TransportMethod::Get => Method::Get, + TransportMethod::Head => Method::Head, + }; + // bashkit clamps the deadline to its own bounds; round sub-second up. + let timeout_secs = req.timeout.map(|t| t.as_secs().max(1)); + let response = http + .request_with_timeout(method, req.url.as_str(), None, &req.headers, timeout_secs) + .await + .map_err(to_transport_error)?; + let body = bytes::Bytes::from(response.body); + Ok(TransportResponse { + status: response.status, + url: req.url, + headers: response.headers, + body: Box::pin(futures_util::stream::once(async move { Ok(body) })), + }) + } +} + +/// Keep bashkit's own one-line messages (no Debug shapes, TM-INF-022). +fn to_transport_error(err: Error) -> TransportError { + match err { + Error::Network(msg) => TransportError::Request(msg), + other => TransportError::Other(other.to_string()), + } +} + +#[cfg(test)] +mod tests { + /// fetchkit internals that must never reach stderr (TM-INF-022). + const FETCHKIT_BANNED: &[&str] = &[ + "RequestError(", + "TransportError", + "FetchError", + "reqwest::", + "Stack backtrace", + "src/", + ]; + + macro_rules! fetchkit_no_leak { + ($name:ident, $script:expr) => { + #[tokio::test] + async fn $name() { + let r = crate::builtins::debug_leak_check::run($script).await; + assert_ne!(r.exit_code, 0, "{}", stringify!($name)); + crate::builtins::debug_leak_check::assert_no_leak( + &r, + stringify!($name), + FETCHKIT_BANNED, + ); + } + }; + } + + fetchkit_no_leak!(no_leak_no_network, "fetchkit fetch https://example.com/"); + fetchkit_no_leak!(no_leak_bad_url, "fetchkit fetch 'http://['"); + fetchkit_no_leak!(no_leak_bad_scheme, "fetchkit fetch file:///etc/passwd"); + fetchkit_no_leak!( + no_leak_refused_flag, + "fetchkit fetch https://x.test --render-rakers" + ); + fetchkit_no_leak!( + no_leak_bad_converter, + "fetchkit fetch https://x.test --converter nope" + ); +} diff --git a/crates/bashkit/src/builtins/markkit.rs b/crates/bashkit/src/builtins/markkit.rs new file mode 100644 index 000000000..3081f0cb8 --- /dev/null +++ b/crates/bashkit/src/builtins/markkit.rs @@ -0,0 +1,352 @@ +//! `markkit` builtin: the markkit CLI, unchanged, on the VFS and network allowlist. +//! +//! # Decisions +//! +//! - **Same CLI, same code.** Arguments go to `markkit_cli::run`, the function +//! the standalone `markkit` binary calls. Flags, help, version, error text +//! and exit codes are therefore identical; nothing here parses arguments. +//! See `knowledge/runtimes/markkit-fetchkit-builtins.md`. +//! - **Host boundary.** [`Host`] is the only bridge: stdout/stderr buffers, +//! pipeline stdin, VFS reads (paths resolved against `cwd`), and URL fetches +//! through the per-`Bash` [`HttpClient`](crate::network::HttpClient), so the +//! allowlist, SSRF precheck, credential injection, bot-auth signing and the +//! host `HttpTransport` all apply. markkit is built without `http_client`: +//! it links no second HTTP stack. +//! - **Redirects** are followed here, one `HttpClient` call per hop, so every +//! hop is allowlist-checked (TM-NET-014), capped by markkit's +//! `max_redirects`. Relative links resolve against the final URL, like the +//! standalone CLI. +//! - **Limits only go down.** `--max-*`/`--timeout` flags may lower markkit's +//! default limits but never raise them (`Host::limits`), so one tenant's +//! script cannot buy a larger CPU/memory budget from a shared process. +//! - **Refused in the sandbox:** `--ocr` (runs a host Tesseract binary, reads +//! host model files) and the `mcp` subcommand (long-running stdio server). +//! Both still parse and show in `--help`; they fail with exit 1. +//! - **Stateless.** No globals; every run builds its own `Markkit`. + +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use markkit::{Extraction, Limits}; +use markkit_cli::{CliHost, HostFeature}; + +use super::{Builtin, Context, resolve_path}; +use crate::error::{Error, Result}; +use crate::fs::FileSystem; +use crate::interpreter::ExecResult; + +/// The `markkit` command (feature `markkit`). +pub struct Markkit; + +#[async_trait] +impl Builtin for Markkit { + async fn execute(&self, ctx: Context<'_>) -> Result { + let host = Host { + stdout: Mutex::new(String::new()), + stderr: Mutex::new(String::new()), + stdin: ctx.stdin.map(|s| s.as_bytes().to_vec()).unwrap_or_default(), + fs: ctx.fs.clone(), + cwd: ctx.cwd.clone(), + #[cfg(feature = "http_client")] + http: ctx.http_client.cloned(), + }; + let args = std::iter::once("markkit".to_string()).chain(ctx.args.iter().cloned()); + let exit_code = markkit_cli::run(args, &host).await; + Ok(ExecResult { + stdout: host.take_stdout().into(), + stderr: host.take_stderr().into(), + exit_code, + ..Default::default() + }) + } + + fn llm_hint(&self) -> Option<&'static str> { + Some( + "markkit: Convert HTML, PDF, DOCX, PPTX, XLSX, ODT, EPUB, email and more to Markdown \ + (`markkit extract FILE --file`, `markkit extract URL`, `... | markkit extract -`). \ + Same CLI as the standalone markkit. URLs need network allowlist; no --ocr, no mcp.", + ) + } +} + +/// [`CliHost`] over one builtin invocation. +struct Host { + stdout: Mutex, + stderr: Mutex, + stdin: Vec, + fs: Arc, + cwd: PathBuf, + #[cfg(feature = "http_client")] + http: Option>, +} + +impl Host { + fn take_stdout(&self) -> String { + std::mem::take(&mut *self.stdout.lock().unwrap_or_else(|p| p.into_inner())) + } + + fn take_stderr(&self) -> String { + std::mem::take(&mut *self.stderr.lock().unwrap_or_else(|p| p.into_inner())) + } +} + +impl CliHost for Host { + fn write_stdout(&self, text: &str) -> io::Result<()> { + self.stdout + .lock() + .unwrap_or_else(|p| p.into_inner()) + .push_str(text); + Ok(()) + } + + fn write_stderr(&self, text: &str) { + self.stderr + .lock() + .unwrap_or_else(|p| p.into_inner()) + .push_str(text); + } + + #[allow(clippy::manual_async_fn)] + fn read_stdin(&self) -> impl Future>> + Send { + let stdin = self.stdin.clone(); + async move { Ok(stdin) } + } + + #[allow(clippy::manual_async_fn)] + fn read_file(&self, path: &Path) -> impl Future>> + Send { + let resolved = resolve_path(&self.cwd, &path.to_string_lossy()); + let fs = self.fs.clone(); + async move { fs.read_file(&resolved).await.map_err(to_io_error) } + } + + #[allow(clippy::manual_async_fn)] + fn extract_url( + &self, + markkit: &markkit::Markkit, + url: &str, + ) -> impl Future> + Send { + let config = markkit.config().clone(); + let url = url.to_string(); + #[cfg(feature = "http_client")] + let http = self.http.clone(); + async move { + #[cfg(feature = "http_client")] + { + let Some(http) = http else { + return Err(not_configured()); + }; + fetch_and_extract(&http, config, &url).await + } + #[cfg(not(feature = "http_client"))] + { + let _ = (config, url); + Err(not_configured()) + } + } + } + + fn limits(&self, requested: Limits) -> Limits { + clamp_limits(requested, &Limits::default()) + } + + /// Fail closed: every host-only feature (today `--ocr`) is refused. + fn refuse(&self, feature: HostFeature) -> Option { + let _ = feature; + Some("is not available in bashkit".to_string()) + } +} + +fn not_configured() -> markkit::Error { + markkit::Error::UrlBlocked( + "network access not configured (see BashBuilder::network)".to_string(), + ) +} + +/// VFS errors as `io::Error`, keeping the kind (NotFound, PermissionDenied...). +fn to_io_error(err: Error) -> io::Error { + match err { + Error::Io(e) => e, + other => io::Error::other(other.to_string()), + } +} + +/// Each limit is the smaller of what the flags asked for and the cap. +fn clamp_limits(requested: Limits, cap: &Limits) -> Limits { + let mut l = requested; + macro_rules! clamp { + ($($f:ident),* $(,)?) => {$( l.$f = l.$f.min(cap.$f); )*}; + } + clamp!( + max_input_size, + max_output_size, + timeout, + max_nesting_depth, + max_nodes, + max_archive_entries, + max_archive_entry_size, + max_archive_total_size, + max_sheet_rows, + max_sheet_cols, + max_spreadsheet_cells, + max_redirects, + max_pdf_pages, + max_pdf_stream_size, + max_pdf_decoded_size, + ); + l +} + +/// GET `url` through the sandbox client, following redirects hop by hop, +/// then extract with `base_url` = the final URL. +#[cfg(feature = "http_client")] +async fn fetch_and_extract( + http: &crate::network::HttpClient, + mut config: markkit::MarkkitBuilder, + url: &str, +) -> markkit::Result { + use markkit::{LimitExceeded, LimitKind}; + + let limits = config.limits.clone(); + let mut current = url::Url::parse(url) + .map_err(|e| markkit::Error::UrlBlocked(format!("invalid URL: {e}")))?; + let mut redirects = 0usize; + let response = loop { + if !matches!(current.scheme(), "http" | "https") { + return Err(markkit::Error::UrlBlocked(format!( + "unsupported scheme: {}", + current.scheme() + ))); + } + let response = tokio::time::timeout(limits.timeout, http.get(current.as_str())) + .await + .map_err(|_| { + markkit::Error::from(LimitExceeded::new( + LimitKind::Timeout, + format!( + "fetch exceeded timeout of {}s", + limits.timeout.as_secs_f64() + ), + )) + })? + .map_err(|e| match e { + Error::Network(msg) if msg.contains("access denied") => { + markkit::Error::UrlBlocked(msg) + } + other => markkit::Error::Http(other.to_string()), + })?; + let location = matches!(response.status, 301 | 302 | 303 | 307 | 308) + .then(|| header(&response.headers, "location")) + .flatten(); + let Some(location) = location else { + break response; + }; + if redirects >= limits.max_redirects { + return Err(markkit::Error::Http(format!( + "too many redirects (max {})", + limits.max_redirects + ))); + } + redirects += 1; + current = current + .join(location) + .map_err(|e| markkit::Error::Http(format!("invalid redirect location: {e}")))?; + }; + + if response.body.len() > limits.max_input_size { + return Err(LimitExceeded::new( + LimitKind::MaxInputSize, + format!( + "response body exceeds max_input_size {}", + limits.max_input_size + ), + ) + .into()); + } + let content_type = header(&response.headers, "content-type").map(str::to_string); + config.base_url = Some(current.to_string()); + markkit::Markkit::new(config).extract_document_sync( + &response.body, + markkit::DocumentHint { + content_type: content_type.as_deref(), + file_name: None, + }, + ) +} + +#[cfg(feature = "http_client")] +fn header<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { + headers + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case(name)) + .map(|(_, v)| v.as_str()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn clamp_never_raises_limits() { + let cap = Limits::default(); + let mut huge = Limits::default(); + huge.max_input_size = usize::MAX; + huge.timeout = std::time::Duration::from_secs(1_000_000); + huge.max_redirects = 1_000; + let got = clamp_limits(huge, &cap); + assert_eq!(got, cap); + } + + #[test] + fn clamp_keeps_lowered_limits() { + let cap = Limits::default(); + let mut low = Limits::default(); + low.max_input_size = 10; + low.max_redirects = 0; + let got = clamp_limits(low.clone(), &cap); + assert_eq!(got, low); + } + + /// markkit internals that must never reach stderr (TM-INF-022). + const MARKKIT_BANNED: &[&str] = &[ + "LimitExceeded {", + "UnsupportedFormat(", + "HtmlParse(", + "Io(", + "Custom {", + "Stack backtrace", + "src/", + ]; + + macro_rules! markkit_no_leak { + ($name:ident, $script:expr) => { + #[tokio::test] + async fn $name() { + let r = crate::builtins::debug_leak_check::run($script).await; + assert_ne!(r.exit_code, 0, "{}", stringify!($name)); + crate::builtins::debug_leak_check::assert_no_leak( + &r, + stringify!($name), + MARKKIT_BANNED, + ); + } + }; + } + + markkit_no_leak!(no_leak_missing_file, "markkit extract --file /missing.pdf"); + markkit_no_leak!( + no_leak_input_limit, + "echo '

hello world

' | markkit extract - --max-input-size 3" + ); + markkit_no_leak!( + no_leak_bad_rules_json, + "echo '{' > /r.json; echo '

x

' | markkit extract - --rules /r.json" + ); + markkit_no_leak!( + no_leak_garbage_pdf, + "printf '%%PDF-1.7 garbage' > /x.pdf; markkit extract --file /x.pdf" + ); + markkit_no_leak!(no_leak_bad_url, "markkit extract 'http://['"); + markkit_no_leak!(no_leak_no_source, "markkit extract"); +} diff --git a/crates/bashkit/src/builtins/mod.rs b/crates/bashkit/src/builtins/mod.rs index 7cbdc726d..0f0fd39ea 100644 --- a/crates/bashkit/src/builtins/mod.rs +++ b/crates/bashkit/src/builtins/mod.rs @@ -59,6 +59,8 @@ mod export; mod expr; mod factor; mod fc; +#[cfg(all(feature = "fetchkit", not(target_arch = "wasm32")))] +mod fetchkit; mod fileops; mod find; mod flow; @@ -86,6 +88,8 @@ mod ls; mod make; mod man; mod mapfile; +#[cfg(all(feature = "markkit", not(target_arch = "wasm32")))] +mod markkit; mod mkfifo; #[cfg(feature = "terminal")] mod nano; @@ -195,6 +199,8 @@ pub use export::Export; pub use expr::Expr; pub use factor::Factor; pub use fc::Fc; +#[cfg(all(feature = "fetchkit", not(target_arch = "wasm32")))] +pub use fetchkit::Fetchkit; pub use fileops::{Chmod, Chown, Cp, Ln, Mkdir, Mktemp, Mv, Rm, Touch}; pub use find::Find; pub use flow::{Break, Colon, Continue, Exit, False, Return, True}; @@ -222,6 +228,8 @@ pub(crate) use ls::{fnmatch, glob_match}; pub use make::Make; pub use man::Man; pub use mapfile::Mapfile; +#[cfg(all(feature = "markkit", not(target_arch = "wasm32")))] +pub use markkit::Markkit; pub use mkfifo::Mkfifo; pub use navigation::{Cd, Pwd}; pub use nl::Nl; diff --git a/crates/bashkit/src/interpreter/mod.rs b/crates/bashkit/src/interpreter/mod.rs index 970764365..cd4c4fcfa 100644 --- a/crates/bashkit/src/interpreter/mod.rs +++ b/crates/bashkit/src/interpreter/mod.rs @@ -2352,6 +2352,10 @@ impl Interpreter { builtins.insert("jq".to_string(), Arc::new(builtins::Jq)); #[cfg(feature = "jq")] builtins.insert("yq".to_string(), Arc::new(builtins::Yq)); + #[cfg(all(feature = "markkit", not(target_arch = "wasm32")))] + builtins.insert("markkit".to_string(), Arc::new(builtins::Markkit)); + #[cfg(all(feature = "fetchkit", not(target_arch = "wasm32")))] + builtins.insert("fetchkit".to_string(), Arc::new(builtins::Fetchkit)); // Custom-construction builtins that need parameters diff --git a/crates/bashkit/tests/integration/fetchkit_builtin_tests.rs b/crates/bashkit/tests/integration/fetchkit_builtin_tests.rs new file mode 100644 index 000000000..e8439a543 --- /dev/null +++ b/crates/bashkit/tests/integration/fetchkit_builtin_tests.rs @@ -0,0 +1,244 @@ +//! `fetchkit` builtin: same CLI as the standalone binary, every request through +//! the sandbox HTTP client. See knowledge/runtimes/markkit-fetchkit-builtins.md. + +#![cfg(feature = "fetchkit")] + +use async_trait::async_trait; +use bashkit::{ + Bash, HttpResponse, HttpTransport, HttpTransportError, HttpTransportRequest, NetworkAllowlist, +}; +use std::sync::{Arc, Mutex}; + +const PAGE: &str = + "T

Hello

From the sandbox.

"; +const BASE: &str = "http://93.184.216.34"; + +/// `/start` redirects to `/page`; `/escape` redirects off the allowlist; +/// everything else serves PAGE (discovery probes get 404). +#[derive(Clone, Default)] +struct Site { + calls: Arc>>, +} + +impl Site { + fn urls(&self) -> Vec { + self.calls + .lock() + .unwrap() + .iter() + .map(|r| r.url.clone()) + .collect() + } +} + +#[async_trait] +impl HttpTransport for Site { + async fn execute( + &self, + request: HttpTransportRequest, + ) -> std::result::Result { + let url = request.url.clone(); + self.calls.lock().unwrap().push(request); + let redirect = |to: &str| HttpResponse { + status: 302, + headers: vec![("Location".into(), to.into())], + body: vec![], + }; + Ok(if url.ends_with("/start") { + redirect("/page") + } else if url.ends_with("/escape") { + redirect("http://93.184.216.99/secret") + } else if url.ends_with("/page") { + HttpResponse { + status: 200, + headers: vec![("Content-Type".into(), "text/html".into())], + body: PAGE.as_bytes().to_vec(), + } + } else { + HttpResponse { + status: 404, + headers: vec![("Content-Type".into(), "text/plain".into())], + body: b"not found".to_vec(), + } + }) + } +} + +fn bash(site: &Site) -> Bash { + Bash::builder() + .network(NetworkAllowlist::new().allow(BASE)) + .http_transport(Arc::new(site.clone())) + .build() +} + +#[tokio::test] +async fn help_and_version_are_the_fetchkit_cli_output() { + let mut bash = Bash::new(); + let help = bash.exec("fetchkit --help").await.unwrap(); + assert_eq!(help.exit_code, 0); + assert_eq!( + help.stdout, + fetchkit_cli::command().render_help().to_string() + ); + let version = bash.exec("fetchkit --version").await.unwrap(); + assert_eq!( + version.stdout, + fetchkit_cli::command().render_version().to_string() + ); + let sub = bash.exec("fetchkit fetch --help").await.unwrap(); + assert!( + sub.stdout.contains("--render-rakers"), + "refused flags still documented" + ); +} + +#[tokio::test] +async fn usage_error_exits_2() { + let mut bash = Bash::new(); + let r = bash.exec("fetchkit fetch --nope x").await.unwrap(); + assert_eq!(r.exit_code, 2); + assert!(r.stderr.contains("--nope"), "{}", r.stderr); +} + +#[tokio::test] +async fn fetch_goes_through_sandbox_client_and_follows_redirects() { + let site = Site::default(); + let mut bash = bash(&site); + let r = bash + .exec(&format!("fetchkit fetch {BASE}/start")) + .await + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert!(r.stdout.starts_with("---\n"), "{}", r.stdout); + assert!(r.stdout.contains("status_code: 200"), "{}", r.stdout); + assert!(r.stdout.contains("# Hello"), "{}", r.stdout); + let urls = site.urls(); + assert_eq!(urls[0], format!("{BASE}/start")); + assert_eq!(urls[1], format!("{BASE}/page")); + assert!(urls.iter().all(|u| u.starts_with(BASE)), "{urls:?}"); +} + +#[tokio::test] +async fn json_output_pipes_into_shell_tools() { + let site = Site::default(); + let mut bash = bash(&site); + let r = bash + .exec(&format!( + "fetchkit fetch {BASE}/page -o json | grep -c status_code" + )) + .await + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!(r.stdout.trim(), "1"); +} + +#[tokio::test] +async fn url_off_allowlist_never_reaches_transport() { + let site = Site::default(); + let mut bash = bash(&site); + let r = bash + .exec("fetchkit fetch http://93.184.216.99/") + .await + .unwrap(); + assert_eq!(r.exit_code, 1); + assert!(r.stderr.starts_with("Error: "), "{}", r.stderr); + assert!(site.urls().is_empty(), "{:?}", site.urls()); +} + +#[tokio::test] +async fn redirect_off_allowlist_is_blocked() { + let site = Site::default(); + let mut bash = bash(&site); + let r = bash + .exec(&format!("fetchkit fetch {BASE}/escape")) + .await + .unwrap(); + assert_eq!(r.exit_code, 1, "{}", r.stdout); + assert!( + !site.urls().iter().any(|u| u.contains("93.184.216.99")), + "{:?}", + site.urls() + ); +} + +#[tokio::test] +async fn without_network_fails_closed() { + let mut bash = Bash::new(); + let r = bash + .exec("fetchkit fetch https://example.com/") + .await + .unwrap(); + assert_eq!(r.exit_code, 1); + assert!( + r.stderr.contains("network access not configured"), + "{}", + r.stderr + ); +} + +#[tokio::test] +async fn host_only_flags_and_mcp_are_refused_before_any_request() { + for flag in [ + "--allow-env-proxy", + "--render-rakers", + "--bot-auth-key k", + "--allow-remote-conversion", + ] { + let site = Site::default(); + let mut bash = bash(&site); + let r = bash + .exec(&format!("fetchkit fetch {BASE}/page {flag}")) + .await + .unwrap(); + assert_eq!(r.exit_code, 1, "{flag}"); + assert!( + r.stderr.ends_with("is not available in bashkit\n"), + "{flag}: {}", + r.stderr + ); + assert!(site.urls().is_empty(), "{flag}"); + } + let mut bash = Bash::new(); + let r = bash.exec("fetchkit mcp < /dev/null").await.unwrap(); + assert_eq!(r.exit_code, 1); + assert_eq!( + r.stderr, + "Error: mcp is not available in this environment\n" + ); +} + +#[tokio::test] +async fn reads_shell_env_not_host_env() { + // CLOUDFLARE_* come from the shell; remote conversion stays refused anyway. + let site = Site::default(); + let mut bash = bash(&site); + let r = bash + .exec(&format!( + "CLOUDFLARE_API_TOKEN=x fetchkit fetch {BASE}/page --converter cloudflare" + )) + .await + .unwrap(); + // Unknown/unavailable converter is a clean one-line error or a normal fetch. + assert!(r.stderr.lines().count() <= 1, "{}", r.stderr); +} + +#[tokio::test] +async fn tenants_are_isolated() { + let a = Site::default(); + let b = Site::default(); + let mut bash_a = bash(&a); + let mut bash_b = Bash::builder() + .network(NetworkAllowlist::new().allow("http://93.184.216.35")) + .http_transport(Arc::new(b.clone())) + .build(); + let script = format!("fetchkit fetch {BASE}/page"); + let (ra, rb) = tokio::join!(bash_a.exec(&script), bash_b.exec(&script)); + assert_eq!(ra.unwrap().exit_code, 0); + assert_eq!( + rb.unwrap().exit_code, + 1, + "tenant B's allowlist does not cover A's host" + ); + assert!(!a.urls().is_empty()); + assert!(b.urls().is_empty()); +} diff --git a/crates/bashkit/tests/integration/main.rs b/crates/bashkit/tests/integration/main.rs index 80e68d20f..f93a0554f 100644 --- a/crates/bashkit/tests/integration/main.rs +++ b/crates/bashkit/tests/integration/main.rs @@ -53,11 +53,13 @@ pub mod credential_injection_tests; pub mod curl_data_compat_tests; pub mod custom_builtins_tests; pub mod custom_fs_tests; +pub mod fetchkit_builtin_tests; pub mod function_metadata_security_tests; pub mod http_binary_output_tests; pub mod input_redirect_budget_tests; pub mod interactive_builtins_tests; pub mod make_substitution_tests; +pub mod markkit_builtin_tests; pub mod memory_growth_security_tests; pub mod streaming_command_tests; pub mod tar_files_from_budget_tests; diff --git a/crates/bashkit/tests/integration/markkit_builtin_tests.rs b/crates/bashkit/tests/integration/markkit_builtin_tests.rs new file mode 100644 index 000000000..c217b2601 --- /dev/null +++ b/crates/bashkit/tests/integration/markkit_builtin_tests.rs @@ -0,0 +1,253 @@ +//! `markkit` builtin: same CLI as the standalone binary, on the VFS and the +//! network allowlist. See knowledge/runtimes/markkit-fetchkit-builtins.md. + +#![cfg(feature = "markkit")] + +use bashkit::Bash; + +const PAGE: &str = "T

Hello

next

"; + +async fn run(bash: &mut Bash, script: &str) -> bashkit::ExecResult { + bash.exec(script).await.unwrap() +} + +#[tokio::test] +async fn help_and_version_are_the_markkit_cli_output() { + let mut bash = Bash::new(); + let help = run(&mut bash, "markkit --help").await; + assert_eq!(help.exit_code, 0); + assert_eq!( + help.stdout, + markkit_cli::command().render_help().to_string() + ); + let version = run(&mut bash, "markkit --version").await; + assert_eq!(version.exit_code, 0); + assert_eq!( + version.stdout, + markkit_cli::command().render_version().to_string() + ); + let sub = run(&mut bash, "markkit extract --help").await; + assert_eq!(sub.exit_code, 0); + assert!(sub.stdout.contains("--max-input-size")); + assert!( + sub.stdout.contains("--ocr"), + "refused flags still documented" + ); +} + +#[tokio::test] +async fn usage_error_exits_2() { + let mut bash = Bash::new(); + let r = run(&mut bash, "markkit extract --no-such-flag x").await; + assert_eq!(r.exit_code, 2); + assert!(r.stderr.contains("--no-such-flag"), "{}", r.stderr); + assert!(r.stdout.is_empty()); +} + +#[tokio::test] +async fn extracts_file_from_vfs_relative_to_cwd() { + let mut bash = Bash::new(); + let script = format!( + "mkdir -p /work && cd /work && printf '%s' '{PAGE}' > doc.html && markkit extract --file doc.html" + ); + let r = run(&mut bash, &script).await; + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert!(r.stdout.contains("# Hello"), "{}", r.stdout); +} + +#[tokio::test] +async fn extracts_from_pipeline_stdin_and_json() { + let mut bash = Bash::new(); + let r = run( + &mut bash, + &format!("printf '%s' '{PAGE}' | markkit extract - --json"), + ) + .await; + assert_eq!(r.exit_code, 0, "{}", r.stderr); + let json: serde_json::Value = serde_json::from_str(&r.stdout).unwrap(); + assert_eq!(json["source"], "-"); + assert!(json["markdown"].as_str().unwrap().contains("# Hello")); +} + +#[tokio::test] +async fn rules_file_read_from_vfs() { + let mut bash = Bash::new(); + let script = format!( + "printf '%s' '[{{\"selector\": \"h1\", \"action\": \"skip\"}}]' > /r.json; printf '%s' '{PAGE}' | markkit extract - --rules /r.json" + ); + let r = run(&mut bash, &script).await; + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert!(!r.stdout.contains("Hello"), "{}", r.stdout); +} + +#[tokio::test] +async fn missing_vfs_file_is_one_line_error() { + let mut bash = Bash::new(); + let r = run(&mut bash, "markkit extract --file /nope.html").await; + assert_eq!(r.exit_code, 1); + assert!(r.stderr.starts_with("Error: "), "{}", r.stderr); + assert!( + !r.stderr.contains("Os {") && !r.stderr.contains("kind:"), + "{}", + r.stderr + ); +} + +#[tokio::test] +async fn host_files_are_not_reachable() { + // The VFS answers, never the host filesystem. + let mut bash = Bash::new(); + let r = run(&mut bash, "markkit extract --file /proc/self/environ").await; + assert_eq!(r.exit_code, 1, "{}", r.stdout); +} + +#[tokio::test] +async fn ocr_and_mcp_are_refused() { + let mut bash = Bash::new(); + let r = run(&mut bash, "echo x | markkit extract - --ocr").await; + assert_eq!(r.exit_code, 1); + assert_eq!(r.stderr, "Error: --ocr is not available in bashkit\n"); + let r = run(&mut bash, "markkit mcp < /dev/null").await; + assert_eq!(r.exit_code, 1); + assert_eq!( + r.stderr, + "Error: mcp is not available in this environment\n" + ); +} + +#[tokio::test] +async fn url_without_network_is_blocked() { + let mut bash = Bash::new(); + let r = run(&mut bash, "markkit extract https://example.com/").await; + assert_eq!(r.exit_code, 1); + assert!( + r.stderr.contains("network access not configured"), + "{}", + r.stderr + ); +} + +#[cfg(feature = "http_client")] +mod network { + use super::*; + use async_trait::async_trait; + use bashkit::{ + HttpResponse, HttpTransport, HttpTransportError, HttpTransportRequest, NetworkAllowlist, + }; + use std::sync::{Arc, Mutex}; + + /// `/start` redirects to `/final/page`; `/escape` redirects off the allowlist. + #[derive(Clone, Default)] + struct Site { + calls: Arc>>, + } + + #[async_trait] + impl HttpTransport for Site { + async fn execute( + &self, + request: HttpTransportRequest, + ) -> std::result::Result { + self.calls.lock().unwrap().push(request.url.clone()); + let redirect = |to: &str| HttpResponse { + status: 302, + headers: vec![("Location".into(), to.into())], + body: vec![], + }; + Ok(if request.url.ends_with("/start") { + redirect("/final/page") + } else if request.url.ends_with("/escape") { + redirect("http://93.184.216.99/secret") + } else if request.url.ends_with("/loop") { + redirect("/loop") + } else { + HttpResponse { + status: 200, + headers: vec![("Content-Type".into(), "text/html".into())], + body: PAGE.as_bytes().to_vec(), + } + }) + } + } + + fn bash(site: &Site) -> Bash { + Bash::builder() + .network(NetworkAllowlist::new().allow("http://93.184.216.34")) + .http_transport(Arc::new(site.clone())) + .build() + } + + #[tokio::test] + async fn url_follows_redirects_through_sandbox_client() { + let site = Site::default(); + let mut bash = bash(&site); + let r = run(&mut bash, "markkit extract http://93.184.216.34/start").await; + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert!(r.stdout.contains("# Hello"), "{}", r.stdout); + // Relative links resolve against the final URL, like the binary. + assert!( + r.stdout.contains("http://93.184.216.34/next"), + "{}", + r.stdout + ); + assert_eq!( + *site.calls.lock().unwrap(), + [ + "http://93.184.216.34/start", + "http://93.184.216.34/final/page" + ] + ); + } + + #[tokio::test] + async fn redirect_off_allowlist_is_blocked() { + let site = Site::default(); + let mut bash = bash(&site); + let r = run(&mut bash, "markkit extract http://93.184.216.34/escape").await; + assert_eq!(r.exit_code, 1); + assert!(r.stderr.contains("URL blocked"), "{}", r.stderr); + assert_eq!( + site.calls.lock().unwrap().len(), + 1, + "blocked hop never sent" + ); + } + + #[tokio::test] + async fn url_off_allowlist_is_blocked() { + let site = Site::default(); + let mut bash = bash(&site); + let r = run(&mut bash, "markkit extract http://93.184.216.99/").await; + assert_eq!(r.exit_code, 1); + assert!(r.stderr.contains("URL blocked"), "{}", r.stderr); + assert!(site.calls.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn redirect_loop_is_capped() { + let site = Site::default(); + let mut bash = bash(&site); + let r = run( + &mut bash, + "markkit extract http://93.184.216.34/loop --max-redirects 2", + ) + .await; + assert_eq!(r.exit_code, 1); + assert!(r.stderr.contains("too many redirects"), "{}", r.stderr); + assert_eq!(site.calls.lock().unwrap().len(), 3); + } + + #[tokio::test] + async fn flags_cannot_raise_redirect_cap() { + let site = Site::default(); + let mut bash = bash(&site); + let r = run( + &mut bash, + "markkit extract http://93.184.216.34/loop --max-redirects 100000", + ) + .await; + assert_eq!(r.exit_code, 1); + // Default cap (5) + the first request. + assert_eq!(site.calls.lock().unwrap().len(), 6); + } +} diff --git a/docs/markkit-fetchkit.md b/docs/markkit-fetchkit.md new file mode 100644 index 000000000..c4a90c0cc --- /dev/null +++ b/docs/markkit-fetchkit.md @@ -0,0 +1,72 @@ +# markkit and fetchkit + +Bashkit can ship two document tools as builtins, with the **same command line +as their standalone binaries**: + +- [`markkit`](https://github.com/everruns/markkit) converts HTML, PDF, DOCX, + PPTX, XLSX/XLS/ODS, ODT, EPUB, email, notebooks and more to Markdown. +- [`fetchkit`](https://github.com/everruns/fetchkit) fetches a URL and returns + LLM-ready Markdown with YAML frontmatter (or JSON), with site-aware fetchers + and bounded crawling. + +Both are **off by default**. Enable the Cargo features: + +```toml +bashkit = { version = "0.20.2", features = ["markkit", "fetchkit"] } +``` + +```bash +markkit extract report.pdf --file +markkit extract https://example.com/post --main-content +cat page.html | markkit extract - --json + +fetchkit fetch https://docs.rs/tokio --content-focus agent +fetchkit fetch https://example.com -o json | jq .status_code +``` + +`markkit --help` and `fetchkit --help` print exactly what the standalone tools +print: bashkit calls the same library code, so flags never drift. + +## Sandbox behavior + +- **Files** come from the virtual filesystem (paths relative to the shell's + working directory). The host filesystem is never read. +- **Network** goes through the same default-deny allowlist as `curl` + ([Networking](networking.md)). Every redirect hop is checked again. + Credential injection, request signing and a custom `HttpTransport` apply. + Without a `NetworkAllowlist`, URLs fail with `network access not configured`. +- **Limits**: markkit's `--max-*` and `--timeout` flags can lower its limits but + not raise them. Response bodies are capped by bashkit's HTTP limits. +- **Environment**: fetchkit reads variables from the shell, never from the + host process. + +```rust +use bashkit::{Bash, NetworkAllowlist}; + +let mut bash = Bash::builder() + .network(NetworkAllowlist::new().allow("https://docs.rs")) + .build(); +let out = bash.exec("fetchkit fetch https://docs.rs/tokio").await?; +``` + +## Not available in the sandbox + +These flags are accepted by the parser and listed in `--help`, then fail with +exit code 1 and `Error: is not available in bashkit`: + +| Command | Flag | Reason | +|---|---|---| +| both | `mcp` | long-running stdio server | +| markkit | `--ocr` | needs a host Tesseract binary or model files | +| fetchkit | `--allow-env-proxy` | host proxy settings | +| fetchkit | `--render-rakers` | embedded JavaScript renderer | +| fetchkit | `--bot-auth-key`, `--bot-auth-agent` | bashkit already signs requests ([Request signing](request-signing.md)) | +| fetchkit | `--allow-remote-conversion` | uploads documents to a third party | + +Both builtins are native only; the browser (wasm) package does not include them. + +## See also + +- [Networking & HTTP](networking.md) +- [Credential injection](../crates/bashkit/docs/credential-injection.md) +- [Security](security.md) diff --git a/knowledge/log.md b/knowledge/log.md index e79335f06..ef673b1f9 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -4,6 +4,8 @@ * **CI nightly pin**: Every nightly job (fuzz compile check, scheduled fuzz, Miri, ASan build-std) is pinned to `nightly-2026-10-30`. Nightly 2026-10-31 (rustc f9ceb4a4d) rejects the `bashkit` lib with E0391, an async-drop query cycle on `parser::ast::CommandList`; reproduced on unmodified `main`, and the 2026-10-30 nightly builds it. Unpin when a later nightly passes `cargo +nightly check -p bashkit --lib`. +* **markkit and fetchkit builtins**: new off-by-default features `markkit` and `fetchkit` register both CLIs with their exact interface by calling the upstream `run(args, host)` libraries. All file access goes to the VFS and all requests through the sandbox `HttpClient`, hop by hop. Host-only flags (`--ocr`, `--allow-env-proxy`, `--render-rakers`, `--bot-auth-*`, `--allow-remote-conversion`) and `mcp` are refused. See [Markkit and Fetchkit Builtins](runtimes/markkit-fetchkit-builtins.md), TM-NET-030, TM-DOS-139, L-DOC-001. + ## 2026-10-30 * **Windows signing preparation**: Separate unsigned Windows build outputs from publishable wheels. Add a credential-isolated Azure OIDC signing job, native signature/timestamp checks, RECORD regeneration, and a Windows packaging test job. Rollout requires verified publisher identity, a Public Trust profile, profile-scoped signer permission, and a GitHub environment restricted to release tags; the signing gate must remain unmerged until those prerequisites are ready. See [Python Package](runtimes/python-package.md#authenticode-release-path). diff --git a/knowledge/operations/limitations.md b/knowledge/operations/limitations.md index 2c0cb2381..7a90c3377 100644 --- a/knowledge/operations/limitations.md +++ b/knowledge/operations/limitations.md @@ -52,6 +52,7 @@ execution model. Evidence is a threat-model ID, a test, or `stance` | L-PIPE-001 | Pipeline stages stream from the first stage that runs shell code (loop, group, function, `eval`) or a registered live `Command`. Leading single-builtin stages run to completion first and hand over their whole output, so they never get SIGPIPE (`sort big \| head -1` gives `PIPESTATUS` `0 0`, bash `141 0`); `yes`, `seq`, plain `cat`, `grep` and `tr` are the exception and stream. Downstream of a streaming stage, a single builtin reads all its input before it starts, except registered live commands, `head`, `read`, plain `cat`, `grep` and `tr` (not `-s`); grep summary/context options can delay output, while `-q` still exits on the first matching record: `while :; do echo; done \| sed p \| head -1` runs until a limit. Pipes hold 4 KiB, not 64 KiB, so a finite loop writing more than that into an early-exiting reader exits 141 where bash would exit 0. Pipelines nested more than 4 subshells deep run their stages in sequence | Builtins return their output as one value, not a stream; streaming them needs a reader/writer `Context`. The smaller pipe bounds how far a producer runs ahead (each byte costs budgeted commands); the nesting cap bounds native stack (TM-DOS-124) | `l_pipe_001_stages_run_sequentially` | | L-SUDO-001 | `sudo` runs its command as the one sandbox user: `-u`, `-g`, `-i` change nothing, `whoami` still prints `sandbox`, and there is never a password prompt. `busybox APPLET` runs bashkit's builtin of that name, not BusyBox's own applet (flags and messages are bashkit's) | The sandbox has no privilege boundary to cross; the VFS has no permission enforcement (L-FS-002) | `sudo-busybox.test.sh` | | L-RAND-001 | `uuidgen` makes random (v4) UUIDs only (no `-t`, `-m`, `-s`); `openssl` implements only `rand` | Time/MAC-based UUIDs would leak host identity; a full TLS/crypto toolkit is out of scope | `uuidgen_time_based_unsupported` | +| L-DOC-001 | `markkit`/`fetchkit` builtins (features `markkit`, `fetchkit`, off by default, native only) refuse `mcp`, markkit `--ocr`, and fetchkit `--allow-env-proxy`, `--render-rakers`, `--bot-auth-key`/`--bot-auth-agent`, `--allow-remote-conversion` with exit 1 (`Error: is not available in bashkit`); markkit `--max-*`/`--timeout` cannot raise limits above markkit defaults | Host binaries, host files, host proxy env, a second signer and third-party uploads all cross the sandbox; `mcp` and rakers decided never in the sandbox (user, 2026-11-01). Help text stays identical to the standalone CLIs | `ocr_and_mcp_are_refused`, `host_only_flags_and_mcp_are_refused_before_any_request` | | L-FS-001 | Symlinks are followed, but `..` after a linked directory resolves lexically (`/link/..` is the link's parent, the `cd -L` view), and `ln` without `-s` (and `link`) makes a symlink, not a hard link | The interpreter normalizes paths before the VFS sees them; the VFS has no inodes to share | `symlink.test.sh`, `ln_default_symbolic` | | L-ROOTFS-001 | Default rootfs is static and read-only: `/proc` has no `self`, pid dirs, `uptime` or live counters; `/etc/passwd` has no root entry; `/dev/zero` yields 1 MiB per read; `/dev/full` is a plain file, so writes to it succeed (bash: `No space left on device`); `/bin`, `/usr/bin` are stubs that dispatch builtins | Host state must not leak (TM-INF-003, TM-ISO-018); fixed values keep runs deterministic | `rootfs_layout`, `threat_etc_passwd_blocked` | | L-FS-002 | No file permission enforcement in the VFS | Single-tenant virtual FS; permissions would be theater | `l_fs_002_no_permission_enforcement` | diff --git a/knowledge/runtimes/index.md b/knowledge/runtimes/index.md index a605bb988..c239f8b75 100644 --- a/knowledge/runtimes/index.md +++ b/knowledge/runtimes/index.md @@ -3,6 +3,7 @@ * [Python Builtin](python-builtin.md) - Embedded Python execution through Monty with security and resource controls. * [CPython WebAssembly Runtime](cpython-wasm.md) - Real CPython 3.14 as python3, compiled to WASI, snapshotted with Wizer and run on Wasmtime's Pulley interpreter over the bashkit VFS. * [ZapCode Runtime](zapcode-runtime.md) - Embedded TypeScript runtime, external functions, VFS bridging, and resource limits. +* [Markkit and Fetchkit Builtins](markkit-fetchkit-builtins.md) - The markkit and fetchkit CLIs as builtins with the identical interface, on the VFS and network allowlist. * [SQLite Builtin](sqlite-builtin.md) - Embedded SQLite through Turso with memory and virtual filesystem backends. * [Coreutils Argument Port](coreutils-args-port.md) - Code generation design for porting uutils clap arguments and uucore modules. * [Vendored jaq-json](jaq-json-vendor.md) - Why and how jq's JSON value type is vendored from upstream jaq-json, the memory meter patched into it, and how to sync upstream releases. diff --git a/knowledge/runtimes/markkit-fetchkit-builtins.md b/knowledge/runtimes/markkit-fetchkit-builtins.md new file mode 100644 index 000000000..b6788dcce --- /dev/null +++ b/knowledge/runtimes/markkit-fetchkit-builtins.md @@ -0,0 +1,117 @@ +--- +type: Subsystem Design +title: Markkit and Fetchkit Builtins +description: The markkit and fetchkit CLIs as bashkit builtins, with the identical interface, on the VFS and network allowlist. +tags: + - bashkit + - builtins + - markkit + - fetchkit + - network +--- + +# `markkit` and `fetchkit` Builtins + +## Status +Implemented, off by default (features `markkit`, `fetchkit`), native only. + +## Decision + +Bashkit ships [markkit](https://github.com/everruns/markkit) (document to +Markdown) and [fetchkit](https://github.com/everruns/fetchkit) (URL to LLM-ready +Markdown) as builtins with the **exact same CLI** as their standalone binaries. + +```bash +markkit extract report.pdf --file +curl -s "$URL" | markkit extract - --main-content +fetchkit fetch https://docs.rs/tokio --content-focus agent +fetchkit fetch https://example.com -o json | jq .status_code +``` + +### Same CLI by sharing code, not copying flags + +Both upstream CLIs are libraries: `markkit_cli::run(args, &impl CliHost)` and +`fetchkit_cli::run(args, &dyn CliHost)` return an exit code. The standalone +binaries are thin wrappers over the same functions. The builtins pass `argv` +through untouched, so flags, `--help`, `--version`, error text and exit codes +(0 ok, 1 runtime error, 2 usage error) cannot drift. Integration tests assert +`--help`/`--version` equal the libraries' clap output. Both upstream crates +build clap without color or suggestions, matching bashkit, because clap +features unify across the dependency graph. + +The `CliHost` trait is the only boundary: + +| Host method | markkit builtin | fetchkit builtin | +|---|---|---| +| stdout / stderr | buffered into `ExecResult` | same | +| stdin | pipeline stdin | n/a | +| file reads (`--file`, `--rules`) | VFS, resolved against `cwd` | n/a | +| env | n/a | shell env (`ctx.env`), never host env | +| URL egress | `HttpClient` per hop, redirects followed here | fetchkit `HttpTransport` over `HttpClient` | +| limits | `--max-*`/`--timeout` may lower, never raise, markkit defaults | bashkit `HttpLimits` cap every body | +| `mcp` subcommand | refused | refused | + +### Network + +Every request goes through the per-`Bash` `HttpClient`: allowlist, SSRF +precheck, credential injection, bot-auth signing, `before_http`/`after_http` +hooks and the host `HttpTransport`. Redirects are one `HttpClient` call per hop, +so every hop is allowlist-checked (TM-NET-014). No network configured: both +commands fail closed with `network access not configured`. + +- markkit is built **without** `http_client`; the builtin fetches bytes and calls + `extract_document` with `base_url` = final URL (same as the binary). +- fetchkit is built **without** `reqwest-tls`. fetchkit's own DNS policy is off + (`block_private_ips(false)`): bashkit already resolves-then-checks, and + fetchkit's resolver is blocking. fetchkit's agent-discovery probes and + specialized-fetcher API calls are ordinary hops through the same client. + +Neither adds a TLS stack, aws-lc-sys, or a second markkit +(`cargo tree -i aws-lc-sys` is empty with both features). + +### Refused flags + +These parse and show in `--help` (same interface), then fail with exit 1 and +`Error: is not available in bashkit` before any input is read or request +is sent. Hosts refuse unknown future host features too (fail closed). + +| Flag | Why | +|---|---| +| markkit `--ocr` (+ `--ocr-*`) | runs a host Tesseract binary, reads host model files | +| fetchkit `--allow-env-proxy` | host proxy env (TM-NET-015) | +| fetchkit `--render-rakers` | embedded QuickJS engine; decided never in the sandbox | +| fetchkit `--bot-auth-key` / `--bot-auth-agent` | bashkit signs in `HttpClient`; a key in argv double-signs and lands in history | +| fetchkit `--allow-remote-conversion` | uploads documents to a third party | +| `mcp` (both) | long-running stdio server; decided never in the sandbox | + +### Isolation and startup + +Builtins are stateless; all policy comes from the `Bash` instance's context. +The only shared values are the upstream crates' immutable clap trees. No +startup cost: registration is a map insert, nothing is loaded until a call. + +## Constraints + +- Native only (`not(target_arch = "wasm32")`): both CLI crates need tokio's + multi-thread runtime. +- Off by default everywhere (core, bashkit-cli, Python, Node) until a separate + decision turns them on (user, 2026-11-01). +- Upstream versions: markkit >= 0.3, fetchkit >= 0.7 (the `CliHost` releases). + fetchkit must depend on the same markkit minor as bashkit, or bashkit links + two markkit copies. + +## Tests + +- `tests/integration/markkit_builtin_tests.rs`, + `tests/integration/fetchkit_builtin_tests.rs`: help/version parity, usage + errors, VFS + stdin, allowlist and redirect enforcement, refused flags, limit + clamping, tenant isolation. +- `no_leak_*` in `builtins/markkit.rs` and `builtins/fetchkit.rs` (TM-INF-022). +- Upstream: `crates/*-cli/tests/host.rs` in each repo (binary vs `run()` byte + parity, host routing, `Send` futures). + +## See also + +- [Threat Model](../security/threat-model.md) TM-NET-030, TM-DOS-139 +- [Limitations](../operations/limitations.md) L-DOC-001 +- [HTTP Transport](../security/http-transport.md) diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 2986140c7..bc52f53eb 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -312,6 +312,7 @@ on default 2 MiB test threads (TM-DOS-044). | TM-DOS-137 | pr pagination amplification | A long `-h` title repeated once per empty line with `-l 11` grows gigabytes before a late output check; eager `-D` date copies amplify across retained inputs | Each row/output append checks the 16 MiB cap before growth; header parts append directly without a temporary title copy. Inputs retain date timestamps, formatting dates one nonempty stream at a time. Input lines, cells, rows and output lease capacity before allocation; cells borrow input text and pages borrow row slices. Input/row/page loops charge shared work, input bytes charge aggregate input, and budgeted growth polls cancellation/deadlines. Regressions: `pr_resource_tests`, cap/UTF-8/cancellation unit tests and `pr_allocation_tests` peak-heap bound | **MITIGATED** | | TM-DOS-107 | Static-analysis command validation amplification | A large comment before thousands of commands makes a whole-script ordered-subsequence scan per command | Build one source-character position index, then validate each analyzed command-name character with a logarithmic position lookup instead of rescanning source | **MITIGATED** | | TM-DOS-138 | Join matching-row output growth | Equal-key row groups multiply generated output and synchronous work before dispatcher capture | Parsed rows, fields, keys, decoded input and generated output own shared-budget capacity before growth. VFS reads charge aggregate input and admit metadata size; output fields append directly without temporary row storage, and the byte-stream copy is admitted. Parsing, grouping (including suppressed matches), comparisons and emissions charge work and poll cancellation/deadlines. Errors abort the request. Ordinary two-row fixtures cover memory/work rejection, cancellation/deadlines, session recovery, and complete pipeline/redirect output | **MITIGATED** | +| TM-DOS-139 | Embedded CLI limit escalation | `markkit extract x --max-input-size 99999999999 --timeout 1e9` buys a larger CPU/memory budget in a shared process | Host clamps every markkit limit to its default after flags (`CliHost::limits`); fetchkit bodies capped by `HttpLimits`; OCR (host binaries) refused | **MITIGATED** | **TM-DOS-051** is historical. The custom parser was deleted with the `yaml` helper; the replacement `yq` parser/evaluator boundary is tracked by TM-DOS-101. @@ -883,6 +884,7 @@ matches literal host strings exactly, never resolved IPs; no DNS lookup. | TM-NET-023 | HTTP-handler SSRF via fail-open precheck and rebind window | `HttpClient::check_private_ip` returned `Ok(())` on URL-parse failures and on URLs with no host, letting malformed targets reach the connect path with no IP filter. Custom `HttpHandler` implementations are doubly exposed because they don't get reqwest's connect-time `PrivateIpFilteringResolver`, even a successful precheck leaves a rebind window between validation and the moment the handler opens its own socket. | (1) `check_private_ip` now fails closed on malformed URLs and on URLs with no host. The direct-IP branch and the successful-DNS branch remain fail-closed against private addresses (including the v4-mapped IPv6 forms covered by TM-NET-022). Tested via `test_check_private_ip_fails_closed_on_invalid_url`, `test_check_private_ip_fails_closed_on_no_host`, `test_check_private_ip_blocks_literal_private_ip`, `test_check_private_ip_blocks_metadata_via_v4_mapped_v6`. (2) The `HttpTransport` trait doc explicitly assigns SSRF responsibility to network-dialing custom transports and points them at `bashkit::network::is_private_ip` for the same classifier the default reqwest path uses; `HttpTransportRequest.pinned_addrs` additionally forwards the precheck's resolve-then-check result so transports (and host egress boundaries behind them) can pin the validated addresses and close the rebind window (see [HTTP Transport](http-transport.md)). DNS-lookup errors at the precheck still pass through (fail-open by design, the rebind / connect-time threat is the handler's responsibility, and failing closed there breaks legitimate `before_http` hook flows that intentionally target unresolved hostnames). | **MITIGATED** (#1570) | | TM-NET-028 | Repeated curl data parts bypass the request-body cap | Many individually valid `-d`/`--data-*` values, file reads, encoding expansion, and inserted `&` separators combine into an oversized allocation/request | One ordered aggregate uses checked appends; every separator and encoded byte counts toward the 10 MB cap; file metadata is compared with remaining capacity before VFS reads. Exact-boundary and one-byte-over tests cover the invariant. | **MITIGATED** | | TM-NET-029 | Private IP behind a v4-embedding or reserved v6/v4 form | Allowlisted host returns AAAA `64:ff9b::a9fe:a9fe` (NAT64), `2002:7f00:1::` (6to4), or A in `0.0.0.0/8`, `198.18.0.0/15`, multicast, `240.0.0.0/4` | `is_private_ip` classifies NAT64/6to4 by the embedded v4 address and blocks all non-global special-purpose ranges (Teredo, local-use NAT64, site-local, multicast) at pre-resolve and connect time | **MITIGATED** | +| TM-NET-030 | Embedded CLI egress bypass | `fetchkit fetch`/`markkit extract URL` open their own sockets, follow redirects off the allowlist, or read host proxy env | Both builtins route every hop (redirects, discovery probes, API subrequests) through the sandbox `HttpClient`; no upstream TLS stack is linked; `--allow-env-proxy` refused ([Markkit and Fetchkit Builtins](../runtimes/markkit-fetchkit-builtins.md)) | **MITIGATED** | **Current Risk**: LOW - Multiple mitigations in place diff --git a/site/src/pages/docs/_meta.ts b/site/src/pages/docs/_meta.ts index 5773a33ce..4ca58d657 100644 --- a/site/src/pages/docs/_meta.ts +++ b/site/src/pages/docs/_meta.ts @@ -260,6 +260,16 @@ export const DOC_META: DocMeta[] = [ collection: "rustdocs", editPath: "crates/bashkit/docs/sqlite.md", }, + { + slug: "markkit-fetchkit", + title: "markkit and fetchkit", + summary: "Document-to-Markdown and URL fetching builtins with the standalone CLIs' interface.", + seoTitle: "Bashkit markkit and fetchkit builtins", + seoDescription: + "Convert PDF, DOCX, HTML and more to Markdown and fetch URLs as LLM-ready Markdown inside Bashkit, with the exact markkit and fetchkit CLIs, the VFS and the network allowlist.", + section: "Runtimes", + editPath: "docs/markkit-fetchkit.md", + }, { slug: "ssh", title: "SSH support", From f80fda5699caefa6cfa30b4acdddcfb35717299b Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Sun, 1 Nov 2026 03:25:33 +0000 Subject: [PATCH 2/5] fix(site): shorten markkit-fetchkit SEO description to 70-160 chars Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Wy7MByPXsW4jHxfLeTfGEc --- site/src/pages/docs/_meta.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/src/pages/docs/_meta.ts b/site/src/pages/docs/_meta.ts index 4ca58d657..744227ddd 100644 --- a/site/src/pages/docs/_meta.ts +++ b/site/src/pages/docs/_meta.ts @@ -266,7 +266,7 @@ export const DOC_META: DocMeta[] = [ summary: "Document-to-Markdown and URL fetching builtins with the standalone CLIs' interface.", seoTitle: "Bashkit markkit and fetchkit builtins", seoDescription: - "Convert PDF, DOCX, HTML and more to Markdown and fetch URLs as LLM-ready Markdown inside Bashkit, with the exact markkit and fetchkit CLIs, the VFS and the network allowlist.", + "Convert PDF, DOCX and HTML to Markdown and fetch URLs inside Bashkit with the exact markkit and fetchkit CLIs, sandboxed.", section: "Runtimes", editPath: "docs/markkit-fetchkit.md", }, From 5feb8de6f6e59b5de4c08dda78d9378e435a06cf Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Sun, 1 Nov 2026 03:28:39 +0000 Subject: [PATCH 3/5] chore(supply-chain): exempt markkit and fetchkit dependency trees in cargo-vet Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Wy7MByPXsW4jHxfLeTfGEc --- .../runtimes/markkit-fetchkit-builtins.md | 5 + supply-chain/config.toml | 340 ++++++++++++++++++ 2 files changed, 345 insertions(+) diff --git a/knowledge/runtimes/markkit-fetchkit-builtins.md b/knowledge/runtimes/markkit-fetchkit-builtins.md index b6788dcce..4a7034b65 100644 --- a/knowledge/runtimes/markkit-fetchkit-builtins.md +++ b/knowledge/runtimes/markkit-fetchkit-builtins.md @@ -99,6 +99,11 @@ startup cost: registration is a map insert, nothing is loaded until a call. - Upstream versions: markkit >= 0.3, fetchkit >= 0.7 (the `CliHost` releases). fetchkit must depend on the same markkit minor as bashkit, or bashkit links two markkit copies. +- Supply chain: the 85 crates these trees add (PDF/DOCX/zip/image parsers, + html5ever, fetchkit's own deps) entered `supply-chain/config.toml` as + `safe-to-deploy` exemptions, not audits, when the features landed. They are + only compiled with `markkit`/`fetchkit` on; audit them before turning either + feature on by default. ## Tests diff --git a/supply-chain/config.toml b/supply-chain/config.toml index 92f834f8c..bf5cc43ac 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -94,10 +94,26 @@ criteria = "safe-to-deploy" version = "0.1.6" criteria = "safe-to-deploy" +[[exemptions.anstream]] +version = "1.0.0" +criteria = "safe-to-deploy" + [[exemptions.anstyle]] version = "1.0.14" criteria = "safe-to-deploy" +[[exemptions.anstyle-parse]] +version = "1.0.0" +criteria = "safe-to-deploy" + +[[exemptions.anstyle-query]] +version = "1.1.5" +criteria = "safe-to-deploy" + +[[exemptions.anstyle-wincon]] +version = "3.0.11" +criteria = "safe-to-deploy" + [[exemptions.antithesis_sdk]] version = "0.2.9" criteria = "safe-to-deploy" @@ -134,6 +150,10 @@ criteria = "safe-to-deploy" version = "0.1.92" criteria = "safe-to-deploy" +[[exemptions.atoi_simd]] +version = "0.18.1" +criteria = "safe-to-deploy" + [[exemptions.atomic-polyfill]] version = "1.0.3" criteria = "safe-to-deploy" @@ -206,6 +226,10 @@ criteria = "safe-to-deploy" version = "0.12.1" criteria = "safe-to-deploy" +[[exemptions.block-padding]] +version = "0.3.3" +criteria = "safe-to-deploy" + [[exemptions.block-padding]] version = "0.4.2" criteria = "safe-to-deploy" @@ -255,6 +279,10 @@ version = "0.6.1" criteria = "safe-to-deploy" notes = "Reviewed the slice-to-FFI stream wrapper, buffer accounting, drop paths, and pure-Rust backend selection; no ambient capabilities." +[[exemptions.calamine]] +version = "0.37.0" +criteria = "safe-to-deploy" + [[exemptions.cast]] version = "0.3.0" criteria = "safe-to-run" @@ -263,6 +291,10 @@ criteria = "safe-to-run" version = "0.2.4" criteria = "safe-to-deploy" +[[exemptions.cbc]] +version = "0.1.2" +criteria = "safe-to-deploy" + [[exemptions.cbc]] version = "0.2.1" criteria = "safe-to-deploy" @@ -271,6 +303,10 @@ criteria = "safe-to-deploy" version = "1.4.3" criteria = "safe-to-deploy" +[[exemptions.cfb]] +version = "0.15.0" +criteria = "safe-to-deploy" + [[exemptions.cfg-if]] version = "1.0.4" criteria = "safe-to-deploy" @@ -340,10 +376,18 @@ criteria = "safe-to-deploy" version = "0.5.4" criteria = "safe-to-deploy" +[[exemptions.codepage]] +version = "0.1.3" +criteria = "safe-to-deploy" + [[exemptions.collection_literals]] version = "1.0.3" criteria = "safe-to-deploy" +[[exemptions.colorchoice]] +version = "1.0.5" +criteria = "safe-to-deploy" + [[exemptions.colored]] version = "3.1.1" criteria = "safe-to-deploy" @@ -460,6 +504,14 @@ criteria = "safe-to-deploy" version = "0.7.2" criteria = "safe-to-deploy" +[[exemptions.cssparser]] +version = "0.38.0" +criteria = "safe-to-deploy" + +[[exemptions.cssparser-macros]] +version = "0.7.1" +criteria = "safe-to-deploy" + [[exemptions.ctor]] version = "1.0.13" criteria = "safe-to-deploy" @@ -488,6 +540,10 @@ criteria = "safe-to-deploy" version = "2.11.1" criteria = "safe-to-deploy" +[[exemptions.debug_unsafe]] +version = "0.1.4" +criteria = "safe-to-deploy" + [[exemptions.defmt]] version = "1.1.1" criteria = "safe-to-deploy" @@ -508,10 +564,22 @@ criteria = "safe-to-deploy" version = "0.8.1" criteria = "safe-to-deploy" +[[exemptions.deranged]] +version = "0.5.9" +criteria = "safe-to-deploy" + [[exemptions.derive-where]] version = "1.6.1" criteria = "safe-to-deploy" +[[exemptions.derive_more]] +version = "2.1.1" +criteria = "safe-to-deploy" + +[[exemptions.derive_more-impl]] +version = "2.1.1" +criteria = "safe-to-deploy" + [[exemptions.des]] version = "0.9.0" criteria = "safe-to-deploy" @@ -536,6 +604,14 @@ criteria = "safe-to-deploy" version = "0.1.12" criteria = "safe-to-deploy" +[[exemptions.dtoa]] +version = "1.0.11" +criteria = "safe-to-deploy" + +[[exemptions.dtoa-short]] +version = "0.3.5" +criteria = "safe-to-deploy" + [[exemptions.dunce]] version = "1.0.5" criteria = "safe-to-deploy" @@ -544,6 +620,10 @@ criteria = "safe-to-deploy" version = "1.0.20" criteria = "safe-to-deploy" +[[exemptions.ecb]] +version = "0.1.2" +criteria = "safe-to-deploy" + [[exemptions.ecdsa]] version = "0.17.0" criteria = "safe-to-deploy" @@ -556,6 +636,10 @@ criteria = "safe-to-deploy" version = "3.0.0" criteria = "safe-to-deploy" +[[exemptions.ego-tree]] +version = "0.11.1" +criteria = "safe-to-deploy" + [[exemptions.either]] version = "1.17.0" criteria = "safe-to-deploy" @@ -568,6 +652,10 @@ criteria = "safe-to-deploy" version = "1.0.0" criteria = "safe-to-run" +[[exemptions.encoding_rs]] +version = "0.8.44" +criteria = "safe-to-deploy" + [[exemptions.endian-type]] version = "0.2.0" criteria = "safe-to-deploy" @@ -576,6 +664,14 @@ criteria = "safe-to-deploy" version = "0.3.13" criteria = "safe-to-deploy" +[[exemptions.env_filter]] +version = "2.0.0" +criteria = "safe-to-deploy" + +[[exemptions.env_logger]] +version = "0.11.11" +criteria = "safe-to-deploy" + [[exemptions.equivalent]] version = "1.0.2" criteria = "safe-to-deploy" @@ -600,6 +696,10 @@ criteria = "safe-to-deploy" version = "0.19.2" criteria = "safe-to-deploy" +[[exemptions.fast-float2]] +version = "0.2.4" +criteria = "safe-to-deploy" + [[exemptions.fastbloom]] version = "0.14.1" criteria = "safe-to-deploy" @@ -608,6 +708,14 @@ criteria = "safe-to-deploy" version = "2.5.0" criteria = "safe-to-deploy" +[[exemptions.fetchkit]] +version = "0.7.0" +criteria = "safe-to-deploy" + +[[exemptions.fetchkit-cli]] +version = "0.7.0" +criteria = "safe-to-deploy" + [[exemptions.ff]] version = "0.14.0" criteria = "safe-to-deploy" @@ -736,6 +844,26 @@ criteria = "safe-to-deploy" version = "0.15.5" criteria = "safe-to-deploy" +[[exemptions.hashify]] +version = "0.2.9" +criteria = "safe-to-deploy" + +[[exemptions.hayro-ccitt]] +version = "0.4.0" +criteria = "safe-to-deploy" + +[[exemptions.hayro-jbig2]] +version = "0.3.1" +criteria = "safe-to-deploy" + +[[exemptions.hayro-jpeg2000]] +version = "0.4.1" +criteria = "safe-to-deploy" + +[[exemptions.hayro-syntax]] +version = "0.8.0" +criteria = "safe-to-deploy" + [[exemptions.heapless]] version = "0.7.17" criteria = "safe-to-deploy" @@ -764,6 +892,10 @@ criteria = "safe-to-deploy" version = "0.5.12" criteria = "safe-to-deploy" +[[exemptions.html5ever]] +version = "0.40.1" +criteria = "safe-to-deploy" + [[exemptions.http]] version = "1.5.0" criteria = "safe-to-deploy" @@ -864,6 +996,14 @@ criteria = "safe-to-deploy" version = "1.0.0" criteria = "safe-to-deploy" +[[exemptions.include_dir]] +version = "0.7.4" +criteria = "safe-to-deploy" + +[[exemptions.include_dir_macros]] +version = "0.7.4" +criteria = "safe-to-deploy" + [[exemptions.indexmap]] version = "2.14.0" criteria = "safe-to-deploy" @@ -900,6 +1040,10 @@ criteria = "safe-to-deploy" version = "0.3.7" criteria = "safe-to-deploy" +[[exemptions.is_terminal_polyfill]] +version = "1.70.2" +criteria = "safe-to-deploy" + [[exemptions.itertools]] version = "0.13.0" criteria = "safe-to-run" @@ -1033,6 +1177,10 @@ criteria = "safe-to-deploy" version = "0.7.2" criteria = "safe-to-deploy" +[[exemptions.lopdf]] +version = "0.42.0" +criteria = "safe-to-deploy" + [[exemptions.lzma-rs]] version = "0.3.0" criteria = "safe-to-deploy" @@ -1041,6 +1189,10 @@ criteria = "safe-to-deploy" version = "0.6.0" criteria = "safe-to-deploy" +[[exemptions.mail-parser]] +version = "0.11.9" +criteria = "safe-to-deploy" + [[exemptions.manyhow]] version = "0.11.4" criteria = "safe-to-deploy" @@ -1049,6 +1201,22 @@ criteria = "safe-to-deploy" version = "0.11.4" criteria = "safe-to-deploy" +[[exemptions.markkit]] +version = "0.3.0" +criteria = "safe-to-deploy" + +[[exemptions.markkit-cli]] +version = "0.3.0" +criteria = "safe-to-deploy" + +[[exemptions.markup5ever]] +version = "0.40.0" +criteria = "safe-to-deploy" + +[[exemptions.md-5]] +version = "0.10.6" +criteria = "safe-to-deploy" + [[exemptions.md-5]] version = "0.11.0" criteria = "safe-to-deploy" @@ -1113,6 +1281,10 @@ criteria = "safe-to-deploy" version = "0.0.19" criteria = "safe-to-deploy" +[[exemptions.multiversion_no_op]] +version = "1.0.0" +criteria = "safe-to-deploy" + [[exemptions.napi]] version = "3.12.4" criteria = "safe-to-deploy" @@ -1133,6 +1305,10 @@ criteria = "safe-to-deploy" version = "3.3.1" criteria = "safe-to-deploy" +[[exemptions.new_debug_unreachable]] +version = "1.0.6" +criteria = "safe-to-deploy" + [[exemptions.nibble_vec]] version = "0.1.0" criteria = "safe-to-deploy" @@ -1145,6 +1321,10 @@ criteria = "safe-to-deploy" version = "0.2.0" criteria = "safe-to-deploy" +[[exemptions.nom]] +version = "8.0.0" +criteria = "safe-to-deploy" + [[exemptions.nonmax]] version = "0.5.5" criteria = "safe-to-deploy" @@ -1153,6 +1333,10 @@ criteria = "safe-to-deploy" version = "0.4.8" criteria = "safe-to-deploy" +[[exemptions.num-conv]] +version = "0.2.2" +criteria = "safe-to-deploy" + [[exemptions.num-integer]] version = "0.1.47" criteria = "safe-to-deploy" @@ -1165,6 +1349,10 @@ criteria = "safe-to-deploy" version = "1.21.4" criteria = "safe-to-deploy" +[[exemptions.once_cell_polyfill]] +version = "1.70.2" +criteria = "safe-to-deploy" + [[exemptions.opaque-debug]] version = "0.3.1" criteria = "safe-to-deploy" @@ -1293,6 +1481,10 @@ criteria = "safe-to-deploy" version = "0.13.0" criteria = "safe-to-deploy" +[[exemptions.pdf-inspector]] +version = "0.1.8" +criteria = "safe-to-deploy" + [[exemptions.pem-rfc7468]] version = "1.0.0" criteria = "safe-to-deploy" @@ -1314,10 +1506,18 @@ notes = "Compile-time perfect-hash runtime used by chrono-tz; no unsafe code." version = "0.13.1" criteria = "safe-to-deploy" +[[exemptions.phf]] +version = "0.14.0" +criteria = "safe-to-deploy" + [[exemptions.phf_codegen]] version = "0.11.3" criteria = "safe-to-deploy" +[[exemptions.phf_codegen]] +version = "0.14.0" +criteria = "safe-to-deploy" + [[exemptions.phf_generator]] version = "0.11.3" criteria = "safe-to-deploy" @@ -1326,10 +1526,18 @@ criteria = "safe-to-deploy" version = "0.13.1" criteria = "safe-to-deploy" +[[exemptions.phf_generator]] +version = "0.14.0" +criteria = "safe-to-deploy" + [[exemptions.phf_macros]] version = "0.13.1" criteria = "safe-to-deploy" +[[exemptions.phf_macros]] +version = "0.14.0" +criteria = "safe-to-deploy" + [[exemptions.phf_shared]] version = "0.11.3" criteria = "safe-to-deploy" @@ -1343,6 +1551,10 @@ notes = "Hash primitives used by phf; no unsafe code." version = "0.13.1" criteria = "safe-to-deploy" +[[exemptions.phf_shared]] +version = "0.14.0" +criteria = "safe-to-deploy" + [[exemptions.pin-project-lite]] version = "0.2.17" criteria = "safe-to-deploy" @@ -1399,10 +1611,18 @@ criteria = "safe-to-deploy" version = "0.1.6" criteria = "safe-to-deploy" +[[exemptions.powerfmt]] +version = "0.2.1" +criteria = "safe-to-deploy" + [[exemptions.ppv-lite86]] version = "0.2.21" criteria = "safe-to-deploy" +[[exemptions.precomputed-hash]] +version = "0.1.1" +criteria = "safe-to-deploy" + [[exemptions.pretty_assertions]] version = "1.4.1" criteria = "safe-to-run" @@ -1435,6 +1655,10 @@ criteria = "safe-to-deploy" version = "1.12.0" criteria = "safe-to-run" +[[exemptions.pulldown-cmark]] +version = "0.13.6" +criteria = "safe-to-deploy" + [[exemptions.pyo3]] version = "0.29.2" criteria = "safe-to-deploy" @@ -1463,6 +1687,14 @@ criteria = "safe-to-deploy" version = "1.2.3" criteria = "safe-to-run" +[[exemptions.quick-xml]] +version = "0.41.0" +criteria = "safe-to-deploy" + +[[exemptions.quick-xml]] +version = "0.42.0" +criteria = "safe-to-deploy" + [[exemptions.quote]] version = "1.0.47" criteria = "safe-to-deploy" @@ -1531,6 +1763,10 @@ criteria = "safe-to-deploy" version = "0.5.0" criteria = "safe-to-run" +[[exemptions.rangemap]] +version = "1.8.0" +criteria = "safe-to-deploy" + [[exemptions.rapidhash]] version = "4.5.1" criteria = "safe-to-deploy" @@ -1547,6 +1783,14 @@ criteria = "safe-to-deploy" version = "0.5.18" criteria = "safe-to-deploy" +[[exemptions.ref-cast]] +version = "1.0.27" +criteria = "safe-to-deploy" + +[[exemptions.ref-cast-impl]] +version = "1.0.27" +criteria = "safe-to-deploy" + [[exemptions.regex]] version = "1.13.1" criteria = "safe-to-deploy" @@ -1687,6 +1931,14 @@ criteria = "safe-to-deploy" version = "0.1.29" criteria = "safe-to-deploy" +[[exemptions.schemars]] +version = "1.2.3" +criteria = "safe-to-deploy" + +[[exemptions.schemars_derive]] +version = "1.2.3" +criteria = "safe-to-deploy" + [[exemptions.scoped-tls]] version = "1.0.1" criteria = "safe-to-deploy" @@ -1695,6 +1947,10 @@ criteria = "safe-to-deploy" version = "1.2.0" criteria = "safe-to-deploy" +[[exemptions.scraper]] +version = "0.29.0" +criteria = "safe-to-deploy" + [[exemptions.scrypt]] version = "0.12.0" criteria = "safe-to-deploy" @@ -1711,6 +1967,10 @@ criteria = "safe-to-deploy" version = "2.17.0" criteria = "safe-to-deploy" +[[exemptions.selectors]] +version = "0.41.0" +criteria = "safe-to-deploy" + [[exemptions.self_cell]] version = "1.3.0" criteria = "safe-to-deploy" @@ -1739,6 +1999,10 @@ criteria = "safe-to-deploy" version = "1.0.229" criteria = "safe-to-deploy" +[[exemptions.serde_derive_internals]] +version = "0.30.0" +criteria = "safe-to-deploy" + [[exemptions.serde_json]] version = "1.0.151" criteria = "safe-to-deploy" @@ -1760,6 +2024,10 @@ criteria = "safe-to-run" version = "4.0.1" criteria = "safe-to-run" +[[exemptions.servo_arc]] +version = "0.5.0" +criteria = "safe-to-deploy" + [[exemptions.sha1]] version = "0.11.0" criteria = "safe-to-deploy" @@ -1868,6 +2136,18 @@ criteria = "safe-to-deploy" version = "1.2.1" criteria = "safe-to-deploy" +[[exemptions.string_cache]] +version = "0.11.0" +criteria = "safe-to-deploy" + +[[exemptions.string_cache_codegen]] +version = "0.11.2" +criteria = "safe-to-deploy" + +[[exemptions.stringprep]] +version = "0.1.5" +criteria = "safe-to-deploy" + [[exemptions.strum]] version = "0.26.3" criteria = "safe-to-deploy" @@ -1920,6 +2200,10 @@ criteria = "safe-to-deploy" version = "3.27.0" criteria = "safe-to-deploy" +[[exemptions.tendril]] +version = "0.5.1" +criteria = "safe-to-deploy" + [[exemptions.termcolor]] version = "1.4.1" criteria = "safe-to-deploy" @@ -1948,6 +2232,18 @@ criteria = "safe-to-deploy" version = "1.1.10" criteria = "safe-to-deploy" +[[exemptions.time]] +version = "0.3.55" +criteria = "safe-to-deploy" + +[[exemptions.time-core]] +version = "0.1.9" +criteria = "safe-to-deploy" + +[[exemptions.time-macros]] +version = "0.2.32" +criteria = "safe-to-deploy" + [[exemptions.tinystr]] version = "0.8.4" criteria = "safe-to-deploy" @@ -2016,6 +2312,10 @@ criteria = "safe-to-deploy" version = "0.2.5" criteria = "safe-to-deploy" +[[exemptions.ttf-parser]] +version = "0.25.1" +criteria = "safe-to-deploy" + [[exemptions.turso_core]] version = "0.8.0-pre.14" criteria = "safe-to-deploy" @@ -2040,6 +2340,10 @@ criteria = "safe-to-deploy" version = "2.0.2" criteria = "safe-to-deploy" +[[exemptions.typed-path]] +version = "0.13.0" +criteria = "safe-to-deploy" + [[exemptions.typenum]] version = "1.20.1" criteria = "safe-to-deploy" @@ -2048,6 +2352,14 @@ criteria = "safe-to-deploy" version = "0.9.10" criteria = "safe-to-deploy" +[[exemptions.unicase]] +version = "2.10.0" +criteria = "safe-to-deploy" + +[[exemptions.unicode-bidi]] +version = "0.3.18" +criteria = "safe-to-deploy" + [[exemptions.unicode-general-category]] version = "1.1.0" criteria = "safe-to-deploy" @@ -2060,6 +2372,10 @@ criteria = "safe-to-deploy" version = "1.0.24" criteria = "safe-to-deploy" +[[exemptions.unicode-properties]] +version = "0.1.5" +criteria = "safe-to-deploy" + [[exemptions.unicode_names2]] version = "1.3.0" criteria = "safe-to-deploy" @@ -2175,10 +2491,18 @@ criteria = "safe-to-deploy" version = "1.1.0" criteria = "safe-to-deploy" +[[exemptions.web_atoms]] +version = "0.3.0" +criteria = "safe-to-deploy" + [[exemptions.webpki-root-certs]] version = "1.0.9" criteria = "safe-to-deploy" +[[exemptions.weezl]] +version = "0.1.12" +criteria = "safe-to-deploy" + [[exemptions.winapi]] version = "0.3.9" criteria = "safe-to-run" @@ -2343,6 +2667,10 @@ criteria = "safe-to-deploy" version = "0.11.4" criteria = "safe-to-deploy" +[[exemptions.zip]] +version = "9.2.0" +criteria = "safe-to-deploy" + [[exemptions.zlib-rs]] version = "0.6.7" criteria = "safe-to-deploy" @@ -2350,3 +2678,15 @@ criteria = "safe-to-deploy" [[exemptions.zmij]] version = "1.0.23" criteria = "safe-to-deploy" + +[[exemptions.zopfli]] +version = "0.8.3" +criteria = "safe-to-deploy" + +[[exemptions.zune-core]] +version = "0.5.3" +criteria = "safe-to-deploy" + +[[exemptions.zune-jpeg]] +version = "0.5.15" +criteria = "safe-to-deploy" From faf18d47bec56bd110f36a583b5ae44a93e68820 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Sun, 1 Nov 2026 03:33:39 +0000 Subject: [PATCH 4/5] docs(security): port TM-NET-030 and TM-DOS-139 to the public threat model Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Wy7MByPXsW4jHxfLeTfGEc --- crates/bashkit/docs/threat-model.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/bashkit/docs/threat-model.md b/crates/bashkit/docs/threat-model.md index 3c1f13d91..8125adfdf 100644 --- a/crates/bashkit/docs/threat-model.md +++ b/crates/bashkit/docs/threat-model.md @@ -33,6 +33,7 @@ through configurable limits. | Output flooding (TM-DOS-002) | `yes \| head -n 1000000000` | Command limit stops loop | MITIGATED | | Width-derived allocation (TM-DOS-103) | `printf x \| od -w1152921504606846976` | Reject widths above 65,536 bytes; stack-pad one numeric field; budget output capacity before fallible growth | MITIGATED | | Pagination amplification (TM-DOS-137) | Long `pr -h` headers repeated on many short pages | Check the 16 MiB cap before every append; lease input, rows and output capacity before growth; borrow cells/page slices and retain date timestamps; charge input/work and poll cancellation/deadlines, including skipped pages | MITIGATED | +| Embedded CLI limit escalation (TM-DOS-139) | `markkit extract x --max-input-size 99999999999` | `markkit` limits can only be lowered below their defaults; `fetchkit` bodies capped by HTTP limits; `--ocr` refused | MITIGATED | | Variable explosion (TM-DOS-003) | `x=$(cat /dev/urandom)` | /dev/urandom returns bounded 8KB | MITIGATED | | Array growth (TM-DOS-004) | `arr+=(element)` in loop | Command limit | MITIGATED | @@ -411,6 +412,7 @@ Network access is disabled by default. When enabled, strict controls apply. | HTTP-transport SSRF via fail-open precheck (TM-NET-023) | Malformed/no-host URL or rebind window bypasses the IP filter | Precheck fails closed on bad URLs; transports receive pinned addresses + `is_private_ip` | MITIGATED | | Repeated curl data bypasses body cap (TM-NET-028) | Many data/file parts plus encoding expansion exceed 10 MB in aggregate | Checked aggregate appends; file metadata checked against remaining capacity before reads | MITIGATED | | Private IP behind NAT64/6to4 or reserved range (TM-NET-029) | AAAA `64:ff9b::a9fe:a9fe` / `2002:7f00:1::`, or A in `0.0.0.0/8`, `198.18.0.0/15`, multicast | `is_private_ip` checks the embedded IPv4 of NAT64/6to4 and blocks all non-global special-purpose ranges | MITIGATED | +| Embedded CLI egress bypass (TM-NET-030) | `fetchkit fetch`/`markkit extract URL` follow a redirect off the allowlist or read host proxy env | Both builtins send every hop through the sandbox HTTP client (allowlist re-checked per hop); no own TLS stack; `--allow-env-proxy` refused | MITIGATED | **Credential Injection (TM-NET-024–027):** From 541653b5f10da5bca008986c7215c1082c2956ce Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Sun, 1 Nov 2026 05:06:39 +0000 Subject: [PATCH 5/5] docs(builtins): add markkit/fetchkit example and rustdoc guide Self-checking example (VFS file, stdin + jq, URL with redirect, fetchkit Markdown and JSON, allowlist denial, refused flag) over an in-process HttpTransport, run in the CI Examples job; LIVE_URL adds a real fetch. The user guide moves to crates/bashkit/docs and is embedded as markkit_fetchkit_guide with a compiled doctest. --- .github/workflows/ci.yml | 2 + crates/bashkit/Cargo.toml | 4 + .../bashkit/docs}/markkit-fetchkit.md | 31 +++- crates/bashkit/examples/markkit_fetchkit.rs | 144 ++++++++++++++++++ crates/bashkit/src/lib.rs | 16 ++ .../runtimes/markkit-fetchkit-builtins.md | 6 + site/src/pages/docs/_meta.ts | 3 +- 7 files changed, 199 insertions(+), 7 deletions(-) rename {docs => crates/bashkit/docs}/markkit-fetchkit.md (71%) create mode 100644 crates/bashkit/examples/markkit_fetchkit.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a67d953e..a7136ddef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -385,6 +385,7 @@ jobs: cargo test -p bashkit --features markkit,fetchkit --lib -- builtins::markkit:: builtins::fetchkit:: cargo test -p bashkit --features markkit,fetchkit --test integration -- markkit_builtin_tests fetchkit_builtin_tests cargo test -p bashkit --features markkit --test integration -- markkit_builtin_tests + cargo test -p bashkit --features markkit,fetchkit --doc -- markkit_fetchkit_guide - name: Run fail-point tests (single-threaded) run: cargo test --features failpoints --test security_failpoint_tests -- --test-threads=1 @@ -534,6 +535,7 @@ jobs: cargo run --example realfs_readwrite --features realfs cargo run --example sqlite_basic --features sqlite cargo run --example sqlite_workflow --features sqlite + cargo run --example markkit_fetchkit --features markkit,fetchkit cargo run --example terminal_vi --features terminal cargo run --example terminal_record --features terminal diff --git a/crates/bashkit/Cargo.toml b/crates/bashkit/Cargo.toml index 2cf04b054..28116567e 100644 --- a/crates/bashkit/Cargo.toml +++ b/crates/bashkit/Cargo.toml @@ -435,6 +435,10 @@ required-features = ["sqlite"] name = "sqlite_workflow" required-features = ["sqlite"] +[[example]] +name = "markkit_fetchkit" +required-features = ["markkit", "fetchkit"] + # Additional tokio features needed only on native (not WASM) [target.'cfg(not(target_arch = "wasm32"))'.dependencies] tokio = { version = "1", features = ["rt-multi-thread", "fs"] } diff --git a/docs/markkit-fetchkit.md b/crates/bashkit/docs/markkit-fetchkit.md similarity index 71% rename from docs/markkit-fetchkit.md rename to crates/bashkit/docs/markkit-fetchkit.md index c4a90c0cc..61b0fc0f3 100644 --- a/docs/markkit-fetchkit.md +++ b/crates/bashkit/docs/markkit-fetchkit.md @@ -32,7 +32,7 @@ print: bashkit calls the same library code, so flags never drift. - **Files** come from the virtual filesystem (paths relative to the shell's working directory). The host filesystem is never read. - **Network** goes through the same default-deny allowlist as `curl` - ([Networking](networking.md)). Every redirect hop is checked again. + ([Networking](../../../docs/networking.md)). Every redirect hop is checked again. Credential injection, request signing and a custom `HttpTransport` apply. Without a `NetworkAllowlist`, URLs fail with `network access not configured`. - **Limits**: markkit's `--max-*` and `--timeout` flags can lower its limits but @@ -40,15 +40,34 @@ print: bashkit calls the same library code, so flags never drift. - **Environment**: fetchkit reads variables from the shell, never from the host process. -```rust +```rust,no_run use bashkit::{Bash, NetworkAllowlist}; +# #[tokio::main] +# async fn main() -> bashkit::Result<()> { let mut bash = Bash::builder() .network(NetworkAllowlist::new().allow("https://docs.rs")) .build(); let out = bash.exec("fetchkit fetch https://docs.rs/tokio").await?; +assert_eq!(out.exit_code, 0); +# Ok(()) +# } ``` +A runnable, self-checking example lives in +[`examples/markkit_fetchkit.rs`](../examples/markkit_fetchkit.rs). It serves pages from an +in-process `HttpTransport`, so it needs no network. Set `LIVE_URL` to also +fetch a real page: + +```bash +cargo run --example markkit_fetchkit --features markkit,fetchkit +LIVE_URL=https://example.com cargo run --example markkit_fetchkit --features markkit,fetchkit +``` + +Specialized fetchers follow the same rule. For example, fetchkit's arXiv +fetcher calls `export.arxiv.org`, so allow that origin too or the fetch fails +with `URL not in allowlist`. + ## Not available in the sandbox These flags are accepted by the parser and listed in `--help`, then fail with @@ -60,13 +79,13 @@ exit code 1 and `Error: is not available in bashkit`: | markkit | `--ocr` | needs a host Tesseract binary or model files | | fetchkit | `--allow-env-proxy` | host proxy settings | | fetchkit | `--render-rakers` | embedded JavaScript renderer | -| fetchkit | `--bot-auth-key`, `--bot-auth-agent` | bashkit already signs requests ([Request signing](request-signing.md)) | +| fetchkit | `--bot-auth-key`, `--bot-auth-agent` | bashkit already signs requests ([Request signing](../../../docs/request-signing.md)) | | fetchkit | `--allow-remote-conversion` | uploads documents to a third party | Both builtins are native only; the browser (wasm) package does not include them. ## See also -- [Networking & HTTP](networking.md) -- [Credential injection](../crates/bashkit/docs/credential-injection.md) -- [Security](security.md) +- [Networking & HTTP](../../../docs/networking.md) +- [Credential injection](credential-injection.md) +- [Security](../../../docs/security.md) diff --git a/crates/bashkit/examples/markkit_fetchkit.rs b/crates/bashkit/examples/markkit_fetchkit.rs new file mode 100644 index 000000000..deb4040d5 --- /dev/null +++ b/crates/bashkit/examples/markkit_fetchkit.rs @@ -0,0 +1,144 @@ +//! `markkit` and `fetchkit` builtins: the standalone CLIs inside the sandbox. +//! +//! Run with: +//! +//! ```bash +//! cargo run --example markkit_fetchkit --features markkit,fetchkit +//! # also fetch a real page through the network allowlist: +//! LIVE_URL=https://example.com cargo run --example markkit_fetchkit --features markkit,fetchkit +//! ``` +//! +//! The offline part serves a page from an in-process `HttpTransport`, so it is +//! deterministic in CI and still exercises the real request path (allowlist, +//! redirects, response limits). Each step asserts its result, so the example +//! fails loudly if the builtins regress. + +use std::sync::Arc; + +use async_trait::async_trait; +use bashkit::{ + Bash, HttpResponse, HttpTransport, HttpTransportError, HttpTransportRequest, NetworkAllowlist, +}; + +/// Stand-in for a web server: `/docs` redirects to `/docs/intro`, which serves HTML. +struct DocsSite; + +#[async_trait] +impl HttpTransport for DocsSite { + async fn execute( + &self, + request: HttpTransportRequest, + ) -> Result { + let html = |body: &str| HttpResponse { + status: 200, + headers: vec![("Content-Type".into(), "text/html; charset=utf-8".into())], + body: body.as_bytes().to_vec(), + }; + Ok(if request.url.ends_with("/docs") { + HttpResponse { + status: 302, + headers: vec![("Location".into(), "/docs/intro".into())], + body: vec![], + } + } else if request.url.ends_with("/docs/intro") { + html( + "Intro\ +

Getting started

\ +

Install with cargo add bashkit.

", + ) + } else { + HttpResponse { + status: 404, + headers: vec![("Content-Type".into(), "text/plain".into())], + body: b"not found".to_vec(), + } + }) + } +} + +fn step(title: &str, r: &bashkit::ExecResult) { + println!("--- {title} (exit {})", r.exit_code); + print!("{}", r.stdout); + if !r.stderr.is_empty() { + print!("stderr: {}", r.stderr); + } +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + const SITE: &str = "http://93.184.216.34"; + let mut bash = Bash::builder() + .network(NetworkAllowlist::new().allow(SITE)) + .http_transport(Arc::new(DocsSite)) + .build(); + + // 1. A file in the VFS -> Markdown. + let r = bash + .exec( + r#"mkdir -p /work && cd /work + printf '

Report

  • alpha
  • beta
' > report.html + markkit extract report.html --file"#, + ) + .await?; + step("markkit extract report.html --file", &r); + anyhow::ensure!(r.exit_code == 0 && r.stdout.contains("# Report")); + anyhow::ensure!(r.stdout.contains("- alpha")); + + // 2. Pipeline stdin, JSON output, piped into jq. + let r = bash + .exec(r#"echo '

Hello world

' | markkit extract - --json | jq -r .markdown"#) + .await?; + step("markkit extract - --json | jq", &r); + anyhow::ensure!(r.exit_code == 0 && r.stdout.contains("Hello **world**")); + + // 3. markkit on a URL: the redirect hop is re-checked against the allowlist. + let r = bash + .exec(&format!("markkit extract {SITE}/docs --main-content")) + .await?; + step("markkit extract URL --main-content", &r); + anyhow::ensure!(r.exit_code == 0 && r.stdout.contains("# Getting started")); + anyhow::ensure!(!r.stdout.contains("skip me")); + + // 4. fetchkit: Markdown with YAML frontmatter, then JSON status via jq. + let r = bash.exec(&format!("fetchkit fetch {SITE}/docs")).await?; + step("fetchkit fetch URL", &r); + anyhow::ensure!(r.exit_code == 0 && r.stdout.starts_with("---")); + anyhow::ensure!(r.stdout.contains("Getting started")); + + let r = bash + .exec(&format!( + "fetchkit fetch {SITE}/docs/intro -o json | jq .status_code" + )) + .await?; + step("fetchkit fetch -o json | jq .status_code", &r); + anyhow::ensure!(r.stdout.trim() == "200"); + + // 5. Off the allowlist: refused before any request leaves the sandbox. + let r = bash.exec("fetchkit fetch https://evil.example/").await?; + step("fetchkit fetch (not allowlisted)", &r); + anyhow::ensure!(r.exit_code != 0); + + // 6. Host-only flags parse but fail closed. + let r = bash + .exec(&format!("fetchkit fetch {SITE}/docs --render-rakers")) + .await?; + step("fetchkit --render-rakers", &r); + anyhow::ensure!(r.exit_code == 1 && r.stderr.contains("is not available in bashkit")); + + // Optional: a real page over the network, e.g. LIVE_URL=https://example.com + if let Ok(url) = std::env::var("LIVE_URL") { + let origin = url::Url::parse(&url)?.origin().ascii_serialization(); + let mut live = Bash::builder() + .network(NetworkAllowlist::new().allow(&origin)) + .build(); + let r = live.exec(&format!("fetchkit fetch {url}")).await?; + step("LIVE fetchkit fetch", &r); + anyhow::ensure!(r.exit_code == 0, "live fetchkit failed"); + let r = live.exec(&format!("markkit extract {url}")).await?; + step("LIVE markkit extract", &r); + anyhow::ensure!(r.exit_code == 0, "live markkit failed"); + } + + println!("ok"); + Ok(()) +} diff --git a/crates/bashkit/src/lib.rs b/crates/bashkit/src/lib.rs index 2a9511ac0..4fc971a22 100644 --- a/crates/bashkit/src/lib.rs +++ b/crates/bashkit/src/lib.rs @@ -4255,6 +4255,22 @@ pub mod cpython_guide {} #[doc = include_str!("../docs/sqlite.md")] pub mod sqlite_guide {} +/// Guide for the `markkit` and `fetchkit` builtins (features `markkit`, `fetchkit`). +/// +/// Topics covered: +/// - Same CLI as the standalone tools (`markkit extract`, `fetchkit fetch`) +/// - VFS files, pipeline stdin, and the network allowlist +/// - Limits that can only be lowered +/// - Flags refused in the sandbox (`mcp`, `--ocr`, `--render-rakers`, ...) +/// +/// **Related:** `BashBuilder::network`, `NetworkAllowlist`, [`threat_model`] +#[cfg(all( + any(feature = "markkit", feature = "fetchkit"), + not(target_arch = "wasm32") +))] +#[doc = include_str!("../docs/markkit-fetchkit.md")] +pub mod markkit_fetchkit_guide {} + /// Guide for embedded TypeScript execution via the ZapCode interpreter. /// /// This guide covers: diff --git a/knowledge/runtimes/markkit-fetchkit-builtins.md b/knowledge/runtimes/markkit-fetchkit-builtins.md index 4a7034b65..7496d50d9 100644 --- a/knowledge/runtimes/markkit-fetchkit-builtins.md +++ b/knowledge/runtimes/markkit-fetchkit-builtins.md @@ -112,6 +112,12 @@ startup cost: registration is a map insert, nothing is loaded until a call. errors, VFS + stdin, allowlist and redirect enforcement, refused flags, limit clamping, tenant isolation. - `no_leak_*` in `builtins/markkit.rs` and `builtins/fetchkit.rs` (TM-INF-022). +- `examples/markkit_fetchkit.rs`: self-checking smoke (VFS, stdin, URL with + redirect, JSON + jq, allowlist denial, refused flag) over an in-process + `HttpTransport`; run in the CI Examples job. `LIVE_URL=` adds a real + fetch (checked by hand on 2026-11-01 against example.com, docs.rs and a W3C + PDF). User guide: `crates/bashkit/docs/markkit-fetchkit.md` (rustdoc + `markkit_fetchkit_guide`, site page `/docs/markkit-fetchkit`). - Upstream: `crates/*-cli/tests/host.rs` in each repo (binary vs `run()` byte parity, host routing, `Send` futures). diff --git a/site/src/pages/docs/_meta.ts b/site/src/pages/docs/_meta.ts index 744227ddd..9a081c6a1 100644 --- a/site/src/pages/docs/_meta.ts +++ b/site/src/pages/docs/_meta.ts @@ -268,7 +268,8 @@ export const DOC_META: DocMeta[] = [ seoDescription: "Convert PDF, DOCX and HTML to Markdown and fetch URLs inside Bashkit with the exact markkit and fetchkit CLIs, sandboxed.", section: "Runtimes", - editPath: "docs/markkit-fetchkit.md", + collection: "rustdocs", + editPath: "crates/bashkit/docs/markkit-fetchkit.md", }, { slug: "ssh",