From 2aa8b40922b44b4e9ad56e47d76d538f2a3ab2c2 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Sun, 1 Nov 2026 02:36:20 +0000 Subject: [PATCH 1/3] feat(cli): embeddable CLI library with CliHost Move the markkit CLI into `markkit_cli::run(args, &impl CliHost)`; the binary is a thin wrapper. Hosts own stdio, file reads, URL extraction and MCP, and may refuse --ocr. URL support in NativeHost is the default native-http feature, so embedders link no reqwest. clap default features off to match bashkit. Lets bashkit ship a `markkit` builtin with the identical interface. --- CHANGELOG.md | 14 + Cargo.lock | 76 ---- Cargo.toml | 5 +- crates/markkit-cli/Cargo.toml | 13 +- crates/markkit-cli/src/lib.rs | 617 ++++++++++++++++++++++++++ crates/markkit-cli/src/main.rs | 395 +---------------- crates/markkit-cli/tests/host.rs | 273 ++++++++++++ knowledge/foundations/architecture.md | 25 +- knowledge/log.md | 1 + 9 files changed, 948 insertions(+), 471 deletions(-) create mode 100644 crates/markkit-cli/src/lib.rs create mode 100644 crates/markkit-cli/tests/host.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 00b22be..3d34e29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added + +- `markkit_cli` library: `run(args, &impl CliHost)` runs the exact `markkit` CLI + inside another program (used by bashkit's `markkit` builtin), with stdio, file + reads, URL fetching and MCP supplied by the host. + +### Changed + +- **Breaking:** `markkit-cli` URL sources sit behind the new default + `native-http` feature; builds with `default-features = false` must enable it. +- CLI help and errors are plain text (clap color and suggestions off), and error + output no longer appends a backtrace when `RUST_BACKTRACE` is set. +- A closed stdout pipe now ends `markkit extract` with exit 0 instead of a panic. + ## [0.2.1] - 2026-11-01 ### Fixed diff --git a/Cargo.lock b/Cargo.lock index 2f253b2..3d65f46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,56 +47,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" -[[package]] -name = "anstream" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - [[package]] name = "anstyle" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" -[[package]] -name = "anstyle-parse" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys 0.61.2", -] - [[package]] name = "anyhow" version = "1.0.104" @@ -397,10 +353,8 @@ version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ - "anstream", "anstyle", "clap_lex", - "strsim", ] [[package]] @@ -448,12 +402,6 @@ dependencies = [ "bytemuck", ] -[[package]] -name = "colorchoice" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" - [[package]] name = "combine" version = "4.6.9" @@ -1346,12 +1294,6 @@ version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" -[[package]] -name = "is_terminal_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" - [[package]] name = "itertools" version = "0.13.0" @@ -1834,12 +1776,6 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - [[package]] name = "oorandom" version = "11.1.5" @@ -3017,12 +2953,6 @@ dependencies = [ "quote", ] -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - [[package]] name = "subtle" version = "2.6.1" @@ -3360,12 +3290,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - [[package]] name = "uuid" version = "1.29.0" diff --git a/Cargo.toml b/Cargo.toml index 39e2072..be7689a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,10 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" # CLI -clap = { version = "4", features = ["derive"] } +# Default features off (no color, no did-you-mean suggestions) to match +# bashkit's clap set: bashkit embeds this CLI as a builtin and clap features +# unify, so color/suggestions here would change output on both sides. +clap = { version = "4", default-features = false, features = ["derive", "std", "help", "usage", "error-context"] } # HTTP reqwest = { version = "0.13", default-features = false, features = ["rustls", "json"] } diff --git a/crates/markkit-cli/Cargo.toml b/crates/markkit-cli/Cargo.toml index a9d8134..eb1c063 100644 --- a/crates/markkit-cli/Cargo.toml +++ b/crates/markkit-cli/Cargo.toml @@ -13,13 +13,20 @@ categories.workspace = true description = "Command line interface for Markkit markdown extraction" readme = "../../README.md" +[lib] +name = "markkit_cli" +path = "src/lib.rs" + [[bin]] name = "markkit" path = "src/main.rs" doc = false [features] -default = ["all_formats", "pdf-ocr"] +default = ["all_formats", "pdf-ocr", "native-http"] +# URL sources in `NativeHost` via markkit's own reqwest client. Embedders that +# fetch through their own egress (bashkit) leave this off: no reqwest, no TLS. +native-http = ["markkit/http_client"] all_formats = ["markkit/all_formats"] pdf = ["markkit/pdf"] pdf-ocr = ["pdf", "markkit/pdf-ocr"] @@ -33,8 +40,8 @@ odt = ["markkit/odt"] epub = ["markkit/epub"] [dependencies] -markkit = { path = "../markkit", version = "0.2.1", features = ["http_client", "tool"] } -tokio = { workspace = true, features = ["macros", "net", "rt", "rt-multi-thread", "time", "fs", "io-std", "io-util"] } +markkit = { path = "../markkit", version = "0.2.1", features = ["tool"] } +tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "io-std", "io-util"] } clap.workspace = true anyhow.workspace = true serde_json.workspace = true diff --git a/crates/markkit-cli/src/lib.rs b/crates/markkit-cli/src/lib.rs new file mode 100644 index 0000000..3f66f30 --- /dev/null +++ b/crates/markkit-cli/src/lib.rs @@ -0,0 +1,617 @@ +//! Markkit CLI as a library. +//! +//! The `markkit` binary is a thin wrapper around [`run`]. Embedders (e.g. the +//! bashkit `markkit` builtin) call the same [`run`] with their own [`CliHost`], +//! so flags, help, version, errors, output and exit codes are the same code +//! everywhere. + +// Decisions: +// - Every process-level effect goes through `CliHost`: stdout/stderr, stdin, +// file reads (`--file`, `--rules`), URL fetching, and MCP stdio. `run` never +// exits the process, prints directly, or touches the host filesystem. +// - URL extraction is a host method that receives the configured `Markkit`. +// `NativeHost` calls `Markkit::extract_url_document` (markkit's own SSRF +// policy, feature `native-http`). A sandbox fetches the bytes through its own +// egress and calls `extract_document` with `base_url` set to the final URL. +// - Flags a host cannot honor stay in the parser (identical help) and are +// refused at run time via `CliHost::refuse`, exit 1, one-line error. +// - Errors print in anyhow's `Error: {err:?}` layout (what `main() -> Result` +// printed before), minus the env-dependent backtrace. clap errors keep `clap::Error::exit` codes: 0 for +// --help/--version, 2 for usage errors. +// - The trait is generic (`run`), not `dyn`, so it needs no +// async-trait dependency. + +pub mod mcp; + +use anyhow::Result; +use clap::{Args, Parser, Subcommand}; +use markkit::{Extraction, Limits, Markkit}; +use std::ffi::OsString; +use std::future::Future; +use std::io; +use std::path::Path; +use std::time::Duration; + +/// Optional behavior a host may refuse. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum HostFeature { + /// `--ocr`: OCR runs a host Tesseract binary or loads host model files. + Ocr, +} + +impl HostFeature { + /// The command line flag that requested this feature. + pub fn flag(self) -> &'static str { + match self { + HostFeature::Ocr => "--ocr", + } + } +} + +/// Environment the CLI runs in. +/// +/// [`NativeHost`] maps this to the real process. Sandboxed hosts map it to +/// their own buffers, virtual filesystem, and egress policy. +pub trait CliHost: Send + Sync { + /// Write to standard output. + fn write_stdout(&self, text: &str) -> io::Result<()>; + + /// Write to standard error. + fn write_stderr(&self, text: &str); + + /// Read all of standard input. + fn read_stdin(&self) -> io::Result>; + + /// Read a whole file (`--file` sources and `--rules`). + fn read_file(&self, path: &Path) -> io::Result>; + + /// Fetch `url` and extract it with `markkit`'s configuration. Relative + /// links must resolve against the final URL after redirects. + fn extract_url( + &self, + markkit: &Markkit, + url: &str, + ) -> impl Future> + Send; + + /// Return `Some(reason)` to refuse a feature requested by a flag. + fn refuse(&self, feature: HostFeature) -> Option { + let _ = feature; + None + } + + /// Serve the tool over MCP. Hosts without a process stdio refuse. + fn serve_mcp( + &self, + tool: markkit::tool::MarkkitTool, + max_message: usize, + ) -> impl Future> + Send { + let _ = (tool, max_message); + async { Err(anyhow::anyhow!("mcp is not available in this environment")) } + } +} + +/// [`CliHost`] for the standalone `markkit` binary: real stdio and filesystem. +#[derive(Debug, Default, Clone, Copy)] +pub struct NativeHost; + +impl CliHost for NativeHost { + fn write_stdout(&self, text: &str) -> io::Result<()> { + use std::io::Write; + let mut out = io::stdout().lock(); + out.write_all(text.as_bytes())?; + out.flush() + } + + fn write_stderr(&self, text: &str) { + use std::io::Write; + let _ = io::stderr().lock().write_all(text.as_bytes()); + } + + fn read_stdin(&self) -> io::Result> { + use std::io::Read; + let mut buf = Vec::new(); + io::stdin().read_to_end(&mut buf)?; + Ok(buf) + } + + fn read_file(&self, path: &Path) -> io::Result> { + std::fs::read(path) + } + + #[allow(clippy::manual_async_fn)] + fn extract_url( + &self, + markkit: &Markkit, + url: &str, + ) -> impl Future> + Send { + async move { + #[cfg(feature = "native-http")] + { + markkit.extract_url_document(url).await + } + #[cfg(not(feature = "native-http"))] + { + let _ = (markkit, url); + Err(markkit::Error::UnsupportedFormat( + "URL sources require the native-http feature".to_string(), + )) + } + } + } + + #[allow(clippy::manual_async_fn)] + fn serve_mcp( + &self, + tool: markkit::tool::MarkkitTool, + max_message: usize, + ) -> impl Future> + Send { + async move { + let stdin = tokio::io::BufReader::new(tokio::io::stdin()); + mcp::serve(&tool, stdin, tokio::io::stdout(), max_message).await?; + Ok(()) + } + } +} + +/// The clap command tree, for help/argument parity checks in embedders. +pub fn command() -> clap::Command { + ::command() +} + +/// Parse `args` (including the program name) and run the command. +/// +/// Returns the process exit code. +pub async fn run(args: I, host: &H) -> i32 +where + H: CliHost, + I: IntoIterator, + T: Into + Clone, +{ + let cli = match Cli::try_parse_from(args) { + Ok(cli) => cli, + Err(e) => { + let text = e.render().to_string(); + if e.use_stderr() { + host.write_stderr(&text); + } else if let Err(err) = host.write_stdout(&text) + && err.kind() != io::ErrorKind::BrokenPipe + { + host.write_stderr(&format!("Error: {err}\n")); + return 1; + } + return e.exit_code(); + } + }; + match execute(cli, host).await { + Ok(()) => 0, + Err(e) => { + host.write_stderr(&format!("Error: {}\n", render_error(&e))); + 1 + } + } +} + +/// anyhow's `{:?}` layout (message, then a "Caused by:" list) without the +/// backtrace that `{:?}` appends under `RUST_BACKTRACE`: output must not +/// depend on the environment, and embedders must not leak internals. +fn render_error(e: &anyhow::Error) -> String { + let mut out = e.to_string(); + let causes: Vec = e.chain().skip(1).map(|c| c.to_string()).collect(); + match causes.as_slice() { + [] => {} + [one] => out.push_str(&format!("\n\nCaused by:\n {one}")), + many => { + out.push_str("\n\nCaused by:"); + for (i, cause) in many.iter().enumerate() { + out.push_str(&format!("\n {i}: {cause}")); + } + } + } + out +} + +#[derive(Parser)] +#[command( + name = "markkit", + about = "Extract clean markdown from any format", + version +)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Extract markdown from a URL or file + Extract { + /// URL or file path to extract from (use "-" to read from stdin) + source: Option, + + /// Treat source as a file path instead of URL + #[arg(long)] + file: bool, + + /// Read HTML from stdin + #[arg(long)] + stdin: bool, + + /// CSS selector to include (can be repeated) + #[arg(long = "include", short = 'i')] + include_selectors: Vec, + + /// CSS selector to exclude (can be repeated) + #[arg(long = "exclude", short = 'e')] + exclude_selectors: Vec, + + /// JSON file with HTML element rules, e.g. + /// `[{"selector": ".note", "action": "block_quote", "title": "Note"}]` + #[arg(long)] + rules: Option, + + /// Output as JSON with metadata + #[arg(long)] + json: bool, + + /// Include PDF page dimensions and positioned text runs in JSON output + #[arg(long, requires = "json")] + layout: bool, + + /// OCR textless PDF pages using the selected local backend + #[arg(long)] + ocr: bool, + /// Installed Tesseract language codes, e.g. eng+deu + #[arg(long, requires = "ocr")] + ocr_language: Option, + /// OCR implementation: tesseract (default) or ocrs (optional feature) + #[arg(long, requires = "ocr", value_parser = ["tesseract", "ocrs"])] + ocr_backend: Option, + /// Local ONNX detection model for the ocrs backend + #[arg(long, requires = "ocr")] + ocr_detection_model: Option, + /// Local ONNX recognition model for the ocrs backend + #[arg(long, requires = "ocr")] + ocr_recognition_model: Option, + /// Suppress headers/footers repeated in PDF page margins + #[arg(long)] + remove_pdf_headers: bool, + + /// Resolve relative links and images in files / stdin against this URL + /// (URLs use the final response URL) + #[arg(long)] + base_url: Option, + + /// Keep only the detected main content (article body) of HTML pages + #[arg(long)] + main_content: bool, + + #[command(flatten)] + limits: LimitArgs, + }, + /// Serve the `markkit` tool over MCP (JSON-RPC on stdin/stdout) + Mcp { + /// Do not offer the `url` source (no network access) + #[arg(long)] + no_url: bool, + + /// Maximum markdown characters returned per call + #[arg(long, default_value_t = markkit::tool::DEFAULT_MAX_OUTPUT_CHARS)] + max_output_chars: usize, + + /// Keep only the detected main content (article body) of HTML pages + #[arg(long)] + main_content: bool, + + /// Locale for tool descriptions and errors (en-US, uk-UA) + #[arg(long, default_value = "en-US")] + locale: String, + + #[command(flatten)] + limits: LimitArgs, + }, +} + +/// Resource limits; unset flags keep the library defaults (see `markkit::Limits`). +#[derive(Args, Debug, Default)] +#[command(next_help_heading = "Limits")] +struct LimitArgs { + /// Maximum input (and fetched body) size in bytes [default: 50 MiB] + #[arg(long, alias = "max-size", value_name = "BYTES")] + max_input_size: Option, + /// Maximum Markdown output size in bytes [default: 128 MiB] + #[arg(long, value_name = "BYTES")] + max_output_size: Option, + /// Time budget for fetch + parse + render, in seconds [default: 60] + #[arg(long, value_name = "SECS", value_parser = parse_secs)] + timeout: Option, + /// Maximum element nesting, hard-capped at 512 [default: 256] + #[arg(long, value_name = "N")] + max_nesting_depth: Option, + /// Maximum parsed nodes / retained PDF layout pages and runs [default: 5000000] + #[arg(long, value_name = "N")] + max_nodes: Option, + /// Maximum entries in a ZIP container [default: 10000] + #[arg(long, value_name = "N")] + max_archive_entries: Option, + /// Maximum decompressed bytes per ZIP entry [default: 64 MiB] + #[arg(long, value_name = "BYTES")] + max_archive_entry_size: Option, + /// Maximum decompressed bytes per ZIP container [default: 256 MiB] + #[arg(long, value_name = "BYTES")] + max_archive_total_size: Option, + /// Maximum data rows rendered per sheet [default: 10000] + #[arg(long, value_name = "N")] + max_sheet_rows: Option, + /// Maximum columns rendered per sheet [default: 256] + #[arg(long, value_name = "N")] + max_sheet_cols: Option, + /// Maximum XLS/ODS cell grid [default: 4000000] + #[arg(long, value_name = "N")] + max_spreadsheet_cells: Option, + /// Maximum redirects followed when fetching a URL [default: 5] + #[arg(long, value_name = "N")] + max_redirects: Option, + /// Maximum PDF pages rendered [default: 10000] + #[arg(long, value_name = "N")] + max_pdf_pages: Option, + /// Maximum decoded size of one PDF stream, bytes [default: 67108864] + #[arg(long, value_name = "BYTES")] + max_pdf_stream_size: Option, + /// Maximum total decoded PDF content, bytes [default: 268435456] + #[arg(long, value_name = "BYTES")] + max_pdf_decoded_size: Option, +} + +impl LimitArgs { + fn apply(&self, mut l: Limits) -> Limits { + macro_rules! set { + ($($f:ident),*) => {$( if let Some(v) = self.$f { l.$f = v; } )*}; + } + set!( + max_input_size, + max_output_size, + timeout, + max_nesting_depth, + max_nodes, + max_archive_entries, + max_archive_entry_size, + max_archive_total_size, + max_sheet_rows, + max_sheet_cols, + max_spreadsheet_cells, + max_redirects, + max_pdf_pages, + max_pdf_stream_size, + max_pdf_decoded_size + ); + l + } +} + +fn parse_secs(s: &str) -> Result { + let secs: f64 = s.parse().map_err(|e| format!("{e}"))?; + Duration::try_from_secs_f64(secs).map_err(|e| format!("{e}")) +} + +async fn execute(cli: Cli, host: &H) -> Result<()> { + match cli.command { + Commands::Mcp { + no_url, + max_output_chars, + main_content, + locale, + limits, + } => { + let limits = limits.apply(Limits::default()); + let max_message = mcp::max_message_bytes(limits.max_input_size); + let tool = markkit::tool::MarkkitTool::builder() + .locale(&locale) + .config(Markkit::builder().limits(limits).main_content(main_content)) + .allow_url(!no_url) + .max_output_chars(max_output_chars) + .build(); + host.serve_mcp(tool, max_message).await?; + } + Commands::Extract { + source, + file, + stdin, + include_selectors, + exclude_selectors, + limits, + rules, + json: output_json, + layout, + ocr, + ocr_language, + ocr_backend, + ocr_detection_model, + ocr_recognition_model, + remove_pdf_headers, + base_url, + main_content, + } => { + if ocr && let Some(reason) = host.refuse(HostFeature::Ocr) { + anyhow::bail!("{} {reason}", HostFeature::Ocr.flag()); + } + let read_stdin = stdin || source.as_deref() == Some("-"); + + if !read_stdin && source.is_none() { + anyhow::bail!("source is required unless --stdin or \"-\" is used"); + } + + let mut builder = Markkit::builder() + .limits(limits.apply(Limits::default())) + .pdf_layout(layout) + .pdf_ocr(ocr) + .pdf_ocr_language(ocr_language.unwrap_or_else(|| "eng".into())) + .pdf_remove_repeated_headers(remove_pdf_headers) + .main_content(main_content); + + if let Some(backend) = ocr_backend { + builder = builder.pdf_ocr_backend(backend.parse().map_err(anyhow::Error::msg)?); + } + if let Some(path) = ocr_detection_model { + builder = builder.pdf_ocr_detection_model(path); + } + if let Some(path) = ocr_recognition_model { + builder = builder.pdf_ocr_recognition_model(path); + } + + for sel in &include_selectors { + builder = builder.include_selector(sel); + } + for sel in &exclude_selectors { + builder = builder.exclude_selector(sel); + } + + if let Some(base) = base_url { + builder = builder.base_url(base); + } + + if let Some(path) = &rules { + let text = String::from_utf8(host.read_file(path)?).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "stream did not contain valid UTF-8", + ) + })?; + builder.html_rules = serde_json::from_str(&text) + .map_err(|e| anyhow::anyhow!("invalid rules file {}: {e}", path.display()))?; + } + + let markkit = builder.build(); + + let source_label = source.clone().unwrap_or_else(|| "".to_string()); + + let extraction = if read_stdin { + let buf = host.read_stdin()?; + markkit + .extract_document_sync(&buf, markkit::DocumentHint::content_type("text/html"))? + } else if file { + let src = source.as_deref().unwrap(); + let bytes = host.read_file(Path::new(src))?; + // Format detected from the extension, then magic bytes. + markkit.extract_document_sync(&bytes, markkit::DocumentHint::file_name(src))? + } else { + host.extract_url(&markkit, source.as_deref().unwrap()) + .await? + }; + + if output_json { + let mut output = serde_json::json!({ + "source": source_label, + "markdown": extraction.markdown, + "format": extraction.format, + "title": extraction.title, + "metadata": extraction.metadata, + "stats": extraction.stats, + "warnings": extraction.warnings, + }); + if let Some(layout) = extraction.layout { + output["layout"] = serde_json::to_value(layout)?; + } + write_out( + host, + &format!("{}\n", serde_json::to_string_pretty(&output)?), + )?; + } else { + write_out(host, &extraction.markdown)?; + } + } + } + + Ok(()) +} + +/// Write to stdout. A closed pipe ends the command quietly (exit 0). +fn write_out(host: &H, text: &str) -> Result<()> { + match host.write_stdout(text) { + Err(e) if e.kind() != io::ErrorKind::BrokenPipe => Err(e.into()), + _ => Ok(()), + } +} +#[cfg(test)] +mod tests { + use super::*; + use clap::CommandFactory; + + #[test] + fn render_error_matches_anyhow_layout_without_backtrace() { + let one = anyhow::anyhow!("inner").context("outer"); + assert_eq!(render_error(&one), "outer\n\nCaused by:\n inner"); + let two = anyhow::anyhow!("c").context("b").context("a"); + assert_eq!(render_error(&two), "a\n\nCaused by:\n 0: b\n 1: c"); + assert_eq!(render_error(&anyhow::anyhow!("only")), "only"); + } + + #[test] + fn verify_cli() { + Cli::command().debug_assert(); + } + + fn limits_from(args: &[&str]) -> Limits { + let cli = Cli::try_parse_from(["markkit", "extract", "x"].iter().chain(args)).unwrap(); + let Commands::Extract { limits, .. } = cli.command else { + panic!("expected extract"); + }; + limits.apply(Limits::default()) + } + + #[test] + fn unset_limit_flags_keep_defaults() { + assert_eq!(limits_from(&[]), Limits::default()); + } + + #[test] + fn every_limit_flag_maps() { + let l = limits_from(&[ + "--max-input-size=1", + "--max-output-size=2", + "--timeout=0.5", + "--max-nesting-depth=3", + "--max-nodes=4", + "--max-archive-entries=5", + "--max-archive-entry-size=6", + "--max-archive-total-size=7", + "--max-sheet-rows=8", + "--max-sheet-cols=9", + "--max-spreadsheet-cells=10", + "--max-redirects=11", + "--max-pdf-pages=12", + "--max-pdf-stream-size=13", + "--max-pdf-decoded-size=14", + ]); + let want = Limits::default() + .max_input_size(1) + .max_output_size(2) + .timeout(Duration::from_millis(500)) + .max_nesting_depth(3) + .max_nodes(4) + .max_archive_entries(5) + .max_archive_entry_size(6) + .max_archive_total_size(7) + .max_sheet_rows(8) + .max_sheet_cols(9) + .max_spreadsheet_cells(10) + .max_redirects(11) + .max_pdf_pages(12) + .max_pdf_stream_size(13) + .max_pdf_decoded_size(14); + assert_eq!(l, want); + } + + #[test] + fn legacy_max_size_alias() { + assert_eq!(limits_from(&["--max-size=9"]).max_input_size, 9); + } + + #[test] + fn bad_timeout_rejected() { + for bad in ["-1", "nan", "abc"] { + let arg = format!("--timeout={bad}"); + assert!(Cli::try_parse_from(["markkit", "extract", "x", &arg]).is_err()); + } + } +} diff --git a/crates/markkit-cli/src/main.rs b/crates/markkit-cli/src/main.rs index 3ec22ac..b3edf1f 100644 --- a/crates/markkit-cli/src/main.rs +++ b/crates/markkit-cli/src/main.rs @@ -1,393 +1,8 @@ -use std::io::Read; - -use anyhow::Result; -use std::time::Duration; - -use clap::{Args, Parser, Subcommand}; -use markkit::{Limits, Markkit}; - -mod mcp; - -#[derive(Parser)] -#[command( - name = "markkit", - about = "Extract clean markdown from any format", - version -)] -struct Cli { - #[command(subcommand)] - command: Commands, -} - -#[derive(Subcommand)] -enum Commands { - /// Extract markdown from a URL or file - Extract { - /// URL or file path to extract from (use "-" to read from stdin) - source: Option, - - /// Treat source as a file path instead of URL - #[arg(long)] - file: bool, - - /// Read HTML from stdin - #[arg(long)] - stdin: bool, - - /// CSS selector to include (can be repeated) - #[arg(long = "include", short = 'i')] - include_selectors: Vec, - - /// CSS selector to exclude (can be repeated) - #[arg(long = "exclude", short = 'e')] - exclude_selectors: Vec, - - /// JSON file with HTML element rules, e.g. - /// `[{"selector": ".note", "action": "block_quote", "title": "Note"}]` - #[arg(long)] - rules: Option, - - /// Output as JSON with metadata - #[arg(long)] - json: bool, - - /// Include PDF page dimensions and positioned text runs in JSON output - #[arg(long, requires = "json")] - layout: bool, - - /// OCR textless PDF pages using the selected local backend - #[arg(long)] - ocr: bool, - /// Installed Tesseract language codes, e.g. eng+deu - #[arg(long, requires = "ocr")] - ocr_language: Option, - /// OCR implementation: tesseract (default) or ocrs (optional feature) - #[arg(long, requires = "ocr", value_parser = ["tesseract", "ocrs"])] - ocr_backend: Option, - /// Local ONNX detection model for the ocrs backend - #[arg(long, requires = "ocr")] - ocr_detection_model: Option, - /// Local ONNX recognition model for the ocrs backend - #[arg(long, requires = "ocr")] - ocr_recognition_model: Option, - /// Suppress headers/footers repeated in PDF page margins - #[arg(long)] - remove_pdf_headers: bool, - - /// Resolve relative links and images in files / stdin against this URL - /// (URLs use the final response URL) - #[arg(long)] - base_url: Option, - - /// Keep only the detected main content (article body) of HTML pages - #[arg(long)] - main_content: bool, - - #[command(flatten)] - limits: LimitArgs, - }, - /// Serve the `markkit` tool over MCP (JSON-RPC on stdin/stdout) - Mcp { - /// Do not offer the `url` source (no network access) - #[arg(long)] - no_url: bool, - - /// Maximum markdown characters returned per call - #[arg(long, default_value_t = markkit::tool::DEFAULT_MAX_OUTPUT_CHARS)] - max_output_chars: usize, - - /// Keep only the detected main content (article body) of HTML pages - #[arg(long)] - main_content: bool, - - /// Locale for tool descriptions and errors (en-US, uk-UA) - #[arg(long, default_value = "en-US")] - locale: String, - - #[command(flatten)] - limits: LimitArgs, - }, -} - -/// Resource limits; unset flags keep the library defaults (see `markkit::Limits`). -#[derive(Args, Debug, Default)] -#[command(next_help_heading = "Limits")] -struct LimitArgs { - /// Maximum input (and fetched body) size in bytes [default: 50 MiB] - #[arg(long, alias = "max-size", value_name = "BYTES")] - max_input_size: Option, - /// Maximum Markdown output size in bytes [default: 128 MiB] - #[arg(long, value_name = "BYTES")] - max_output_size: Option, - /// Time budget for fetch + parse + render, in seconds [default: 60] - #[arg(long, value_name = "SECS", value_parser = parse_secs)] - timeout: Option, - /// Maximum element nesting, hard-capped at 512 [default: 256] - #[arg(long, value_name = "N")] - max_nesting_depth: Option, - /// Maximum parsed nodes / retained PDF layout pages and runs [default: 5000000] - #[arg(long, value_name = "N")] - max_nodes: Option, - /// Maximum entries in a ZIP container [default: 10000] - #[arg(long, value_name = "N")] - max_archive_entries: Option, - /// Maximum decompressed bytes per ZIP entry [default: 64 MiB] - #[arg(long, value_name = "BYTES")] - max_archive_entry_size: Option, - /// Maximum decompressed bytes per ZIP container [default: 256 MiB] - #[arg(long, value_name = "BYTES")] - max_archive_total_size: Option, - /// Maximum data rows rendered per sheet [default: 10000] - #[arg(long, value_name = "N")] - max_sheet_rows: Option, - /// Maximum columns rendered per sheet [default: 256] - #[arg(long, value_name = "N")] - max_sheet_cols: Option, - /// Maximum XLS/ODS cell grid [default: 4000000] - #[arg(long, value_name = "N")] - max_spreadsheet_cells: Option, - /// Maximum redirects followed when fetching a URL [default: 5] - #[arg(long, value_name = "N")] - max_redirects: Option, - /// Maximum PDF pages rendered [default: 10000] - #[arg(long, value_name = "N")] - max_pdf_pages: Option, - /// Maximum decoded size of one PDF stream, bytes [default: 67108864] - #[arg(long, value_name = "BYTES")] - max_pdf_stream_size: Option, - /// Maximum total decoded PDF content, bytes [default: 268435456] - #[arg(long, value_name = "BYTES")] - max_pdf_decoded_size: Option, -} - -impl LimitArgs { - fn apply(&self, mut l: Limits) -> Limits { - macro_rules! set { - ($($f:ident),*) => {$( if let Some(v) = self.$f { l.$f = v; } )*}; - } - set!( - max_input_size, - max_output_size, - timeout, - max_nesting_depth, - max_nodes, - max_archive_entries, - max_archive_entry_size, - max_archive_total_size, - max_sheet_rows, - max_sheet_cols, - max_spreadsheet_cells, - max_redirects, - max_pdf_pages, - max_pdf_stream_size, - max_pdf_decoded_size - ); - l - } -} - -fn parse_secs(s: &str) -> Result { - let secs: f64 = s.parse().map_err(|e| format!("{e}"))?; - Duration::try_from_secs_f64(secs).map_err(|e| format!("{e}")) -} +//! Markkit CLI binary. All logic lives in the library (`markkit_cli::run`) so +//! embedders such as bashkit expose the exact same interface. #[tokio::main] -async fn main() -> Result<()> { - let cli = Cli::parse(); - - match cli.command { - Commands::Mcp { - no_url, - max_output_chars, - main_content, - locale, - limits, - } => { - let limits = limits.apply(Limits::default()); - let max_message = mcp::max_message_bytes(limits.max_input_size); - let tool = markkit::tool::MarkkitTool::builder() - .locale(&locale) - .config(Markkit::builder().limits(limits).main_content(main_content)) - .allow_url(!no_url) - .max_output_chars(max_output_chars) - .build(); - let stdin = tokio::io::BufReader::new(tokio::io::stdin()); - mcp::serve(&tool, stdin, tokio::io::stdout(), max_message).await?; - } - Commands::Extract { - source, - file, - stdin, - include_selectors, - exclude_selectors, - limits, - rules, - json: output_json, - layout, - ocr, - ocr_language, - ocr_backend, - ocr_detection_model, - ocr_recognition_model, - remove_pdf_headers, - base_url, - main_content, - } => { - let read_stdin = stdin || source.as_deref() == Some("-"); - - if !read_stdin && source.is_none() { - anyhow::bail!("source is required unless --stdin or \"-\" is used"); - } - - let mut builder = Markkit::builder() - .limits(limits.apply(Limits::default())) - .pdf_layout(layout) - .pdf_ocr(ocr) - .pdf_ocr_language(ocr_language.unwrap_or_else(|| "eng".into())) - .pdf_remove_repeated_headers(remove_pdf_headers) - .main_content(main_content); - - if let Some(backend) = ocr_backend { - builder = builder.pdf_ocr_backend(backend.parse().map_err(anyhow::Error::msg)?); - } - if let Some(path) = ocr_detection_model { - builder = builder.pdf_ocr_detection_model(path); - } - if let Some(path) = ocr_recognition_model { - builder = builder.pdf_ocr_recognition_model(path); - } - - for sel in &include_selectors { - builder = builder.include_selector(sel); - } - for sel in &exclude_selectors { - builder = builder.exclude_selector(sel); - } - - if let Some(base) = base_url { - builder = builder.base_url(base); - } - - if let Some(path) = &rules { - let text = std::fs::read_to_string(path)?; - builder.html_rules = serde_json::from_str(&text) - .map_err(|e| anyhow::anyhow!("invalid rules file {}: {e}", path.display()))?; - } - - let markkit = builder.build(); - - let source_label = source.clone().unwrap_or_else(|| "".to_string()); - - let extraction = if read_stdin { - let mut buf = Vec::new(); - std::io::stdin().read_to_end(&mut buf)?; - markkit - .extract_document_sync(&buf, markkit::DocumentHint::content_type("text/html"))? - } else if file { - let src = source.as_deref().unwrap(); - let bytes = tokio::fs::read(src).await?; - // Format detected from the extension, then magic bytes. - markkit.extract_document_sync(&bytes, markkit::DocumentHint::file_name(src))? - } else { - markkit - .extract_url_document(source.as_deref().unwrap()) - .await? - }; - - if output_json { - let mut output = serde_json::json!({ - "source": source_label, - "markdown": extraction.markdown, - "format": extraction.format, - "title": extraction.title, - "metadata": extraction.metadata, - "stats": extraction.stats, - "warnings": extraction.warnings, - }); - if let Some(layout) = extraction.layout { - output["layout"] = serde_json::to_value(layout)?; - } - println!("{}", serde_json::to_string_pretty(&output)?); - } else { - print!("{}", extraction.markdown); - } - } - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use clap::CommandFactory; - - #[test] - fn verify_cli() { - Cli::command().debug_assert(); - } - - fn limits_from(args: &[&str]) -> Limits { - let cli = Cli::try_parse_from(["markkit", "extract", "x"].iter().chain(args)).unwrap(); - let Commands::Extract { limits, .. } = cli.command else { - panic!("expected extract"); - }; - limits.apply(Limits::default()) - } - - #[test] - fn unset_limit_flags_keep_defaults() { - assert_eq!(limits_from(&[]), Limits::default()); - } - - #[test] - fn every_limit_flag_maps() { - let l = limits_from(&[ - "--max-input-size=1", - "--max-output-size=2", - "--timeout=0.5", - "--max-nesting-depth=3", - "--max-nodes=4", - "--max-archive-entries=5", - "--max-archive-entry-size=6", - "--max-archive-total-size=7", - "--max-sheet-rows=8", - "--max-sheet-cols=9", - "--max-spreadsheet-cells=10", - "--max-redirects=11", - "--max-pdf-pages=12", - "--max-pdf-stream-size=13", - "--max-pdf-decoded-size=14", - ]); - let want = Limits::default() - .max_input_size(1) - .max_output_size(2) - .timeout(Duration::from_millis(500)) - .max_nesting_depth(3) - .max_nodes(4) - .max_archive_entries(5) - .max_archive_entry_size(6) - .max_archive_total_size(7) - .max_sheet_rows(8) - .max_sheet_cols(9) - .max_spreadsheet_cells(10) - .max_redirects(11) - .max_pdf_pages(12) - .max_pdf_stream_size(13) - .max_pdf_decoded_size(14); - assert_eq!(l, want); - } - - #[test] - fn legacy_max_size_alias() { - assert_eq!(limits_from(&["--max-size=9"]).max_input_size, 9); - } - - #[test] - fn bad_timeout_rejected() { - for bad in ["-1", "nan", "abc"] { - let arg = format!("--timeout={bad}"); - assert!(Cli::try_parse_from(["markkit", "extract", "x", &arg]).is_err()); - } - } +async fn main() { + let code = markkit_cli::run(std::env::args_os(), &markkit_cli::NativeHost).await; + std::process::exit(code); } diff --git a/crates/markkit-cli/tests/host.rs b/crates/markkit-cli/tests/host.rs new file mode 100644 index 0000000..edf8965 --- /dev/null +++ b/crates/markkit-cli/tests/host.rs @@ -0,0 +1,273 @@ +//! `markkit_cli::run` with a custom `CliHost`: the embedding contract. +//! +//! Proves an embedder (e.g. the bashkit `markkit` builtin) gets the same CLI as +//! the binary, with stdio, file reads, URL fetching and refused flags routed +//! through the host. No test here touches the network or the real filesystem +//! through the host. + +use markkit::{DocumentHint, Extraction, Markkit}; +use markkit_cli::{CliHost, HostFeature, run}; +use std::collections::HashMap; +use std::future::Future; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::Mutex; + +#[derive(Default)] +struct TestHost { + stdout: Mutex, + stderr: Mutex, + stdin: Vec, + files: HashMap>, + /// url => (final url, content type, body) + pages: HashMap)>, + fetched: Mutex>, + refused: Vec, + broken_pipe: bool, +} + +impl TestHost { + fn out(&self) -> String { + self.stdout.lock().unwrap().clone() + } + fn err(&self) -> String { + self.stderr.lock().unwrap().clone() + } +} + +impl CliHost for TestHost { + fn write_stdout(&self, text: &str) -> io::Result<()> { + if self.broken_pipe { + return Err(io::Error::from(io::ErrorKind::BrokenPipe)); + } + self.stdout.lock().unwrap().push_str(text); + Ok(()) + } + fn write_stderr(&self, text: &str) { + self.stderr.lock().unwrap().push_str(text); + } + fn read_stdin(&self) -> io::Result> { + Ok(self.stdin.clone()) + } + fn read_file(&self, path: &Path) -> io::Result> { + self.files + .get(path) + .cloned() + .ok_or_else(|| io::Error::from(io::ErrorKind::NotFound)) + } + fn extract_url( + &self, + markkit: &Markkit, + url: &str, + ) -> impl Future> + Send { + self.fetched.lock().unwrap().push(url.to_string()); + let page = self.pages.get(url).cloned(); + let mut config = markkit.config().clone(); + async move { + let Some((final_url, content_type, body)) = page else { + return Err(markkit::Error::UnsupportedFormat("no such page".into())); + }; + config.base_url = Some(final_url); + Markkit::new(config).extract_document_sync( + &body, + DocumentHint { + content_type: Some(&content_type), + file_name: None, + }, + ) + } + } + fn refuse(&self, feature: HostFeature) -> Option { + self.refused + .contains(&feature) + .then(|| "is not available in this environment".to_string()) + } +} + +async fn run_with(host: &TestHost, args: &[&str]) -> i32 { + run(std::iter::once("markkit").chain(args.iter().copied()), host).await +} + +fn binary(args: &[&str]) -> std::process::Output { + Command::new(env!("CARGO_BIN_EXE_markkit")) + .args(args) + .output() + .expect("run markkit binary") +} + +const PAGE: &[u8] = b"T

Hello

next

"; + +#[tokio::test] +async fn help_and_version_match_binary_byte_for_byte() { + for args in [ + &["--help"][..], + &["extract", "--help"], + &["mcp", "--help"], + &["--version"], + ] { + let host = TestHost::default(); + assert_eq!(run_with(&host, args).await, 0, "{args:?}"); + let bin = binary(args); + assert_eq!(bin.status.code(), Some(0), "{args:?}"); + assert_eq!(host.out(), String::from_utf8_lossy(&bin.stdout), "{args:?}"); + assert!(host.err().is_empty()); + } +} + +#[tokio::test] +async fn usage_errors_match_binary_and_exit_2() { + for args in [ + &["extract", "--no-such-flag", "x"][..], + &[], + &["extract", "--layout", "x"], + ] { + let host = TestHost::default(); + assert_eq!(run_with(&host, args).await, 2, "{args:?}"); + let bin = binary(args); + assert_eq!(bin.status.code(), Some(2), "{args:?}"); + assert_eq!(host.err(), String::from_utf8_lossy(&bin.stderr), "{args:?}"); + } +} + +#[tokio::test] +async fn runtime_error_matches_binary_and_exits_1() { + let host = TestHost::default(); + assert_eq!(run_with(&host, &["extract"]).await, 1); + let bin = binary(&["extract"]); + assert_eq!(bin.status.code(), Some(1)); + assert_eq!(host.err(), String::from_utf8_lossy(&bin.stderr)); +} + +#[tokio::test] +async fn stdin_goes_through_host() { + for args in [&["extract", "--stdin"][..], &["extract", "-"]] { + let host = TestHost { + stdin: PAGE.to_vec(), + ..Default::default() + }; + assert_eq!(run_with(&host, args).await, 0, "{}", host.err()); + assert!(host.out().contains("# Hello"), "{}", host.out()); + } +} + +#[tokio::test] +async fn file_and_rules_read_through_host() { + let mut files = HashMap::new(); + files.insert(PathBuf::from("/vfs/doc.html"), PAGE.to_vec()); + files.insert( + PathBuf::from("/vfs/rules.json"), + br#"[{"selector": "h1", "action": "skip"}]"#.to_vec(), + ); + let host = TestHost { + files, + ..Default::default() + }; + let code = run_with( + &host, + &[ + "extract", + "--file", + "/vfs/doc.html", + "--rules", + "/vfs/rules.json", + ], + ) + .await; + assert_eq!(code, 0, "{}", host.err()); + assert!(!host.out().contains("Hello"), "{}", host.out()); + assert!(host.out().contains("next"), "{}", host.out()); +} + +#[tokio::test] +async fn missing_file_is_error_exit_1() { + let host = TestHost::default(); + assert_eq!( + run_with(&host, &["extract", "--file", "/nope.html"]).await, + 1 + ); + assert!(host.err().starts_with("Error: "), "{}", host.err()); + assert!(host.out().is_empty()); +} + +#[tokio::test] +async fn invalid_utf8_rules_is_error() { + let mut files = HashMap::new(); + files.insert(PathBuf::from("/r.json"), vec![0xff, 0xfe]); + let host = TestHost { + files, + stdin: PAGE.to_vec(), + ..Default::default() + }; + assert_eq!( + run_with(&host, &["extract", "-", "--rules", "/r.json"]).await, + 1 + ); + assert!(host.err().contains("valid UTF-8"), "{}", host.err()); +} + +#[tokio::test] +async fn url_goes_through_host_and_resolves_against_final_url() { + let mut pages = HashMap::new(); + pages.insert( + "https://a.test/start".to_string(), + ( + "https://b.test/final/".to_string(), + "text/html".to_string(), + PAGE.to_vec(), + ), + ); + let host = TestHost { + pages, + ..Default::default() + }; + assert_eq!( + run_with(&host, &["extract", "https://a.test/start", "--json"]).await, + 0, + "{}", + host.err() + ); + let json: serde_json::Value = serde_json::from_str(&host.out()).unwrap(); + assert_eq!(json["source"], "https://a.test/start"); + let md = json["markdown"].as_str().unwrap(); + assert!(md.contains("https://b.test/next"), "{md}"); + assert_eq!(*host.fetched.lock().unwrap(), ["https://a.test/start"]); +} + +#[tokio::test] +async fn refused_ocr_fails_before_reading_input() { + let host = TestHost { + refused: vec![HostFeature::Ocr], + ..Default::default() + }; + assert_eq!( + run_with(&host, &["extract", "https://a.test/x.pdf", "--ocr"]).await, + 1 + ); + assert_eq!( + host.err(), + "Error: --ocr is not available in this environment\n" + ); + assert!(host.fetched.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn mcp_is_refused_by_default_host() { + let host = TestHost::default(); + assert_eq!(run_with(&host, &["mcp"]).await, 1); + assert_eq!( + host.err(), + "Error: mcp is not available in this environment\n" + ); +} + +#[tokio::test] +async fn broken_pipe_on_stdout_exits_0() { + let host = TestHost { + stdin: PAGE.to_vec(), + broken_pipe: true, + ..Default::default() + }; + assert_eq!(run_with(&host, &["extract", "-"]).await, 0); + assert!(host.err().is_empty()); +} diff --git a/knowledge/foundations/architecture.md b/knowledge/foundations/architecture.md index 8b9a4d3..94486d2 100644 --- a/knowledge/foundations/architecture.md +++ b/knowledge/foundations/architecture.md @@ -52,12 +52,35 @@ are trusted embedder code; the first error aborts and is returned as is | Crate | Purpose | |-------|---------| | `markkit` | Core library – parsing, conversion, API | -| `markkit-cli` | Command-line tool | +| `markkit-cli` | Command-line tool; also the `markkit_cli` library (see below) | | `markkit-bench` | Benchmark harness, results committed in `results/` | | `markkit-eval` | Extraction-quality eval study (mira) | | `markkit-python` | Python bindings (PyO3) | | `markkit-js` | Node.js bindings (NAPI-RS) | +### Embeddable CLI + +The CLI is a library: `markkit_cli::run(args, &impl CliHost) -> i32`. The +`markkit` binary is a thin wrapper with `NativeHost`; bashkit's `markkit` +builtin calls the same `run`, so flags, help, version, errors, output and exit +codes are one implementation. + +- `CliHost` owns stdout/stderr, stdin, file reads (`--file`, `--rules`), URL + extraction and MCP stdio (`serve_mcp`, refused by default). `run` never exits + the process or touches the host filesystem. +- URL extraction receives the configured `Markkit`. `NativeHost` uses + `extract_url_document` (feature `native-http`, default; markkit's SSRF + policy). Sandboxes fetch through their own egress, set `base_url` to the final + URL and call `extract_document`; built without `native-http` the CLI links no + reqwest/TLS. +- Hosts may refuse `--ocr` (`HostFeature::Ocr`: host Tesseract binary, host + model files). Refused flags still parse and show in help; exit 1. +- Exit codes: 0 success / help / version / closed stdout pipe; 1 runtime error; + 2 usage error. Errors use anyhow's `Error: {:?}` layout without the + `RUST_BACKTRACE`-dependent backtrace. +- clap is built with default features off (no color, no suggestions), matching + bashkit's set so feature unification does not change either side's output. + ## Configuration Via `MarkkitBuilder`: diff --git a/knowledge/log.md b/knowledge/log.md index 9f9693f..84557df 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-11-01 +* **Embeddable CLI**: the CLI moved into the `markkit_cli` library (`run(args, &impl CliHost)`) so bashkit can ship a `markkit` builtin with the identical interface. Hosts own stdio, files, URL fetching and MCP, and may refuse `--ocr`. URL support in the native host is the default `native-http` feature; clap default features off; error output no longer includes `RUST_BACKTRACE` backtraces. See [Architecture](foundations/architecture.md). * **Release 0.2.1**: patch release carrying the portable manylinux2014 PyPI wheel build. No library or CLI behavior changes; crates.io, PyPI and native npm publish from the same verified tag. See [Release Process](operations/release-process.md) and `CHANGELOG.md`. ## 2026-10-31 From 2099742ee3165638e00929540a3d4d8ebc708432 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Sun, 1 Nov 2026 02:38:55 +0000 Subject: [PATCH 2/3] feat(cli): async host file/stdin reads, Send check Hosts with an async virtual filesystem (bashkit) must not block. --- crates/markkit-cli/src/lib.rs | 32 +++++++++++++++++++------------- crates/markkit-cli/tests/host.rs | 22 +++++++++++++++------- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/crates/markkit-cli/src/lib.rs b/crates/markkit-cli/src/lib.rs index 3f66f30..d1e3aff 100644 --- a/crates/markkit-cli/src/lib.rs +++ b/crates/markkit-cli/src/lib.rs @@ -61,10 +61,11 @@ pub trait CliHost: Send + Sync { fn write_stderr(&self, text: &str); /// Read all of standard input. - fn read_stdin(&self) -> io::Result>; + fn read_stdin(&self) -> impl Future>> + Send; - /// Read a whole file (`--file` sources and `--rules`). - fn read_file(&self, path: &Path) -> io::Result>; + /// Read a whole file (`--file` sources and `--rules`). Async so hosts + /// with an async virtual filesystem need not block. + fn read_file(&self, path: &Path) -> impl Future>> + Send; /// Fetch `url` and extract it with `markkit`'s configuration. Relative /// links must resolve against the final URL after redirects. @@ -108,15 +109,20 @@ impl CliHost for NativeHost { let _ = io::stderr().lock().write_all(text.as_bytes()); } - fn read_stdin(&self) -> io::Result> { - use std::io::Read; - let mut buf = Vec::new(); - io::stdin().read_to_end(&mut buf)?; - Ok(buf) + #[allow(clippy::manual_async_fn)] + fn read_stdin(&self) -> impl Future>> + Send { + async { + use std::io::Read; + let mut buf = Vec::new(); + io::stdin().read_to_end(&mut buf)?; + Ok(buf) + } } - fn read_file(&self, path: &Path) -> io::Result> { - std::fs::read(path) + #[allow(clippy::manual_async_fn)] + fn read_file(&self, path: &Path) -> impl Future>> + Send { + let path = path.to_path_buf(); + async move { std::fs::read(path) } } #[allow(clippy::manual_async_fn)] @@ -471,7 +477,7 @@ async fn execute(cli: Cli, host: &H) -> Result<()> { } if let Some(path) = &rules { - let text = String::from_utf8(host.read_file(path)?).map_err(|_| { + let text = String::from_utf8(host.read_file(path).await?).map_err(|_| { io::Error::new( io::ErrorKind::InvalidData, "stream did not contain valid UTF-8", @@ -486,12 +492,12 @@ async fn execute(cli: Cli, host: &H) -> Result<()> { let source_label = source.clone().unwrap_or_else(|| "".to_string()); let extraction = if read_stdin { - let buf = host.read_stdin()?; + let buf = host.read_stdin().await?; markkit .extract_document_sync(&buf, markkit::DocumentHint::content_type("text/html"))? } else if file { let src = source.as_deref().unwrap(); - let bytes = host.read_file(Path::new(src))?; + let bytes = host.read_file(Path::new(src)).await?; // Format detected from the extension, then magic bytes. markkit.extract_document_sync(&bytes, markkit::DocumentHint::file_name(src))? } else { diff --git a/crates/markkit-cli/tests/host.rs b/crates/markkit-cli/tests/host.rs index edf8965..8b10175 100644 --- a/crates/markkit-cli/tests/host.rs +++ b/crates/markkit-cli/tests/host.rs @@ -47,14 +47,13 @@ impl CliHost for TestHost { fn write_stderr(&self, text: &str) { self.stderr.lock().unwrap().push_str(text); } - fn read_stdin(&self) -> io::Result> { - Ok(self.stdin.clone()) + fn read_stdin(&self) -> impl Future>> + Send { + let stdin = self.stdin.clone(); + async move { Ok(stdin) } } - fn read_file(&self, path: &Path) -> io::Result> { - self.files - .get(path) - .cloned() - .ok_or_else(|| io::Error::from(io::ErrorKind::NotFound)) + fn read_file(&self, path: &Path) -> impl Future>> + Send { + let file = self.files.get(path).cloned(); + async move { file.ok_or_else(|| io::Error::from(io::ErrorKind::NotFound)) } } fn extract_url( &self, @@ -271,3 +270,12 @@ async fn broken_pipe_on_stdout_exits_0() { assert_eq!(run_with(&host, &["extract", "-"]).await, 0); assert!(host.err().is_empty()); } + +#[test] +fn run_future_is_send() { + // Embedders run the CLI inside `Send` async contexts (bashkit builtins). + fn assert_send(_: T) {} + let host = TestHost::default(); + assert_send(run(["markkit", "--help"], &host)); + assert_send(run(["markkit", "--help"], &markkit_cli::NativeHost)); +} From 9d46e1aeae2bbab1b4c23189ea6e5354291368db Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Sun, 1 Nov 2026 02:39:50 +0000 Subject: [PATCH 3/3] feat(cli): host clamps resource limits after flags --- crates/markkit-cli/src/lib.rs | 11 +++++++++-- crates/markkit-cli/tests/host.rs | 20 ++++++++++++++++++++ knowledge/foundations/architecture.md | 2 ++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/crates/markkit-cli/src/lib.rs b/crates/markkit-cli/src/lib.rs index d1e3aff..0922bfd 100644 --- a/crates/markkit-cli/src/lib.rs +++ b/crates/markkit-cli/src/lib.rs @@ -75,6 +75,13 @@ pub trait CliHost: Send + Sync { url: &str, ) -> impl Future> + Send; + /// Final say on resource limits after `--max-*`/`--timeout` flags were + /// applied. A shared host clamps here so a flag cannot raise a cap past + /// its budget. + fn limits(&self, requested: Limits) -> Limits { + requested + } + /// Return `Some(reason)` to refuse a feature requested by a flag. fn refuse(&self, feature: HostFeature) -> Option { let _ = feature; @@ -409,7 +416,7 @@ async fn execute(cli: Cli, host: &H) -> Result<()> { locale, limits, } => { - let limits = limits.apply(Limits::default()); + let limits = host.limits(limits.apply(Limits::default())); let max_message = mcp::max_message_bytes(limits.max_input_size); let tool = markkit::tool::MarkkitTool::builder() .locale(&locale) @@ -448,7 +455,7 @@ async fn execute(cli: Cli, host: &H) -> Result<()> { } let mut builder = Markkit::builder() - .limits(limits.apply(Limits::default())) + .limits(host.limits(limits.apply(Limits::default()))) .pdf_layout(layout) .pdf_ocr(ocr) .pdf_ocr_language(ocr_language.unwrap_or_else(|| "eng".into())) diff --git a/crates/markkit-cli/tests/host.rs b/crates/markkit-cli/tests/host.rs index 8b10175..43fd80e 100644 --- a/crates/markkit-cli/tests/host.rs +++ b/crates/markkit-cli/tests/host.rs @@ -25,6 +25,7 @@ struct TestHost { fetched: Mutex>, refused: Vec, broken_pipe: bool, + max_input: Option, } impl TestHost { @@ -77,6 +78,12 @@ impl CliHost for TestHost { ) } } + fn limits(&self, mut requested: markkit::Limits) -> markkit::Limits { + if let Some(cap) = self.max_input { + requested.max_input_size = requested.max_input_size.min(cap); + } + requested + } fn refuse(&self, feature: HostFeature) -> Option { self.refused .contains(&feature) @@ -279,3 +286,16 @@ fn run_future_is_send() { assert_send(run(["markkit", "--help"], &host)); assert_send(run(["markkit", "--help"], &markkit_cli::NativeHost)); } + +#[tokio::test] +async fn host_clamps_limits_raised_by_flags() { + let host = TestHost { + stdin: PAGE.to_vec(), + max_input: Some(16), + ..Default::default() + }; + let code = run_with(&host, &["extract", "-", "--max-input-size", "1000000000"]).await; + assert_eq!(code, 1); + assert!(host.err().contains("input"), "{}", host.err()); + assert!(host.out().is_empty()); +} diff --git a/knowledge/foundations/architecture.md b/knowledge/foundations/architecture.md index 94486d2..37b89d1 100644 --- a/knowledge/foundations/architecture.md +++ b/knowledge/foundations/architecture.md @@ -73,6 +73,8 @@ codes are one implementation. policy). Sandboxes fetch through their own egress, set `base_url` to the final URL and call `extract_document`; built without `native-http` the CLI links no reqwest/TLS. +- `CliHost::limits` gets the final say on `Limits` after `--max-*`/`--timeout` + flags, so a shared host can stop a flag from raising a cap past its budget. - Hosts may refuse `--ocr` (`HostFeature::Ocr`: host Tesseract binary, host model files). Refused flags still parse and show in help; exit 1. - Exit codes: 0 success / help / version / closed stdout pipe; 1 runtime error;