From 157a08eb75b354c255f1583dff57d2f89be46776 Mon Sep 17 00:00:00 2001 From: Ryan Fowler Date: Wed, 5 Aug 2026 16:42:23 -0400 Subject: [PATCH] feat: upgrade legible to 0.5.0 and remove htmd dependency Legible 0.5.0 adds a markdown_content field to Article, providing CommonMark output directly. This lets us remove the htmd dependency that was used solely for HTML-to-Markdown conversion. Also updates documentation references to reflect the simplified pipeline. --- Cargo.lock | 233 +++----------------------------------- Cargo.toml | 3 +- docs/cli-reference.md | 2 +- docs/output-formatting.md | 4 +- src/format/article.rs | 6 +- 5 files changed, 22 insertions(+), 226 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8ad4850a..14eb61f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -32,12 +32,6 @@ dependencies = [ "alloc-no-stdlib", ] -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - [[package]] name = "anstream" version = "1.0.0" @@ -207,21 +201,6 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9" -[[package]] -name = "bit-set" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" -dependencies = [ - "bit-vec 0.8.0", -] - -[[package]] -name = "bit-vec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" - [[package]] name = "bit-vec" version = "0.9.1" @@ -522,29 +501,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "cssparser" -version = "0.37.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9cdaae01d5ed7882b04d795e7f752f46ff52d2fa3b50a20d28c464510bba98" -dependencies = [ - "cssparser-macros", - "dtoa-short", - "itoa", - "phf", - "smallvec", -] - -[[package]] -name = "cssparser-macros" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a2a99df6e410a8ff4245aa2006499ea662245f967cc7c0a38c83ef8eb44dbf" -dependencies = [ - "quote", - "syn 2.0.117", -] - [[package]] name = "ctutils" version = "0.4.2" @@ -580,27 +536,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -[[package]] -name = "derive_more" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" -dependencies = [ - "derive_more-impl", -] - -[[package]] -name = "derive_more-impl" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" -dependencies = [ - "proc-macro2", - "quote", - "rustc_version", - "syn 2.0.117", -] - [[package]] name = "difflib" version = "0.4.0" @@ -639,36 +574,6 @@ dependencies = [ "litrs", ] -[[package]] -name = "dom_query" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fac5fca71e65e94cc718a6e2af65d6e0f9c6027751c2aa562fbb5087fda639bc" -dependencies = [ - "bit-set", - "cssparser", - "foldhash 0.2.0", - "html5ever 0.39.0", - "precomputed-hash", - "selectors", - "tendril", -] - -[[package]] -name = "dtoa" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" - -[[package]] -name = "dtoa-short" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" -dependencies = [ - "dtoa", -] - [[package]] name = "dunce" version = "1.0.5" @@ -747,7 +652,6 @@ dependencies = [ "h3", "h3-quinn", "hmac", - "htmd", "http", "http-body", "http-body-util", @@ -847,12 +751,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" -[[package]] -name = "foldhash" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1061,7 +959,7 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash 0.1.5", + "foldhash", ] [[package]] @@ -1069,11 +967,6 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash 0.2.0", -] [[package]] name = "heck" @@ -1090,27 +983,6 @@ dependencies = [ "digest", ] -[[package]] -name = "htmd" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5a1c7113c831fec68cbd79cd8bf281a84e5b6943f51473dc266b0b88a6a017e" -dependencies = [ - "html5ever 0.38.0", - "markup5ever_rcdom", - "phf", -] - -[[package]] -name = "html5ever" -version = "0.38.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1054432bae2f14e0061e33d23402fbaa67a921d319d56adc6bcf887ddad1cbc2" -dependencies = [ - "log", - "markup5ever 0.38.0", -] - [[package]] name = "html5ever" version = "0.39.0" @@ -1118,7 +990,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46a1761807faccc9a19e86944bbf40610014066306f96edcdedc2fb714bcb7b8" dependencies = [ "log", - "markup5ever 0.39.0", + "markup5ever", ] [[package]] @@ -1481,16 +1353,17 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "legible" -version = "0.4.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c1f98ecdbfc2350a51384cf7998b00e7651095e3e8dc882cd947533fb78d9e4" +checksum = "41ec7f7fa1fc97c139420025a96185aa4d1db3a7d2a4270dbb93d0866f7af830" dependencies = [ - "dom_query", - "hashbrown 0.17.1", - "once_cell", + "html5ever", + "memchr", "regex", "serde", "serde_json", + "smallvec", + "tendril", "thiserror", "url", ] @@ -1549,17 +1422,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" -[[package]] -name = "markup5ever" -version = "0.38.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8983d30f2915feeaaab2d6babdd6bc7e9ed1a00b66b5e6d74df19aa9c0e91862" -dependencies = [ - "log", - "tendril", - "web_atoms", -] - [[package]] name = "markup5ever" version = "0.39.0" @@ -1571,18 +1433,6 @@ dependencies = [ "web_atoms", ] -[[package]] -name = "markup5ever_rcdom" -version = "0.38.0+unofficial" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "333171ccdf66e915257740d44e38ea5b1b19ce7b45d33cc35cb6f118fbd981ff" -dependencies = [ - "html5ever 0.38.0", - "markup5ever 0.38.0", - "tendril", - "xml5ever", -] - [[package]] name = "md-5" version = "0.11.0" @@ -1788,7 +1638,6 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ - "phf_macros", "phf_shared", "serde", ] @@ -1813,19 +1662,6 @@ dependencies = [ "phf_shared", ] -[[package]] -name = "phf_macros" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" -dependencies = [ - "phf_generator", - "phf_shared", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "phf_shared" version = "0.13.1" @@ -2180,9 +2016,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.3" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -2192,9 +2028,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -2203,9 +2039,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "ring" @@ -2386,25 +2222,6 @@ dependencies = [ "libc", ] -[[package]] -name = "selectors" -version = "0.38.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8adfa1c298912827b8a28b223b3b874357397ae706e6190acd9bf28cee99114d" -dependencies = [ - "bitflags", - "cssparser", - "derive_more", - "log", - "new_debug_unreachable", - "phf", - "phf_codegen", - "precomputed-hash", - "rustc-hash", - "servo_arc", - "smallvec", -] - [[package]] name = "semver" version = "1.0.28" @@ -2465,15 +2282,6 @@ dependencies = [ "zmij", ] -[[package]] -name = "servo_arc" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" -dependencies = [ - "stable_deref_trait", -] - [[package]] name = "sha1" version = "0.11.0" @@ -2578,7 +2386,6 @@ dependencies = [ "parking_lot", "phf_shared", "precomputed-hash", - "serde", ] [[package]] @@ -3490,23 +3297,13 @@ dependencies = [ "rustix", ] -[[package]] -name = "xml5ever" -version = "0.38.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dc9559429edf0cd3f327cc0afd9d6b36fa8cec6d93107b7fbe64f806b5f2d9" -dependencies = [ - "log", - "markup5ever 0.38.0", -] - [[package]] name = "yasna" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" dependencies = [ - "bit-vec 0.9.1", + "bit-vec", "time", ] diff --git a/Cargo.toml b/Cargo.toml index 68a5c23a..d8d7f89d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,6 @@ h2 = "=0.4.15" h3 = "=0.0.8" h3-quinn = "=0.0.10" hmac = "=0.13.0" -htmd = "=0.5.5" http = "=1.5.0" http-body = "=1.1.0" http-body-util = "=0.1.4" @@ -40,7 +39,7 @@ hyper = { version = "=1.11.0", features = ["client", "http1", "http2"] } hyper-util = { version = "=0.1.20", features = ["client-legacy", "client-proxy", "client-proxy-system", "http1", "http2", "tokio"] } image = { version = "=0.25.10", default-features = false, features = ["jpeg", "png", "tiff", "webp"] } ipnet = "=2.12.1" -legible = "=0.4.3" +legible = "=0.5.0" md-5 = "=0.11.0" memchr = "=2.8.3" mime = "=0.3.17" diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 6222c7fe..a91b10e4 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -280,7 +280,7 @@ fetch -m HEAD example.com # Avoid body transfer when supported ### `--article` Convert an HTML response to Markdown by extracting its main readable content -with Legible and converting the extracted HTML with htmd. +with Legible. If the response is already `text/markdown` or `text/x-markdown`, its body is used directly. HTML output begins with YAML frontmatter containing the title, byline, site name, published time, language, direction, text length, excerpt, diff --git a/docs/output-formatting.md b/docs/output-formatting.md index ac9ea46f..39f75478 100644 --- a/docs/output-formatting.md +++ b/docs/output-formatting.md @@ -125,8 +125,8 @@ fetch example.com #### Convert readable HTML to Markdown -Use `--article` to extract the main readable content from an HTML response with -Legible and convert that extracted HTML to Markdown with htmd: +Use `--article` to extract the main readable content from an HTML response and +convert it to Markdown with Legible: ```sh fetch --article example.com/post diff --git a/src/format/article.rs b/src/format/article.rs index 6e9ddea0..2e19cc37 100644 --- a/src/format/article.rs +++ b/src/format/article.rs @@ -6,9 +6,8 @@ pub fn extract_markdown(html: &str, url: &str) -> Result, String> { let options = Options::new().max_elems_to_parse(MAX_ARTICLE_ELEMENTS); let article = legible::parse(html, Some(url), Some(options)) .map_err(|err| format!("failed to extract readable article: {err}"))?; - let markdown = htmd::convert(&article.content) - .map_err(|err| format!("failed to convert extracted article to Markdown: {err}"))?; - Ok(render_document(&article, url, &markdown).into_bytes()) + let markdown = &article.markdown_content; + Ok(render_document(&article, url, markdown).into_bytes()) } pub fn add_url_frontmatter(markdown: &str, url: &str) -> Vec { @@ -114,6 +113,7 @@ mod tests { lang: None, content: String::new(), text_content: String::new(), + markdown_content: String::new(), length: 0, excerpt: None, site_name: None,