From daa46ec074cf7830a0993c71187d98a9cd843922 Mon Sep 17 00:00:00 2001 From: Milan Bhandari Date: Mon, 12 Oct 2026 10:32:04 -0500 Subject: [PATCH] Pack diffr-web's grammar tables and unpack each language on first use The browser engine carried every grammar's parse tables uncompressed, 111 MB of its 127 MB. The wasm build now packs them the way native builds do, with ruzstd standing in for libzstd, so the engine is 25 MB and only the languages a comparison uses are unpacked. Agent-Session: 429958d8-68f3-4f68-bd3b-a8371e7feb8e Agent-Session: 5eb4067a-b2f6-4508-a6dd-0471ff2c14d1 Agent-Session: c4502e33-93da-4505-ac5c-2015d3e90139 --- diffr/Cargo.lock | 8 +++ diffr/crates/diffr-grammars/Cargo.toml | 8 ++- diffr/crates/diffr-grammars/build.rs | 76 +++++++++++++++++++------- diffr/crates/diffr-grammars/src/lib.rs | 28 +++++++++- 4 files changed, 98 insertions(+), 22 deletions(-) diff --git a/diffr/Cargo.lock b/diffr/Cargo.lock index 434af80cf..f78458092 100644 --- a/diffr/Cargo.lock +++ b/diffr/Cargo.lock @@ -897,6 +897,7 @@ dependencies = [ "cc", "object", "rayon", + "ruzstd", "serde_json", "tree-sitter", "tree-sitter-ada", @@ -958,6 +959,7 @@ dependencies = [ "tree-sitter-yaml", "tree-sitter-zig", "ts-parser-perl", + "wasmparser 0.259.0", "zstd", "zstd-sys", ] @@ -3536,6 +3538,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "ruzstd" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fad02996bfc73da3e301efe90b1837be9ed8f4a462b6ed410aa35d00381de89f" + [[package]] name = "ryu" version = "1.0.17" diff --git a/diffr/crates/diffr-grammars/Cargo.toml b/diffr/crates/diffr-grammars/Cargo.toml index 1a9124697..de964bda3 100644 --- a/diffr/crates/diffr-grammars/Cargo.toml +++ b/diffr/crates/diffr-grammars/Cargo.toml @@ -22,11 +22,14 @@ include = [ [dependencies] tree-sitter-language = "0.1.7" -# The grammars' C code calls ZSTD_decompress, except on wasm32, where the -# tables are not packed. +# The grammars' C code calls ZSTD_decompress: libzstd's, or on wasm32, where it +# does not build, src/lib.rs's over ruzstd. [target.'cfg(not(target_arch = "wasm32"))'.dependencies] zstd-sys = { version = "2.1.0", default-features = false } +[target.'cfg(target_arch = "wasm32")'.dependencies] +ruzstd = { version = "0.7.3", default-features = false, features = ["std"] } + [build-dependencies] cc = "1.2.48" object = { version = "0.40", default-features = false, features = ["read_core", "coff", "elf", "macho", "std"] } @@ -34,6 +37,7 @@ rayon = "1.7.0" serde_json = "1.0" tree-sitter = "0.26.10" tree-sitter-c = "0.24" +wasmparser = { version = "0.259.0", default-features = false } zstd = { version = "0.13.3", default-features = false } # The build script reads the parser sources from these packages. diff --git a/diffr/crates/diffr-grammars/build.rs b/diffr/crates/diffr-grammars/build.rs index b4955005d..58d99a846 100644 --- a/diffr/crates/diffr-grammars/build.rs +++ b/diffr/crates/diffr-grammars/build.rs @@ -7,10 +7,10 @@ //! function, `diffr_unpack_`, fills the arrays, and `src/lib.rs` runs it //! once before the grammar's first use. //! -//! A wasm32 build compiles each grammar as generated, with an empty unpack -//! function: a page is served compressed, so packing would only cost the -//! decompressor, and wasm object files hold their tables in a form this -//! script does not read. +//! A wasm32 build packs them too. Wasm copies a module's data into every +//! instance's memory, so unpacked tables would cost each worker all ~110 MB of +//! them; packed, a worker holds only the grammars it has parsed with. There the +//! decompressor is `src/lib.rs`'s pure-Rust `ZSTD_decompress`. use object::{Object, ObjectSection, ObjectSymbol}; use rayon::prelude::*; use std::collections::BTreeMap; @@ -190,21 +190,6 @@ fn package_roots() -> BTreeMap { /// fills them, and the grammar's constructor renamed to `diffr_`, then compile it. fn pack(symbol: &str, source: &Path, out: &Path) { std::fs::create_dir_all(out).unwrap(); - if std::env::var("CARGO_CFG_TARGET_ARCH").unwrap() == "wasm32" { - let grammar = format!( - "#define {symbol} diffr_{symbol}\n\ - #include \"parser.c\"\n\ - void diffr_unpack_{symbol}(void) {{}}\n" - ); - std::fs::write(out.join("grammar.c"), grammar).unwrap(); - let mut build = compiler(source, out); - build.file(out.join("grammar.c")); - if source.join("scanner.c").is_file() { - build.file(source.join("scanner.c")); - } - build.compile(symbol); - return; - } let parser = std::fs::read_to_string(source.join("parser.c")).unwrap(); let header = std::fs::read_to_string(source.join("tree_sitter/parser.h")).unwrap(); let tables = tables(&parser, &header); @@ -289,6 +274,9 @@ fn table_bytes( .unwrap(); let data = std::fs::read(object_path).unwrap(); + if data.starts_with(b"\0asm") { + return wasm_table_bytes(symbol, &data, tables); + } let file = object::File::parse(&*data).unwrap(); let prefix = if file.format() == object::BinaryFormat::MachO { "_" @@ -322,6 +310,56 @@ fn table_bytes( .collect() } +/// The same tables read from a wasm object file. Its `linking` section places each defined +/// data symbol in a data segment, at an offset and with a size, so no size globals are needed. +fn wasm_table_bytes(symbol: &str, data: &[u8], tables: &[Table]) -> Vec> { + let mut segments = Vec::new(); + let mut symbols = BTreeMap::new(); + for payload in wasmparser::Parser::new(0).parse_all(data) { + match payload.unwrap() { + wasmparser::Payload::DataSection(reader) => { + for segment in reader { + segments.push(segment.unwrap().data); + } + } + wasmparser::Payload::CustomSection(section) if section.name() == "linking" => { + let linking = wasmparser::LinkingSectionReader::new(wasmparser::BinaryReader::new( + section.data(), + section.data_offset(), + )) + .unwrap(); + for subsection in linking { + let wasmparser::Linking::SymbolTable(table) = subsection.unwrap() else { + continue; + }; + for info in table { + if let wasmparser::SymbolInfo::Data { + name, + symbol: Some(defined), + .. + } = info.unwrap() + { + symbols.insert(name, defined); + } + } + } + } + _ => {} + } + } + tables + .iter() + .map(|Table { name, .. }| { + let defined = symbols + .get(name) + .unwrap_or_else(|| panic!("{name} is missing from the {symbol} tables object")); + let start = usize::try_from(defined.offset).unwrap(); + let end = start + usize::try_from(defined.size).unwrap(); + segments[usize::try_from(defined.index).unwrap()][start..end].to_vec() + }) + .collect() +} + fn compiler(source: &Path, out: &Path) -> cc::Build { let mut build = cc::Build::new(); build.include(source).warnings(false).out_dir(out); diff --git a/diffr/crates/diffr-grammars/src/lib.rs b/diffr/crates/diffr-grammars/src/lib.rs index 35848e6a7..eb20b4849 100644 --- a/diffr/crates/diffr-grammars/src/lib.rs +++ b/diffr/crates/diffr-grammars/src/lib.rs @@ -4,10 +4,36 @@ //! The first call unpacks that grammar's tables. use std::sync::Once; use tree_sitter_language::LanguageFn; -// Links libzstd for the grammars' C code; wasm32 grammars are not packed. +// Links libzstd for the grammars' C code; wasm32 has the pure-Rust decoder below. #[cfg(not(target_arch = "wasm32"))] use zstd_sys as _; +/// libzstd's one-shot decompression, which the grammars' unpack functions call, for wasm32, +/// where libzstd does not build. Returns the bytes written, or `usize::MAX` on a bad frame. +/// +/// # Safety +/// +/// `dst` must be writable for `dst_capacity` bytes and `src` readable for `src_size`. +#[cfg(target_arch = "wasm32")] +#[no_mangle] +pub unsafe extern "C" fn ZSTD_decompress( + dst: *mut u8, + dst_capacity: usize, + src: *const u8, + src_size: usize, +) -> usize { + // Safety: the caller passes a table's own storage and its packed bytes, with their sizes. + let (output, input) = unsafe { + ( + std::slice::from_raw_parts_mut(dst, dst_capacity), + std::slice::from_raw_parts(src, src_size), + ) + }; + ruzstd::FrameDecoder::new() + .decode_all(input, output) + .unwrap_or(usize::MAX) +} + macro_rules! grammar { ($name:ident, $unpack:ident, $constructor:ident) => { pub const $name: LanguageFn = {