diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt
index afc90b173081..70976a979c6f 100644
--- a/apps/CMakeLists.txt
+++ b/apps/CMakeLists.txt
@@ -53,6 +53,7 @@ add_app(cuda_mat_mul)
 add_app(depthwise_separable_conv)
 add_app(fft)
 add_app(gaussian_blur)
+add_app(ggml)
 add_app(hannk)
 add_app(harris)
 # add_app(HelloAndroid)  # don't build HelloAndroid here because it is driven by gradle
diff --git a/apps/ggml/CMakeLists.txt b/apps/ggml/CMakeLists.txt
new file mode 100644
index 000000000000..a06682271249
--- /dev/null
+++ b/apps/ggml/CMakeLists.txt
@@ -0,0 +1,81 @@
+cmake_minimum_required(VERSION 3.28)
+project(ggml)
+
+enable_testing()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED YES)
+set(CMAKE_CXX_EXTENSIONS NO)
+
+# GGML is expected to be provided externally -- in this tree, via the
+# apps/vcpkg/ports/ggml overlay port (see its portfile.cmake). GGML ships its
+# own CMake package config (installed to <prefix>/share/ggml/ by vcpkg's
+# vcpkg_cmake_config_fixup), so no Find module is needed here.
+find_package(ggml CONFIG REQUIRED)
+
+# Halide is already found once by the parent apps/CMakeLists.txt when this app
+# is built as part of the full apps/ tree (a no-op re-find in that case); this
+# call makes apps/ggml also independently configurable/buildable on its own,
+# matching every other app's convention (see e.g. apps/blur/CMakeLists.txt).
+find_package(Halide REQUIRED)
+
+# halide/ contains a from-scratch Halide reimplementation of GGML's Q4_0
+# quantize/dequantize kernels (ggml_quants_halide), benchmarked against
+# GGML's own reference by providers/halide_provider.cpp below.
+add_subdirectory(halide)
+
+add_executable(
+    kernel-bench
+    src/main.cpp
+    src/report.cpp
+    src/bench_quantize.cpp
+    src/bench_dequantize.cpp
+    src/bench_vecdot.cpp
+    src/bench_repack.cpp
+    providers/ggml_provider.cpp
+    providers/halide_provider.cpp
+)
+
+target_include_directories(kernel-bench PRIVATE include providers src halide)
+
+target_link_libraries(kernel-bench PRIVATE ggml::ggml ggml_quants_halide)
+
+# GGML_VERSION isn't exposed through any runtime API, but ggml-config.cmake
+# sets it as a plain CMake variable (baked in from the exporting build's own
+# GGML_VERSION), so this is exactly the version of the library we just linked
+# against -- reliable without touching GGML internals.
+if (DEFINED GGML_VERSION)
+    target_compile_definitions(kernel-bench PRIVATE KERNEL_BENCH_GGML_VERSION="${GGML_VERSION}")
+endif ()
+
+# ggml-config.cmake.in only creates ggml::<backend> import targets
+# (including ggml::ggml-cpu) when GGML was built with GGML_BACKEND_DL=OFF
+# (the default) -- see its `if (NOT GGML_BACKEND_DL)` guard. In DL mode the
+# CPU backend is a runtime-loaded module with no link-time target at all, and
+# the private ABI symbols this tool depends on (see
+# providers/ggml_internal_abi.h) are then unreachable through the CMake
+# package. Fall back to locating the library file directly by name; fail
+# loudly if that isn't possible either, rather than producing a mysterious
+# link error.
+if (NOT TARGET ggml::ggml-cpu)
+    find_library(
+        GGML_CPU_DL_LIB
+        NAMES ggml-cpu
+        HINTS "${ggml_LIB_DIR}" "${ggml_LIB_DIR}/ggml"
+        PATH_SUFFIXES lib lib/ggml bin
+    )
+    if (GGML_CPU_DL_LIB)
+        message(
+            STATUS "kernel-bench: linking ggml-cpu directly (GGML_BACKEND_DL build): ${GGML_CPU_DL_LIB}"
+        )
+        target_link_libraries(kernel-bench PRIVATE "${GGML_CPU_DL_LIB}")
+    else ()
+        message(
+            FATAL_ERROR "kernel-bench: could not find ggml::ggml-cpu or a standalone ggml-cpu library. "
+            "This GGML install appears to have been built with -DGGML_BACKEND_DL=ON, which "
+            "loads the CPU backend as a runtime module instead of linking it -- kernel-bench "
+            "needs to link directly against its internal symbols. Rebuild GGML with "
+            "-DGGML_BACKEND_DL=OFF (the default) and reinstall."
+        )
+    endif ()
+endif ()
diff --git a/apps/ggml/PERF_NOTES.md b/apps/ggml/PERF_NOTES.md
new file mode 100644
index 000000000000..da18185c6be0
--- /dev/null
+++ b/apps/ggml/PERF_NOTES.md
@@ -0,0 +1,393 @@
+# vec_dot performance notes
+
+Working notes for bringing `apps/ggml`'s vec_dot kernels up to `ggml-cpu` speed
+on ARM (measured on an M3 Max). Branch `alexreinking/ggml-on-qk`, worktree
+`~/dev/Halide/ggml-on-qk`.
+
+## Where things stand
+
+Measured at n=4096, best of many runs (see "Measuring" below):
+
+| type | ggml-cpu | halide   | ratio | before this work |
+| ---- | -------- | -------- | ----- | ---------------- |
+| q4_0 | 87.8 ns  | 90.7 ns  | 0.97x | 0.97x            |
+| q4_1 | 106.7 ns | 110.2 ns | 0.97x | 0.80x            |
+| q5_0 | 122.2 ns | 130.2 ns | 0.94x | 0.51x            |
+| q5_1 | 143.5 ns | 153.7 ns | 0.93x | 0.50x            |
+| q8_0 | 69.1 ns  | 70.4 ns  | 0.98x | 0.95x            |
+
+q5_K (still on the float path) also improved 8615 -> 6730 ns from the same
+qh-expansion table (it shares the 1-bit `PlanarBitPack` decode).
+
+Everything else in the table is still on the unscheduled float path
+(0.01x-0.12x) and untouched. All 28 roundtrip tests pass; `kernel-bench --all`
+reports no mismatches; odd block-count tails are correct (verified at n = 32,
+96, 160, 224, 1056).
+
+Commits, oldest first:
+
+- `c66ab336d` apps/ggml: bring q4_0/q8_0 vec_dot up to ggml-cpu speed
+- `c8ec1934d` Add `Stage::distribute()`, and one accumulator per term in
+  `hoist_invariants()`
+- `6c72fb4c1` apps/ggml: take the affine vec_dots (q4_1, q5_1) to SDOT
+- `a6e9b0962` `hoist_invariants()`: return one Func per accumulator, not a Tuple
+
+## Dev aids
+
+The `getenv("GGML_PER_BLOCK_PROBE")` branch (original "variant A" via
+`sdot_partial`) is kept, now reachable only for the *symmetric* SDOT formats
+(q4_0/q8_0/q5_0) -- a dev aid to measure per-block vs the lane-split default.
+The affine formats reach `sever_sum` first and never see it.
+
+Note the generator reads the env var at *generator run time*, so changing it
+does not invalidate ninja's outputs. Force regeneration:
+
+```sh
+rm -f build/apps/ggml/halide/q4_0_vec_dot.o build/apps/ggml/halide/libq4_0_vec_dot.a
+GGML_PER_BLOCK_PROBE=1 cmake --build build/apps/ggml -j --target q4_0_vec_dot
+cmake --build build/apps/ggml -j
+```
+
+## Building
+
+libHalide is consumed from `install/macOS`, so an app change needs only the app
+build, but a Halide change needs build + install first:
+
+```sh
+cmake --build build/macOS -j --target Halide
+cmake --install build/macOS --prefix install/macOS
+cmake --build build/apps/ggml -j
+```
+
+`build/macOS` is configured with `WITH_TESTS=OFF`, so `correctness_*` targets do
+not exist. Compile a test directly instead:
+
+```sh
+c++ -O1 -std=c++17 -DHALIDE_KEEP_MACROS -DHALIDE_WITH_EXCEPTIONS \
+	-I install/macOS/include -I test/common -I tools \
+	test/correctness/rfactor.cpp \
+	-L install/macOS/lib -lHalide -Wl,-rpath,$PWD/install/macOS/lib -o /tmp/rfactor && /tmp/rfactor
+```
+
+`HALIDE_KEEP_MACROS` is required (`internal_assert` is `#undef`'d at the end of
+the installed `Halide.h`); `HALIDE_WITH_EXCEPTIONS` is required or the
+exception-guarded tests silently do not compile. To find which sub-test fails,
+shard it: `TEST_TOTAL_SHARDS=40 TEST_SHARD_INDEX=N ./rfactor`.
+
+## Measuring
+
+macOS moves the process between P- and E-cores between runs, so a single run's
+absolute numbers are not comparable -- swings of 30%+ are normal. Take the best
+of several runs and always read the halide/ggml-cpu *ratio* within a run.
+
+`src/bench_vecdot.cpp` has two dev aids added this session:
+`KERNEL_BENCH_FILTER=q4_0,q8_0` (substring match on type name) and
+`KERNEL_BENCH_N=<n>` (vector length; default 4096). The n sweep is what
+separates per-call overhead from per-block cost -- fit the slope.
+
+Repeat-and-take-best wrapper:
+
+```sh
+#!/bin/zsh
+B=~/dev/Halide/ggml-on-qk/build/apps/ggml
+N=${N:-5}
+FILTER=${FILTER:-q4_0,q4_1,q8_0}
+TMP=$(mktemp -d)
+for i in $(seq $N); do KERNEL_BENCH_FILTER=$FILTER $B/kernel-bench --vecdot --csv $TMP/r$i.csv >/dev/null; done
+cat $TMP/*.csv | awk -F, '
+  $3!="role" && $1=="vec_dot" { k=$2 SUBSEP $4; if (!(k in best) || $5+0 < best[k]) best[k]=$5+0; ok[k]=$9; types[$2]=1 }
+  END { n=0; for (t in types) st[++n]=t
+        for(a=1;a<n;a++)for(b=a+1;b<=n;b++)if(st[a]>st[b]){tmp=st[a];st[a]=st[b];st[b]=tmp}
+        for (i=1;i<=n;i++) { t=st[i]; c=best[t,"ggml-cpu"]; h=best[t,"halide"]
+          printf "%-8s %9.1f ns %10.1f ns %7.2fx  %s\n", t, c, h, (h>0?c/h:0), (ok[t,"halide"]==1?"yes":"NO") } }'
+rm -rf $TMP
+```
+
+To read generated code, re-run the generator by hand with extra outputs. Grab
+the exact command from `build.ninja` (`grep 'COMMAND = .*-n q4_1_vec_dot '`) and
+swap `-e c_header,object` for `-e stmt,assembly`. Add
+`-no_asserts-no_bounds_query` to the target to see what actually ships.
+
+Diagnose SDOT vs fallback by grepping the `.s` for `sdot.4s`; grep the `.stmt`
+for `vector_reduce_add(int32x..(widening_mul(int8x.., int8x..)))`.
+
+## What the q4_0/q8_0 speedup actually was
+
+Four independent things, roughly equal in size:
+
+1. **Lanes from `r.x`, not `r.y`.** `rfactor({{rxo, lane}, {r.y, u}})` keeps the
+   sdot's four Int(32) lanes alive into the float accumulator, so no block pays
+   a horizontal reduce. Lanes must come from `r.x`: blocks are interleaved
+   `{scale, codes}` records, so a lane per block gathers both the codes and the
+   scales.
+2. **Chained sdot.** Cut `r.x` into chunks of 16 run serially, so both sdots
+   accumulate into the *same* register. Reducing straight to 4 lanes makes
+   `CodeGen_ARM` lower the wide reduce as two independent sdots plus an `addp`
+   (`codegen_dot_product_vector_reduce` only matches factor 4 and recurses).
+3. **Interleave 4 blocks into independent accumulators.** Widening the vector
+   does not help -- every lane of one accumulator advances on every block, so
+   only interleaving blocks shortens the multiply-add chain. Un-interleaved, the
+   kernel is latency-bound at ~4 cycles/block.
+4. **Per-call overhead.** vec_dot is called once per output element of a matvec,
+   so nothing amortizes. Three `Halide::Runtime::Buffer` constructions cost ~13
+   ns flat; the assert/bounds-query prologue another ~10 ns. Fixed by filling a
+   `halide_buffer_t` in place (`StackBuffer` in `ggml_quants.cpp`) and building
+   these libraries with `FEATURES no_asserts no_bounds_query`.
+
+Two traps found along the way:
+
+- **A predicated tail is not a local cost.** Splitting the block RVar with
+  `GuardWithIf` makes the per-block sdot a dynamic-extent allocation that Halide
+  has to `bzero` and accumulate *through memory*, roughly doubling the cost of
+  every block. Fixed by giving the main reduction an exactly divisible extent
+  (`(nb / unroll_blocks) * unroll_blocks`) and sweeping the remainder in a
+  second update at the default schedule. `unroll_blocks` must be a power of two
+  -- 3 and 6 measured 40% worse because the simplifier cannot discharge the
+  tail.
+- **`specialize()` inherits the schedule as of the call**, so scheduling
+  directives applied *after* `specialize()` do not reach the specialized branch.
+  It silently dropped the vectorize/unroll and made things 4x slower.
+
+### q4_0/q8_0 core-composition cleanup
+
+The tuned schedule above is now fed by faithful, public core Approximation
+compositions rather than ggml's legacy `StructBlockLayout` and code-pack
+wrappers:
+
+- q4_0 `{Float16 d; UInt8 qs[16]}` is
+  `StructLayout -> StorageCast -> PlanarFieldPack -> AdditiveOffset -> SymmetricBlockQuantize -> BlockReshape`.
+  `AdditiveOffset<int8_t, uint8_t>{8}` is the representation policy that maps
+  signed codes `[-8, 7]` to stored nibbles `[0, 15]`; planar packing remains an
+  exact, policy-free bit layout.
+- q8_0 `{Float16 d; Int8 qs[32]}` is
+  `StructLayout -> StorageCast -> SymmetricBlockQuantize -> BlockReshape`. Its
+  faithful signed array means the old UInt8 `BytePack` reinterpretation is
+  unnecessary.
+
+Both formats share one traced weight/activation decode graph between the main
+and remainder updates. The shared stages are eagerly inlined only into the tail
+before the four-block main update is transformed. The main assembly remains
+eight SDOTs per iteration with paired 128-bit code loads, four persistent vector
+accumulators, an explicitly unrolled epilogue, and no accumulator spill. The
+scalar tail is still intentionally proportional to its one-to-three-block
+remainder; include those cases in the planned named scaling/odd-tail
+`kernel-bench` mode.
+
+A struct-typed q8_0 activation input was also tested. It was correct, but it
+broadened load-shape changes across q4_0 and q5_0 without removing any target
+weight/codec representation debt, so the shared activation ABI remains on its
+stable byte path. The standalone q8_0 codecs and q8_0 weight operand use the
+faithful struct composition. Q1_0 and struct-aware reblocking are the remaining
+symmetric compatibility work.
+
+Ten final paired n=4096 runs measured q4_0 at 95.088 ns GGML / 97.000 ns Halide
+(0.9803x), versus a worktree baseline of 92.585 / 95.784 ns (0.9666x). Q8_0
+measured 72.973 / 74.951 ns (0.9736x), versus 70.618 / 74.870 ns (0.9432x).
+Absolute times moved with core placement; the paired ratios improved, and Halide
+time changed by +1.27% and +0.11%, respectively. No compiler change was
+required: the simplifier folded q4's offset/planar stages into the expected
+mask, shifts, and vector add, while q8's signed field lowered to direct vector
+loads.
+
+## The q4_1 story so far
+
+q4_1 is affine: the weight decodes to `d*code + m`, so the per-block product
+`(d*code + m) * (d_act*act)` has no single scale to hoist and it was left on the
+default (fully scalar, unscheduled) float reduction at 3843 ns.
+
+`Stage::distribute()` multiplies the product out to
+`d*d_act * sum(code*act) + m*d_act * sum(act)`, and `hoist_invariants()` gives
+each term its own accumulator. Both bodies are integer, so both reach SDOT --
+the first as the ordinary dot, the second as a dot with a vector of ones (ARM
+already matches `i32(int8x)`). That is ggml's own decomposition, and it got q4_1
+to 133.8 ns / 0.80x.
+
+**Design decisions worth not re-litigating:**
+
+- Multiplying out is `distribute()`, a separate schedule directive, *not*
+  something `hoist_invariants()` decides. A first attempt did it unconditionally
+  and broke `hoist_invariants test (predicated RDom)`: `require(...) * (r + 1)`
+  distributes into two accumulators when one was optimal. The predicate that
+  would have rescued it ("ignore constants, don't traverse call arguments...")
+  is exactly the kind of heuristic that needs tuning forever. Whether to
+  multiply out depends on what the terms turn out to contain, which is a cost
+  question, so it belongs in the schedule.
+- `hoist_invariants()` returns **one single-valued Func per accumulator**, not a
+  Tuple. The Tuple version worked and measured identically, but it blocked
+  `compute_offline` (which cannot sever one value of a Tuple) and forced
+  `change_type` to grow multi-output support. Fusing the terms' loop nests is
+  `compute_with`'s job.
+
+## The stored block sum -- DONE (q4_1 0.97x)
+
+ggml does *not* recompute `sum(act)` at vec_dot time -- it reads the `s` field
+that `block_q8_1` stores at quantize time
+(`{ggml_half d; ggml_half s; int8_t qs[32]}`, 36 bytes). Our Q8_1 codec already
+computes that field (`AppendSums{block_size, SumMode::ScaledFloat}` in
+`make_symmetric_byte_sum_block_scheme`). We now sever the offset term's
+accumulator straight to it via `compute_offline`, which needs no new Halide
+directive: its contract already *is* this claim -- sever a Func's computation,
+replace calls with an ImageParam read, discard the recomputing reduction.
+
+**Result: variant A + sever measured 110.2 ns / 0.97x** (q5_1: 208 ns / 0.64x).
+This *matched* the lane-split+sever projection, so the second route below
+(teaching `distribute()` to split into separate update definitions for per-term
+rfactors) is **not needed** -- skip it. All correct: 28 roundtrips,
+`kernel-bench --all` clean, odd tails at n = 32/96/160/224/1056.
+
+The recipe, as implemented in `vec_dot_generator_base.h`'s `sever_sum` branch
+(reached when `distribute_terms && act_has_block_sums`, i.e. affine x Q8_1):
+
+1. `acc_dot = Acc.update().rfactor({{r.y, u}})` -- whole-block partials (variant
+   A). Inline **only the weight's** decode chain (replacement + inlinable
+   handles, multi-pass), leaving the activation decode `Act`
+   (`act_r.replacement`) whole. `distribute()`, then `hoist_invariants()`. The
+   offset term's accumulator body is then `Act(r.x, u)`, so the accumulator *is*
+   `sum_k decode_act(k, blk)` = the stored `s`.
+2. `parts[1].change_type(Float(16))` -- makes the severed Func's type match the
+   data. Faithful: the encoder rounds `s` to fp16 too (the ~6e-06 rel err is
+   exactly that rounding, same as ggml's).
+3. `Pipeline({Acc}).compute_offline({s16}, {s_blocks})` -- second
+   `compute_offline` on the pipeline (the first, at configure top, severs the
+   encode halves to x_blocks/y_blocks). `s_blocks` is the third Input.
+4. Product term: inline `Act`'s **full** chain into `parts[0]` (one Act
+   eager_inline is not enough -- the chain has intermediate levels, and a single
+   inline leaves the second hoist with no visible d_act factor and it errors),
+   re-`hoist_invariants()` to pull d_act out, `change_type(Int(32))` -- the
+   survivor reaches SDOT. Verified: 8 `sdot` in the `.s`; writeback is
+   `d_w*(d_act*int32_dot) + m_w*s_blocks[blk]`, ggml's exact decomposition, with
+   no `sum(act)` reduction left anywhere in the stmt.
+
+Plumbing (all landed, see Uncommitted):
+
+- `s_blocks`: 1-D `Float(16)` Input, `dim(0).set_stride(act_bytes/2)` (= 18 for
+  Q8_1) -- pinning it makes the read an immediate offset, *and* is required:
+  left dynamic, Halide's default constrains the innermost stride to 1 and the
+  bound-check fails against the strided view.
+- ABI: `StackBuffer::blocks_field_f16(base, nb, byte_offset=2, block_bytes=36)`
+  -- a zero-copy fp16 view of the `s` slot, stride `block_bytes/2`.
+- Format knowledge lives with the codec: `SchemeAndBytes::has_block_sums` set by
+  `make_symmetric_byte_sum_block_scheme`, carried to
+  `VecDotSpec::act_has_block_sums` (guarded `&& a_nat == wbs`, so a Reblock'd
+  activation stays off it).
+
+**The structural conflict (why variant A, not lane-split).** The two terms want
+different rfactors: `sum(code*act)` wants `{rxo->lane, r.y->u}` (the lane split
+is the q4_0/q8_0 win); `sum(act)` must be the *whole-block* sum to equal `s`, so
+`{r.y->u}` only. One update rfactors one way. Variant A gives both `{r.y->u}`;
+severing then deletes the offset accumulator entirely, so its per-lane-partial
+problem never arises -- and the surviving product dot, alone in its block loop,
+schedules close enough to the lane-split base that the ~8 ns gap projected
+between the routes did not materialize. `compute_with` (verified to fuse a
+lane-split with a block-only reduction using `AlignStart` + matching split-var
+names) is therefore unnecessary here; keep it in mind for formats that keep two
+live accumulators.
+
+## The q5_x 5-bit high bit -- DONE (q5_0 0.94x, q5_1 0.93x)
+
+q5_0/q5_1 reach SDOT, but every code carries a per-element high bit unpacked
+from the `qh` field, and that reconstruction -- not the dot, not (for q5_1) the
+offset -- is the whole gap vs q4_x.
+
+**What the reconstruction was.** `PlanarBitPack::decode`'s 1-bit case emitted
+`(qh[kk/8] >> (kk%8)) & 1`: a per-lane variable shift plus a `transpose_vector`
+to broadcast the two `qh` bytes across the 16 sdot lanes
+(`dup.8b + dup.4h + uzp2` per sdot, ~24 NEON ops/block). ggml avoids this with a
+1 KB `table_b2b` memory LUT (byte -> 8 expanded bytes, one contiguous load per
+`qh` byte).
+
+**What was done.** Mirrored the LUT: compile-time b2b tables embedded in the
+binary. The q5 tables contain the final high-bit contribution (`-16/0` for q5_0,
+`0/16` for q5_1), so the lookup folds the shift and q5_0 zero-point into the
+load rather than reconstructing a raw bit and applying them afterward. Several
+other details are necessary:
+
+1. The table read is only a *contiguous* 8-byte load
+   (`b0[ramp(qh_byte*8, 1, 8)]`, matching ggml) when the `qh` byte is a
+   **scalar** and the 8 bit positions are the vector lanes. Inlined into the
+   sdot it is the opposite (the byte varies per lane -> a 16-wide per-lane
+   gather, `ld1` per lane, measured 0.12x). So the reconstructed codes are
+   **materialized** per block (`combine_bits_code` compute_at the block loop,
+   `kk` split `(byte, pos)`: pos vectorizes the load, byte unrolls to a scalar
+   index). To keep the codes in the vector register file rather than round-trip
+   a stack buffer, the materialization is `store_in(MemoryType::Register)` with
+   `kk` split further into 16-code units (one sdot chunk = two `qh` bytes x 8
+   positions) so the store width matches the sdot's read width -- otherwise the
+   8-wide table-load store vs 16-wide sdot load mismatch keeps it in memory.
+2. Materializing needs the codes leaf kept out of `sdot_partial()`'s deep inline
+   (`can_be_inlined()` ignores compute level, so a `compute_root`/`compute_at`
+   schedule alone does not stop `eager_inline`). `sdot_partial()` now takes
+   resolved `Func` identities, not generated-name strings; q5_0 obtains the
+   identity from its `AdditiveRadixSplit` stage key.
+3. q5_0 uses its faithful packed type `{Float16 d; UInt8 qh[4]; UInt8 qs[16]}`.
+   `LittleEndianScalarPack<uint32_t>` decodes qh with `concat_bits()`;
+   `LowerStructTypes` and LLVM consequently issue one unaligned `ldr/ldur w`,
+   matching ggml, rather than four `ldrb`s. q5_1 retains the older scalar
+   `UInt(32)` struct field as explicit transitional debt.
+4. q5_0's odd-block **tail** now shares the main Approximation graph. Before the
+   main update materializes the keyed reconstructed-code and qh-word Funcs, the
+   tail update alone eagerly inlines that decode chain. This removes the second
+   q5_0 weight `approximate_by` graph while keeping the tail inline and the main
+   loop register-resident. q5_1 retains its separate legacy tail chain.
+
+The transpose is gone (verified: no `uzp2`/`dup.8b`); reconstruction is a
+handful of ops + 4 contiguous LUT loads/block. Bonus: the same decode change
+sped up q5_K on the float path (8615 -> 6730 ns).
+
+**The schedule changes still matter after the compiler fix.** The compiler
+change is the dominant improvement, but it does not make the schedule work
+redundant:
+
+- Removing the explicit qh `compute_at(...).store_in(Register)` schedule made
+  q5_0 regress from about 122 to 129 ns and q5_1 from about 152 to 162 ns, even
+  though both generated versions contained a word load. Keep it: it changes
+  placement/reuse around the four bit extracts, not merely the load width.
+- Interleaving two q5 blocks gives better latency hiding/register pressure than
+  the four-block setting used by q4/q8. Applying two globally regressed q4_0 and
+  q4_1, so this is a per-format `VecDotSpec` choice.
+- Explicitly unrolling the fixed-size final rfactor reductions removes a stack
+  spill and one-iteration epilogue loops (roughly another 1 ns here).
+
+**q5_1 needs a different reduction shape.** The generic affine path horizontally
+reduced each block's Int32 dot before accumulating floats. The q5_1-specific
+schedule keeps four float dot lanes live across the entire block loop, maintains
+two scalar `m*s` accumulators alongside them, and fuses their two-block loops.
+The generated steady state now matches the important shape of ggml's kernel: two
+SDOTs per block, persistent vector FMAs, and scalar offset FMAs, with one
+horizontal reduction at the end.
+
+After the core-composition migration, ten paired n=4096 runs had q5_0 medians of
+122.147 ns GGML / 130.645 ns Halide and a median paired ratio of 0.9349x. The
+committed baseline was 122.172 / 130.259 ns and 0.9379x, so the refactor is
+performance-neutral (+0.30% Halide time). q5_1 remained within noise at 143.728
+/ 153.956 ns and 0.9336x. Use repeated paired runs rather than treating either
+sample as a fixed score.
+
+**Core composition now mirrors the representation.** q5_0 is
+`StructLayout -> StorageCast -> LittleEndianScalarPack -> BinaryAlphabetPack -> PlanarFieldPack -> AdditiveRadixSplit -> SymmetricBlockQuantize -> BlockReshape`.
+Opaque Approximation stage keys carry the reconstructed-code and qh-word
+identities into `VecDotSpec`; generated Func names no longer control q5_0's
+schedule. `BlockReshape` and symmetric block quantization now live in the public
+Approximation component library, with ggml compatibility aliases/wrappers.
+
+Reusable performance experiments should become discoverable `kernel-bench`
+modes. In particular, promote the odd-tail n sweep (32, 96, 160, 224, 1056)
+instead of preserving it only as a shell loop.
+
+**Build-flag bug.** q5_0/q5_1's `add_halide_library` were missing
+`FEATURES no_asserts no_bounds_query` (every other tuned vec_dot has it), so
+they paid the assert/bounds-query prologue -- ~11 ns of pure startup on a ~170
+ns call. Adding it was worth 0.63 -> 0.67x on its own; the register store
+another 0.67 -> 0.69x (q5_1 to 0.78x). Check this first on any new kernel.
+
+Group-level codes compute (all blocks first) measured worse than the final
+per-block materialization. Likewise, removing qh materialization because the new
+compiler lowering already produced `ldr w` was a measured regression; the two
+optimizations address different parts of the generated loop.
+
+## Not started
+
+Everything else (k-quants, IQ family, tq\*) is still on the default unscheduled
+float reduction and would benefit from the same treatment as q4_1 -- the
+k-quants' two-level scales are also sums of scaled sub-reductions, which is what
+`distribute()` was built for.
diff --git a/apps/ggml/Q4_0_Q8_0_APPROXIMATION_PLAN.md b/apps/ggml/Q4_0_Q8_0_APPROXIMATION_PLAN.md
new file mode 100644
index 000000000000..079fe18e1921
--- /dev/null
+++ b/apps/ggml/Q4_0_Q8_0_APPROXIMATION_PLAN.md
@@ -0,0 +1,138 @@
+# Clean q4_0 and q8_0 Approximation Implementation
+
+## Current status
+
+- Source baseline commit: `ca685e94e91c33d924f10c238ffa4ae6dab183a4`
+- Working baseline: completed q5_0 core-composition refactor in this worktree
+- Status: complete; all acceptance gates passed
+- [x] Phase 0: inspect the existing schemes and create this progress document
+- [x] Phase 1: collect ten paired baseline runs at n=4096
+- [x] Phase 2: compose q4_0 and q8_0 from reusable core components
+- [x] Phase 3: add focused correctness coverage
+- [x] Phase 4: preserve identity-based scheduling and tuned tails
+- [x] Phase 5: correctness, odd-tail, generated-code, and full-suite validation
+- [x] Phase 6: paired performance validation and durable documentation
+
+## Goal and constraints
+
+Apply the q5_0 cleanup methodology to q4_0 and q8_0. Their schemes should use
+faithful packed struct types and reusable public Halide Approximations; the
+generic vec-dot generator should retain only the base reduction,
+`approximate_by`, `compute_offline`, and scheduling. Preserve bit-exact
+quantization, dequantization/vec-dot correctness, and the tuned four-block SDOT
+shape. Keep median paired performance within 5% of this worktree baseline and at
+least 0.90x GGML.
+
+Unrelated legacy formats remain compatibility debt. In particular, q1_0 keeps
+the legacy symmetric layout until it is migrated deliberately, and q4_1/q5_1
+remain on their affine/legacy paths.
+
+## Target compositions
+
+q4_0 faithful type: `{d: Float16, qs: UInt8[16]}`.
+
+1. `StructLayout`, logical `{qs, d}` to physical fields.
+2. `Apply` `StorageCast<Float32, Float16>` to `d`.
+3. `Apply` `PlanarFieldPack{4, 16}` to stored nibbles.
+4. `Apply` a reusable additive-offset component mapping signed codes to on-disk
+   `[0, 15]` values.
+5. `SymmetricBlockQuantize`, qmax 8, extreme-signed scale selection, and
+   truncate-half-up-with-offset rounding.
+6. `BlockReshape{32}` with the requested row/block-indexed layout.
+
+q8_0 faithful type: `{d: Float16, qs: Int8[32]}`.
+
+1. `StructLayout`, logical `{qs, d}` to physical fields.
+2. `Apply` `StorageCast<Float32, Float16>` to `d`.
+3. `SymmetricBlockQuantize`, qmax 127, absolute-max scale selection, and nearest
+   rounding.
+4. `BlockReshape{32}` with the requested row/block-indexed layout.
+
+The standalone q8_0 codecs and q8_0 weight path use the faithful core scheme.
+The shared activation ABI remains byte-addressed: a struct-typed experiment
+broadened generated-code changes across q4_0/q5_0 without removing reusable
+representation logic from either target's weight/codec pipeline. Mismatched
+consumers also require that byte path for the existing `Reblock` component.
+
+## Validation gates
+
+- q4_0 and q8_0 quantize outputs are bit-exact with GGML; dequantize and vec-dot
+  pass existing tolerances.
+- Focused component tests cover the additive offset and both compositions.
+- `kernel-bench --all` has no failures.
+- Odd block counts pass at n=32, 96, 160, 224, and 1056.
+- ARM main loops retain SDOT, four blocks in flight, wide contiguous code loads,
+  persistent accumulators, and fully unrolled fixed-size epilogues without
+  accumulator stack spills or one-iteration epilogue loops.
+- Median paired performance is no more than 5% slower than baseline and remains
+  at least 0.90x GGML; q5_0 and affine shared-format checks show no accidental
+  regression.
+
+## Benchmark experiment policy
+
+Any useful or repeatable experimental setup must be promoted into `kernel-bench`
+as a named mode rather than left as an ad hoc shell recipe. Size and scaling
+sweeps used here should feed the same scaling/odd-tail mode already identified
+by the q5_0 work.
+
+## Baseline results
+
+Ten paired filtered runs at `KERNEL_BENCH_N=4096`:
+
+| Format | GGML CPU   | Halide     | Paired GGML/Halide |
+| ------ | ---------- | ---------- | ------------------ |
+| q4_0   | 92.585 ns  | 95.784 ns  | 0.9666x            |
+| q4_1   | 104.726 ns | 118.332 ns | 0.8850x            |
+| q5_0   | 119.370 ns | 128.931 ns | 0.9258x            |
+| q5_1   | 135.748 ns | 153.946 ns | 0.8818x            |
+| q8_0   | 70.618 ns  | 74.870 ns  | 0.9432x            |
+
+All candidate correctness flags were true. Raw CSV files are in
+`/tmp/q48-baseline.4xCRUB` for this work session.
+
+## Experiment log
+
+| #   | Change                                                                                                                              | Correctness                                               | GGML / Halide timings                                                                                 | Generated-code observations                                                                              | Decision                                         |
+| --- | ----------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | ------------------------------------------------ |
+| 0   | Completed q5_0 worktree baseline                                                                                                    | All filtered vec-dot checks passed                        | q4_0: 92.585 / 95.784 ns, 0.9666x; q8_0: 70.618 / 74.870 ns, 0.9432x                                  | Existing four-block SDOT paths are the generated-code reference                                          | Reference                                        |
+| 1   | Add core `AdditiveOffset` and compose faithful q4_0/q8_0 structs from public components                                             | Component composition and standalone q4_0/q8_0 tests pass | Ten-run probe: q4_0 94.177 / 96.975 ns, 0.9712x; q8_0 72.942 / 74.824 ns, 0.9748x                     | Core stages simplify to the existing signed-code SDOT inputs                                             | Keep                                             |
+| 2   | Use faithful struct q8_0 for the shared activation operand                                                                          | Correct, including q5_0                                   | Single sample moved absolute timings with core placement; paired ratios did not indicate a regression | Broadened input/load-shape changes across q4_0 and q5_0                                                  | Revert; keep the established byte activation ABI |
+| 3   | Share the traced weight and activation decode graphs between q4_0/q8_0 main and tail updates, eagerly inlining only the tail update | Standalone and odd-size tests pass                        | Ten-run probe: q4_0 96.039 / 99.147 ns, 0.9687x; q8_0 75.801 / 77.443 ns, 0.9788x                     | Removes duplicate tail Approximation graphs; main remains four-block SDOT and the remainder stays scalar | Keep                                             |
+| 4   | Full validation and ten final paired runs                                                                                           | `kernel-bench --all` clean; all paired flags true         | q4_0: 95.088 / 97.000 ns, 0.9803x; q8_0: 72.973 / 74.951 ns, 0.9736x                                  | Eight SDOTs/four blocks, paired 128-bit code loads, persistent accumulators, no accumulator spill        | Final                                            |
+
+## Final paired results
+
+Median of ten n=4096 paired runs:
+
+| Format | GGML CPU   | Halide     | Paired GGML/Halide | Halide vs baseline |
+| ------ | ---------- | ---------- | ------------------ | ------------------ |
+| q4_0   | 95.088 ns  | 97.000 ns  | 0.9803x            | +1.27%             |
+| q4_1   | 106.761 ns | 117.951 ns | 0.9051x            | -0.32%             |
+| q5_0   | 122.167 ns | 130.589 ns | 0.9355x            | +1.29%             |
+| q5_1   | 143.560 ns | 153.727 ns | 0.9339x            | -0.14%             |
+| q8_0   | 72.973 ns  | 74.951 ns  | 0.9736x            | +0.11%             |
+
+Negative deltas are improvements. Raw final CSV files are in
+`/tmp/q48-final.5IktYc` for this work session.
+
+## Framework/compiler issues
+
+- No compiler change was needed. The simplifier folds q4_0's core
+  `AdditiveOffset` and `PlanarFieldPack` into the same mask/shift/vector-add
+  operations consumed by SDOT, and q8_0's signed struct array lowers to direct
+  128-bit loads.
+- A struct-typed q8_0 activation was correct but unnecessarily broadened load
+  shape changes across q4_0/q5_0. The stable shared activation ABI remains the
+  compatibility byte path; q8_0's codecs and weight path are fully
+  core-composed.
+- Shared q4_0/q8_0 tails must be eagerly inlined into the tail update before the
+  main SDOT schedule is applied. The remainder is deliberately scalar and its
+  cost scales with one to three blocks; this reinforces the need for a named
+  size-sweep benchmark mode.
+
+## Final follow-up items
+
+- Add the reusable size/scaling experiments from this work as named
+  `kernel-bench` modes.
+- Migrate q1_0 from `StructBlockLayout` and make `Reblock` struct-aware before
+  removing the symmetric compatibility layout and byte activation path.
diff --git a/apps/ggml/Q5_0_APPROXIMATION_PLAN.md b/apps/ggml/Q5_0_APPROXIMATION_PLAN.md
new file mode 100644
index 000000000000..0b3d3c7eb710
--- /dev/null
+++ b/apps/ggml/Q5_0_APPROXIMATION_PLAN.md
@@ -0,0 +1,150 @@
+# Clean q5_0 Approximation Implementation
+
+## Current status
+
+- Baseline commit: `ca685e94e91c33d924f10c238ffa4ae6dab183a4`
+- Status: complete; all acceptance gates passed
+- [x] Phase 0: confirm clean baseline and create this progress document
+- [x] Phase 1: collect ten paired baseline runs at n=4096
+- [x] Phase 2: add stage-key tracing and reusable core Approximation components
+- [x] Phase 3: add focused core correctness coverage
+- [x] Phase 4: refactor q5_0 composition and schedule lookup
+- [x] Phase 5: correctness, odd-tail, generated-code, and full-suite validation
+- [x] Phase 6: paired performance validation and durable documentation
+
+## Goal and constraints
+
+Refactor q5_0 so its generator contains only the base reduction,
+`approximate_by`, `compute_offline`, and scheduling. Compose all q5_0
+representation logic from reusable core Halide Approximations. Preserve
+correctness, keep median paired performance within 5% of the committed baseline,
+and remain at least 0.90x GGML. q5_1 is explicitly unchanged transitional debt.
+
+## Core Approximation APIs
+
+- Add an opaque, copyable `ApproximationStageKey` to every Approximation
+  instance.
+- Extend traced invocation through `Compose`, `Apply`, `TrustedInverse`, and
+  `Func::approximate_by` so `ApproximationResult` resolves encoded and decoded
+  outputs by `(StageKey, port)` while preserving flat `handles` and `encoded`.
+- Add public standard components in a dedicated header included by `Halide.h`:
+  `StructLayout`, `StorageCast`, `LittleEndianScalarPack`, `BinaryAlphabetPack`,
+  `AdditiveRadixSplit`, `PlanarFieldPack`, `BlockReshape`, and the symmetric
+  block quantizer/policies.
+- Leave ggml compatibility aliases/wrappers so unrelated formats do not migrate.
+- Test nested combinators, repeated types with distinct keys, invalid lookups,
+  encode/decode lookup, and component correctness, including one- and
+  two-dimensional `StructLayout` records.
+
+## q5_0 target composition and schedule
+
+Faithful packed type: `{d: Float16, qh: UInt8[4], qs: UInt8[16]}`.
+
+Outer to inner:
+
+1. `StructLayout`, logical `{qs, qh, d}` to physical fields.
+2. `Apply` `StorageCast<Float32, Float16>` to `d`.
+3. `Apply` `LittleEndianScalarPack<UInt32>` to `qh`.
+4. `Apply` `BinaryAlphabetPack<int8_t>{32, UInt32, -16, 0}` to high
+   contributions.
+5. `Apply` `PlanarFieldPack{4, 16}` to low nibbles.
+6. `Apply` `AdditiveRadixSplit{16, 16}` to signed codes.
+7. Symmetric block quantization, qmax 16, extreme-signed scale selection,
+   truncate-half-up rounding.
+8. `BlockReshape{32, block_indexed}`.
+
+Capture stage keys for reconstructed signed codes (`AdditiveRadixSplit`) and the
+qh word (`LittleEndianScalarPack`), carry them through scheme metadata into
+`VecDotSpec`, and resolve scheduling Funcs from `ApproximationResult`. Replace
+the duplicate q5_0 tail Approximation with stage-scoped eager inlining into the
+tail update. Preserve the measured two-block SDOT schedule and use Func
+identity, not names, for `sdot_partial` exclusions.
+
+## Validation gates
+
+- q5_0 quantize is bit-exact with GGML; dequantize and vec_dot pass tolerances.
+- `kernel-bench --all` has no failures.
+- Odd block counts pass at n=32, 96, 160, 224, and 1056.
+- ARM assembly contains SDOT, one qh word load per block, contiguous LUT loads,
+  no qh byte-load sequence in the main loop, and no accumulator stack spill or
+  one-iteration epilogue loop.
+- Median paired q5_0 is no more than 5% slower than baseline and at least 0.90x
+  GGML; shared formats show no accidental regression.
+- Any compiler optimization is general, separately tested, and
+  target-independent where possible. No new scheduling directive is planned.
+
+## Benchmark experiment policy
+
+Any experimental setup that proves useful or repeatable should be promoted into
+`kernel-bench` as a named mode rather than left as an ad hoc shell recipe. This
+includes scaling/size sweeps such as the q5_0 odd-block checks at n=32, 96, 160,
+224, and 1056. One-off scripts are acceptable for initial exploration, but the
+durable form should make the experiment discoverable, reproducible, and usable
+for future regressions from the benchmark utility itself.
+
+## Baseline results
+
+Ten paired filtered runs, `KERNEL_BENCH_N=4096`, filter
+`q4_0,q4_1,q5_0,q5_1,q8_0`:
+
+Median of ten per-run timings and median paired ratio:
+
+| Format | GGML CPU   | Halide     | Paired GGML/Halide |
+| ------ | ---------- | ---------- | ------------------ |
+| q4_0   | 94.437 ns  | 96.893 ns  | 0.9726x            |
+| q4_1   | 107.819 ns | 118.424 ns | 0.9020x            |
+| q5_0   | 122.172 ns | 130.259 ns | 0.9379x            |
+| q5_1   | 143.575 ns | 153.717 ns | 0.9339x            |
+| q8_0   | 73.202 ns  | 74.829 ns  | 0.9764x            |
+
+All candidate correctness flags were true. Raw CSV files are in
+`/tmp/q50-baseline.tX9dNF` for this work session.
+
+## Experiment log
+
+| #   | Change                                                                                                                | Correctness                                                                                        | GGML / Halide timings                                             | Generated-code observations                                                                                          | Decision  |
+| --- | --------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | --------- |
+| 0   | Clean committed baseline                                                                                              | All filtered vec_dot checks passed                                                                 | q5_0: 122.172 / 130.259 ns, 0.9379x paired median                 | Existing q5_0 path is the generated-code reference                                                                   | Reference |
+| 1   | Add opaque stage keys and traces through Compose, Apply, TrustedInverse, and approximate_by                           | Nested/repeated/directional/invalid lookup tests pass                                              | Not performance-sensitive                                         | Flat encoded/handle compatibility retained                                                                           | Keep      |
+| 2   | Add public core components; move BlockReshape and symmetric quantization behind ggml compatibility alias/wrapper      | Focused component suite passes, including 1-D and 2-D StructLayout and inline StorageCast rounding | Not measured independently                                        | `strict_float` makes fp16 storage rounding survive eager inlining                                                    | Keep      |
+| 3   | Replace q5_0 legacy split-code composition with the eight reusable stages                                             | q5_0 quantize/dequantize/vec_dot pass                                                              | Focused q5_0 run: 117.7 / 126.2 ns in one sample                  | Faithful UInt8[4] qh lowers through concat_bits to a word load                                                       | Keep      |
+| 4   | Resolve codes/qh by stage key, stop sdot inlining by Func identity, and share/eager-inline the q5_0 tail decode graph | Odd n=32/96/160/224/1056 all pass                                                                  | Included in final medians                                         | Main loop has two blocks in flight, four SDOTs/pair, persistent vector accumulators; tail is scalar and fully inline | Keep      |
+| 5   | Full validation and ten final paired runs after the final StorageCast change                                          | `kernel-bench --all` clean; focused tests and odd sizes pass; all paired flags true                | q5_0: 122.147 / 130.645 ns, 0.9349x; baseline delta +0.30% Halide | One unaligned qh word load/block, contiguous LUT loads, no accumulator spill or one-iteration epilogue               | Final     |
+
+## Final paired results
+
+Median of ten n=4096 paired runs:
+
+| Format | GGML CPU   | Halide     | Paired GGML/Halide | Halide vs baseline |
+| ------ | ---------- | ---------- | ------------------ | ------------------ |
+| q4_0   | 94.187 ns  | 96.832 ns  | 0.9727x            | -0.06%             |
+| q4_1   | 106.760 ns | 118.423 ns | 0.9015x            | 0.00%              |
+| q5_0   | 122.147 ns | 130.645 ns | 0.9349x            | +0.30%             |
+| q5_1   | 143.728 ns | 153.956 ns | 0.9336x            | +0.16%             |
+| q8_0   | 72.774 ns  | 74.687 ns  | 0.9744x            | -0.19%             |
+
+Negative deltas are improvements. Raw final CSV files are in
+`/tmp/q50-final-strict.vULMku` for this work session.
+
+## Framework/compiler issues
+
+- No compiler change was needed. Existing `LowerStructTypes` handling of
+  `concat_bits()` recovered one unaligned qh word load from the faithful
+  `UInt8[4]` field.
+- The existing Halide build had `WITH_TESTS=OFF`; it was reconfigured with tests
+  enabled to build the two focused correctness targets.
+- Installing only the development component updates headers and GenGen but not
+  the changed shared library; a full `cmake --install` was required before the
+  standalone ggml generator could link the new stage lookup methods.
+- A plain fp32-to-fp16-to-fp32 cast chain can fuse away when fully inlined.
+  `StorageCast` uses `strict_float` on both conversions so storage rounding is
+  schedule-independent; its correctness test intentionally leaves the stage
+  inline. q5_0 also materializes the fp16 value in its packed struct field.
+
+## Final follow-up items
+
+- Add a `kernel-bench` scaling/odd-tail mode that captures the useful n-sweep
+  performed during this work; follow the benchmark experiment policy above for
+  future reusable setups.
+- Remove the legacy q5-specific components and specialized q5_1 reduction after
+  q5_1 is migrated to reusable components; this work intentionally leaves it.
diff --git a/apps/ggml/README.md b/apps/ggml/README.md
new file mode 100644
index 000000000000..1d5adc675b90
--- /dev/null
+++ b/apps/ggml/README.md
@@ -0,0 +1,66 @@
+# kernel-bench
+
+Benchmarks GGML's CPU quantize / dequantize / vec_dot / repack kernels against a
+designated correctness reference, and is structured so that a from-scratch
+implementation of any of those kernels can be dropped in and compared too. See
+`providers/README.md` for how to add one.
+
+This is a **standalone** CMake project. It is not built as part of GGML itself
+and consumes an already-built-and-installed GGML purely as an external
+dependency via `find_package`.
+
+## Build
+
+```sh
+# 1. Build and install GGML somewhere (skip if you already have an install).
+#    GGML_BACKEND_DL=OFF (the default) is required -- see the "private ABI" note below.
+cmake -S /path/to/ggml -B /path/to/ggml/build -DCMAKE_BUILD_TYPE=Release
+cmake --build /path/to/ggml/build -j
+cmake --install /path/to/ggml/build --prefix /path/to/ggml/install
+
+# 2. Build kernel-bench against that install.
+cmake -S . -B build -DCMAKE_PREFIX_PATH=/path/to/ggml/install
+cmake --build build -j
+
+./build/kernel-bench --all
+```
+
+## What the report means
+
+Each row shows a `ggml_type` (or, for repack, a specific interleave layout like
+`q4_0_4x4_q8_0`), GGML's designated **reference** implementation and its
+timing/throughput, and one column per **candidate** implementation registered
+for that kernel: its timing/throughput, speedup relative to the reference, and
+whether its output matched the reference (quantize/repack packing is checked
+byte-for-byte; dequantize/vec_dot/gemv/gemm results are checked within a
+relative-error tolerance, since those involve floating point accumulation that
+different implementations may order differently).
+
+A candidate flagged "identical to reference" has the exact same function address
+as the reference -- this happens whenever the current CPU architecture has no
+separate optimized kernel for that type (GGML's `src/ggml-cpu/arch-fallback.h`
+collapses the two names onto one symbol in that case), so timing it separately
+would only measure noise.
+
+The dequantize table currently shows only a reference column with no candidates:
+GGML has exactly one dequantize implementation per type (arch-independent, in
+`src/ggml-quants.c`), so there's nothing to compare it against yet -- this is
+intentionally the first place to plug in a new provider (see
+`providers/README.md`).
+
+## Why this needs a private ABI header
+
+GGML's public API (`ggml_get_type_traits` / `ggml_get_type_traits_cpu` in
+`include/ggml.h` / `include/ggml-cpu.h`) exposes exactly one reference and one
+CPU-dispatched implementation per type, which is sufficient for the quantize and
+dequantize benchmarks without touching anything private. It does **not** expose
+the always-available pure-C fallback for `vec_dot`, nor anything for the repack
+`quantize_mat`/`gemv`/`gemm` kernels. Those are only reachable because
+`ggml-cpu` is built without `-fvisibility=hidden`, so its internal (but
+non-`static`) C symbols end up with default/exported linker visibility by
+accident of the build configuration rather than by design.
+`providers/ggml_internal_abi.h` redeclares exactly the symbols needed, copied
+from GGML's uninstalled `src/ggml-cpu/quants.h` / `repack.h` as of the commit
+this tool was written against. If a future GGML release renames or changes the
+signature of one of these functions, that header (and
+`providers/ggml_provider.cpp`) are the only places that need updating.
diff --git a/apps/ggml/halide/CMakeLists.txt b/apps/ggml/halide/CMakeLists.txt
new file mode 100644
index 000000000000..8adce0e4b9ea
--- /dev/null
+++ b/apps/ggml/halide/CMakeLists.txt
@@ -0,0 +1,773 @@
+add_halide_generator(
+    quants.generator
+    SOURCES
+    f16_generators.cpp
+    bf16_generators.cpp
+    repack_quantize_mat_generators.cpp
+    repack_matmul_generator.cpp
+    symmetric_quant_generators.cpp
+    symmetric_vec_dot_generator.cpp
+    lookup_table_quant_generators.cpp
+    lookup_table_vec_dot_generator.cpp
+    k_quant_vec_dot_generator.cpp
+    k_quant_generators.cpp
+)
+
+# Q4_0's and Q8_0's quantize/dequantize/vec_dot kernels are GENERATOR_ARGS
+# instantiations of the generic, reusable Approximation-based
+# symmetric_quantize/symmetric_dequantize/symmetric_vec_dot generators (see
+# symmetric_quant_generators.cpp/symmetric_vec_dot_generator.cpp and
+# quant_components.h) -- not their own per-format C++ Generator classes. See
+# quant_components.h's RoundingMode/ScaleAnchor for what these params mean.
+add_halide_library(
+    q4_0_quantize
+    FROM quants.generator
+    GENERATOR symmetric_quantize
+    PARAMS
+    block_size=32 qmax=8 code_bits=4 rounding=truncate_half_up_with_offset anchor=extreme_signed
+)
+add_halide_library(
+    q4_0_dequantize
+    FROM quants.generator
+    GENERATOR symmetric_dequantize
+    PARAMS
+    block_size=32 qmax=8 code_bits=4 rounding=truncate_half_up_with_offset anchor=extreme_signed
+)
+add_halide_library(
+    q4_0_vec_dot
+    FROM quants.generator
+    GENERATOR symmetric_vec_dot
+    FEATURES no_asserts no_bounds_query
+    PARAMS
+    w_kind=symmetric block_size=32 w_qmax=8 w_code_bits=4 w_rounding=truncate_half_up_with_offset
+    w_anchor=extreme_signed a_kind=q8_0 a_qmax=127
+)
+# Q4_1 is affine (min+scale, not symmetric): quant_components.h's
+# AffineQuantize + NibblePack, matching block_q4_1's {fp16 d; fp16 m; qs[16];}.
+add_halide_library(
+    q4_1_quantize
+    FROM quants.generator
+    GENERATOR symmetric_quantize
+    PARAMS kind=affine block_size=32 levels=15 code_bits=4 affine_rounding=clamped_int8
+)
+add_halide_library(
+    q4_1_dequantize
+    FROM quants.generator
+    GENERATOR symmetric_dequantize
+    PARAMS kind=affine block_size=32 levels=15 code_bits=4 affine_rounding=clamped_int8
+)
+add_halide_library(
+    q4_1_vec_dot
+    FROM quants.generator
+    GENERATOR symmetric_vec_dot
+    FEATURES no_asserts no_bounds_query
+    PARAMS
+    w_kind=affine block_size=32 w_levels=15 w_code_bits=4 w_affine_rounding=clamped_int8 a_kind=q8_1
+    a_qmax=127
+)
+# Q5_0 is symmetric like Q4_0, but 5-bit: quant_components.h's
+# SymmetricAffineQuantize + FiveBitPack, matching block_q5_0's
+# {fp16 d; qh[4]; qs[16];}.
+add_halide_library(
+    q5_0_quantize
+    FROM quants.generator
+    GENERATOR symmetric_quantize
+    PARAMS kind=symmetric_5bit block_size=32 qmax=16
+)
+add_halide_library(
+    q5_0_dequantize
+    FROM quants.generator
+    GENERATOR symmetric_dequantize
+    PARAMS kind=symmetric_5bit block_size=32 qmax=16
+)
+add_halide_library(
+    q5_0_vec_dot
+    FROM quants.generator
+    GENERATOR symmetric_vec_dot
+    PARAMS w_kind=symmetric_5bit block_size=32 w_qmax=16 a_kind=q8_0 a_qmax=127
+    FEATURES no_asserts no_bounds_query
+)
+# Q5_1 is affine like Q4_1, but 5-bit: quant_components.h's AffineQuantize +
+# FiveBitPack, matching block_q5_1's {fp16 d; fp16 m; qh[4]; qs[16];}.
+add_halide_library(
+    q5_1_quantize
+    FROM quants.generator
+    GENERATOR symmetric_quantize
+    PARAMS kind=affine_5bit block_size=32 levels=31 affine_rounding=unclamped_uint8
+)
+add_halide_library(
+    q5_1_dequantize
+    FROM quants.generator
+    GENERATOR symmetric_dequantize
+    PARAMS kind=affine_5bit block_size=32 levels=31 affine_rounding=unclamped_uint8
+)
+add_halide_library(
+    q5_1_vec_dot
+    FROM quants.generator
+    GENERATOR symmetric_vec_dot
+    PARAMS
+        w_kind=affine_5bit block_size=32 w_levels=31 w_affine_rounding=unclamped_uint8 a_kind=q8_1
+        a_qmax=127
+    FEATURES no_asserts no_bounds_query
+)
+add_halide_library(
+    q8_0_quantize
+    FROM quants.generator
+    GENERATOR symmetric_quantize
+    PARAMS block_size=32 qmax=127 code_bits=8 rounding=nearest anchor=abs_max
+)
+add_halide_library(
+    q8_0_dequantize
+    FROM quants.generator
+    GENERATOR symmetric_dequantize
+    PARAMS block_size=32 qmax=127 code_bits=8 rounding=nearest anchor=abs_max
+)
+add_halide_library(
+    q8_0_vec_dot
+    FROM quants.generator
+    GENERATOR symmetric_vec_dot
+    FEATURES no_asserts no_bounds_query
+    PARAMS
+    w_kind=symmetric block_size=32 w_qmax=127 w_code_bits=8 w_rounding=nearest w_anchor=abs_max
+    a_kind=q8_0 a_qmax=127
+)
+# Q8_1 is symmetric byte-packed like Q8_0, plus AppendCodeSum's derived 's'
+# field, matching block_q8_1's {fp16 d; fp16 s; qs[32];}. Activation-only
+# (no public to_float, see q8_1_generators.cpp's header comment -- it's
+# gone now, but this scheme's decode() is still used by any vec_dot pairing
+# against Q8_1), so there's no q8_1_dequantize library.
+add_halide_library(
+    q8_1_quantize
+    FROM quants.generator
+    GENERATOR symmetric_quantize
+    PARAMS kind=symmetric_byte_sum block_size=32 qmax=127
+)
+# Q8_K's quantize kernel (activation-only, no dequantize -- see
+# q8_k_generators.cpp) is a GENERATOR_ARGS instantiation of the generic,
+# reusable Approximation-based symmetric_quantize generator (see
+# symmetric_quant_generators.cpp and quant_components.h's
+# AppendGroupSumsInt16/F32Pack/RoundingMode::NearestEvenClampedHigh/
+# ScaleAnchor::ExtremeSignedValueTwoStep).
+add_halide_library(
+    q8_k_quantize
+    FROM quants.generator
+    GENERATOR symmetric_quantize
+    PARAMS kind=q8k block_size=256 qmax=127
+)
+# Q2_K's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based k_quant_quantize/
+# k_quant_dequantize generators (see k_quant_generators.cpp and
+# quant_components.h's KQuantDequantize/NibblePairPack/PlanarBitPack).
+add_halide_library(
+    q2_k_quantize
+    FROM quants.generator
+    GENERATOR k_quant_quantize
+    PARAMS family=q2_k
+)
+add_halide_library(
+    q2_k_dequantize
+    FROM quants.generator
+    GENERATOR k_quant_dequantize
+    PARAMS family=q2_k
+)
+add_halide_library(q2_k_vec_dot FROM quants.generator GENERATOR k_quant_vec_dot PARAMS family=q2_k)
+# Q6_K's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based k_quant_quantize/
+# k_quant_dequantize generators (see k_quant_generators.cpp and
+# quant_components.h's KQuantDequantize/CombinedBitsCode/BytePack).
+add_halide_library(
+    q6_k_quantize
+    FROM quants.generator
+    GENERATOR k_quant_quantize
+    PARAMS family=q6_k
+)
+add_halide_library(
+    q6_k_dequantize
+    FROM quants.generator
+    GENERATOR k_quant_dequantize
+    PARAMS family=q6_k
+)
+add_halide_library(q6_k_vec_dot FROM quants.generator GENERATOR k_quant_vec_dot PARAMS family=q6_k)
+# Q4_K's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based k_quant_quantize/
+# k_quant_dequantize generators (see k_quant_generators.cpp and
+# quant_components.h's KQuantDequantize/K4ScaleMinPack/PlanarBitPack).
+add_halide_library(
+    q4_k_quantize
+    FROM quants.generator
+    GENERATOR k_quant_quantize
+    PARAMS family=q4_k
+)
+add_halide_library(
+    q4_k_dequantize
+    FROM quants.generator
+    GENERATOR k_quant_dequantize
+    PARAMS family=q4_k
+)
+add_halide_library(q4_k_vec_dot FROM quants.generator GENERATOR k_quant_vec_dot PARAMS family=q4_k)
+# Q5_K's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based k_quant_quantize/
+# k_quant_dequantize generators (see k_quant_generators.cpp and
+# quant_components.h's KQuantDequantize/K4ScaleMinPack/CombinedBitsCode).
+add_halide_library(
+    q5_k_quantize
+    FROM quants.generator
+    GENERATOR k_quant_quantize
+    PARAMS family=q5_k
+)
+add_halide_library(
+    q5_k_dequantize
+    FROM quants.generator
+    GENERATOR k_quant_dequantize
+    PARAMS family=q5_k
+)
+add_halide_library(q5_k_vec_dot FROM quants.generator GENERATOR k_quant_vec_dot PARAMS family=q5_k)
+# Q3_K's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based k_quant_quantize/
+# k_quant_dequantize generators (see k_quant_generators.cpp and
+# quant_components.h's KQuantDequantize/Q3KScalePack/CombinedBitsCode).
+add_halide_library(
+    q3_k_quantize
+    FROM quants.generator
+    GENERATOR k_quant_quantize
+    PARAMS family=q3_k
+)
+add_halide_library(
+    q3_k_dequantize
+    FROM quants.generator
+    GENERATOR k_quant_dequantize
+    PARAMS family=q3_k
+)
+add_halide_library(q3_k_vec_dot FROM quants.generator GENERATOR k_quant_vec_dot PARAMS family=q3_k)
+# Q1_0 is symmetric with a mean-abs scale and sign-only (1-bit) codes:
+# quant_components.h's SymmetricAffineQuantize (ScaleAnchor::MeanAbs,
+# RoundingMode::SignOnly) + BitPack, matching block_q1_0's {fp16 d; qs[16];}.
+add_halide_library(
+    q1_0_quantize
+    FROM quants.generator
+    GENERATOR symmetric_quantize
+    PARAMS block_size=128 qmax=1 code_bits=1 rounding=sign_only anchor=mean_abs
+)
+add_halide_library(
+    q1_0_dequantize
+    FROM quants.generator
+    GENERATOR symmetric_dequantize
+    PARAMS block_size=128 qmax=1 code_bits=1 rounding=sign_only anchor=mean_abs
+)
+add_halide_library(
+    q1_0_vec_dot
+    FROM quants.generator
+    GENERATOR symmetric_vec_dot
+    PARAMS
+    w_kind=symmetric block_size=128 w_qmax=1 w_code_bits=1 w_rounding=sign_only w_anchor=mean_abs
+    a_kind=q8_0 a_qmax=127
+)
+# MXFP4/IQ4_NL's quantize/dequantize kernels are GENERATOR_ARGS
+# instantiations of the generic, reusable Approximation-based
+# lookup_table_quantize/lookup_table_dequantize generators (see
+# lookup_table_quant_generators.cpp and quant_components.h's
+# LookupTableQuantize/E8M0Pack) -- not their own per-format C++ Generator
+# classes. vec_dot is still its own hand-rolled, unscheduled Generator (see
+# mxfp4_generators.cpp/iq4_nl_generators.cpp), matching Q4_0/Q8_0's own
+# treatment before they got the symmetric_vec_dot generic generator.
+add_halide_library(
+    mxfp4_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=mxfp4
+)
+add_halide_library(
+    mxfp4_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=mxfp4
+)
+add_halide_library(
+    mxfp4_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=mxfp4
+)
+# NVFP4's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based lookup_table_quantize/
+# lookup_table_dequantize generators (see lookup_table_quant_generators.cpp
+# and quant_components.h's LookupTableQuantize's num_scales/UE4M3Pack).
+add_halide_library(
+    nvfp4_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=nvfp4
+)
+add_halide_library(
+    nvfp4_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=nvfp4
+)
+add_halide_library(
+    nvfp4_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=nvfp4
+)
+add_halide_library(
+    iq4_nl_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=iq4_nl
+)
+add_halide_library(
+    iq4_nl_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq4_nl
+)
+add_halide_library(
+    iq4_nl_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq4_nl
+)
+# IQ4_XS's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based lookup_table_quantize/
+# lookup_table_dequantize generators (see lookup_table_quant_generators.cpp
+# and quant_components.h's IQ4XSDequantize).
+add_halide_library(
+    iq4_xs_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=iq4_xs
+)
+add_halide_library(
+    iq4_xs_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq4_xs
+)
+add_halide_library(
+    iq4_xs_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq4_xs
+)
+# TQ1_0's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based lookup_table_quantize/
+# lookup_table_dequantize generators (see lookup_table_quant_generators.cpp
+# and quant_components.h's LookupTableQuantize/TritPack).
+add_halide_library(
+    tq1_0_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=tq1_0
+)
+add_halide_library(
+    tq1_0_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=tq1_0
+)
+add_halide_library(
+    tq1_0_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=tq1_0
+)
+# TQ2_0's quantize/dequantize kernels are GENERATOR_ARGS instantiations of
+# the generic, reusable Approximation-based lookup_table_quantize/
+# lookup_table_dequantize generators (see lookup_table_quant_generators.cpp
+# and quant_components.h's LookupTableQuantize/PlanarBitPack).
+add_halide_library(
+    tq2_0_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=tq2_0
+)
+add_halide_library(
+    tq2_0_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=tq2_0
+)
+add_halide_library(
+    tq2_0_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=tq2_0
+)
+add_halide_library(
+    iq2_xxs_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq2_xxs
+)
+add_halide_library(
+    iq2_xxs_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq2_xxs
+)
+add_halide_library(
+    iq2_xs_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq2_xs
+)
+add_halide_library(
+    iq2_xs_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq2_xs
+)
+# IQ2_S/IQ3_XXS/IQ3_S's quantize/dequantize kernels are GENERATOR_ARGS
+# instantiations of the generic, reusable Approximation-based
+# lookup_table_quantize/lookup_table_dequantize generators (see
+# lookup_table_quant_generators.cpp and quant_components.h's
+# IQ2SGridDequantize/IQ3XXSGridDequantize/IQ3SGridDequantize).
+add_halide_library(
+    iq2_s_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=iq2_s
+)
+add_halide_library(
+    iq2_s_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq2_s
+)
+add_halide_library(
+    iq2_s_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq2_s
+)
+add_halide_library(
+    iq3_xxs_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=iq3_xxs
+)
+add_halide_library(
+    iq3_xxs_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq3_xxs
+)
+add_halide_library(
+    iq3_xxs_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq3_xxs
+)
+add_halide_library(
+    iq3_s_quantize
+    FROM quants.generator
+    GENERATOR lookup_table_quantize
+    PARAMS family=iq3_s
+)
+add_halide_library(
+    iq3_s_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq3_s
+)
+add_halide_library(
+    iq3_s_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq3_s
+)
+add_halide_library(
+    iq1_s_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq1_s
+)
+add_halide_library(
+    iq1_s_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq1_s
+)
+add_halide_library(
+    iq1_m_dequantize
+    FROM quants.generator
+    GENERATOR lookup_table_dequantize
+    PARAMS family=iq1_m
+)
+add_halide_library(
+    iq1_m_vec_dot
+    FROM quants.generator
+    GENERATOR lookup_table_vec_dot
+    PARAMS family=iq1_m
+)
+add_halide_library(f16_quantize FROM quants.generator GENERATOR f16_quantize)
+add_halide_library(f16_dequantize FROM quants.generator GENERATOR f16_dequantize)
+add_halide_library(bf16_quantize FROM quants.generator GENERATOR bf16_quantize)
+add_halide_library(bf16_dequantize FROM quants.generator GENERATOR bf16_dequantize)
+add_halide_library(q8_0_4x4_quantize_mat FROM quants.generator GENERATOR q8_0_4x4_quantize_mat)
+add_halide_library(q8_0_4x8_quantize_mat FROM quants.generator GENERATOR q8_0_4x8_quantize_mat)
+add_halide_library(q8_k_4x4_quantize_mat FROM quants.generator GENERATOR q8_k_4x4_quantize_mat)
+add_halide_library(q8_k_4x8_quantize_mat FROM quants.generator GENERATOR q8_k_4x8_quantize_mat)
+
+add_halide_library(
+    q4_0_4x4_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q4_0 n_cols=4 blocklen=4
+)
+add_halide_library(
+    q4_0_4x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q4_0 n_cols=4 blocklen=8
+)
+add_halide_library(
+    q4_0_8x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q4_0 n_cols=8 blocklen=8
+)
+add_halide_library(
+    q8_0_4x4_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q8_0 n_cols=4 blocklen=4
+)
+add_halide_library(
+    q8_0_4x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q8_0 n_cols=4 blocklen=8
+)
+add_halide_library(
+    q4_0_4x4_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q4_0 n_cols=4 blocklen=4
+)
+add_halide_library(
+    q4_0_4x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q4_0 n_cols=4 blocklen=8
+)
+add_halide_library(
+    q4_0_8x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q4_0 n_cols=8 blocklen=8
+)
+add_halide_library(
+    q8_0_4x4_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q8_0 n_cols=4 blocklen=4
+)
+add_halide_library(
+    q8_0_4x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q8_0 n_cols=4 blocklen=8
+)
+add_halide_library(
+    iq4_nl_4x4_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=iq4_nl n_cols=4 blocklen=4
+)
+add_halide_library(
+    iq4_nl_8x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=iq4_nl n_cols=8 blocklen=8
+)
+add_halide_library(
+    mxfp4_4x4_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=mxfp4 n_cols=4 blocklen=4
+)
+add_halide_library(
+    mxfp4_8x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=mxfp4 n_cols=8 blocklen=8
+)
+add_halide_library(
+    iq4_nl_4x4_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=iq4_nl n_cols=4 blocklen=4
+)
+add_halide_library(
+    iq4_nl_8x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=iq4_nl n_cols=8 blocklen=8
+)
+add_halide_library(
+    mxfp4_4x4_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=mxfp4 n_cols=4 blocklen=4
+)
+add_halide_library(
+    mxfp4_8x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=mxfp4 n_cols=8 blocklen=8
+)
+add_halide_library(
+    q4_k_8x4_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q4_k n_cols=8 blocklen=4
+)
+add_halide_library(
+    q4_k_8x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q4_k n_cols=8 blocklen=8
+)
+add_halide_library(
+    q4_k_8x4_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q4_k n_cols=8 blocklen=4
+)
+add_halide_library(
+    q4_k_8x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q4_k n_cols=8 blocklen=8
+)
+add_halide_library(
+    q5_k_8x4_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q5_k n_cols=8 blocklen=4
+)
+add_halide_library(
+    q5_k_8x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q5_k n_cols=8 blocklen=8
+)
+add_halide_library(
+    q5_k_8x4_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q5_k n_cols=8 blocklen=4
+)
+add_halide_library(
+    q5_k_8x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q5_k n_cols=8 blocklen=8
+)
+add_halide_library(
+    q6_k_8x4_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q6_k n_cols=8 blocklen=4
+)
+add_halide_library(
+    q6_k_8x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q6_k n_cols=8 blocklen=8
+)
+add_halide_library(
+    q6_k_8x4_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q6_k n_cols=8 blocklen=4
+)
+add_halide_library(
+    q6_k_8x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q6_k n_cols=8 blocklen=8
+)
+add_halide_library(
+    q2_k_8x8_gemv
+    FROM quants.generator
+    GENERATOR repack_gemv
+    PARAMS family=q2_k n_cols=8 blocklen=8
+)
+add_halide_library(
+    q2_k_8x8_gemm
+    FROM quants.generator
+    GENERATOR repack_gemm
+    PARAMS family=q2_k n_cols=8 blocklen=8
+)
+
+# ggml_extern_quantize.cpp implements the extern-stage C functions that
+# several of the quantize pipelines above call out to -- see that file for
+# why. It's the one source in this library that depends on GGML, hence the
+# extra ggml::ggml link below (every other source here is GGML-independent).
+add_library(ggml_quants_halide ggml_quants.cpp ggml_extern_quantize.cpp)
+target_link_libraries(
+    ggml_quants_halide
+    PUBLIC
+    q4_0_quantize q4_0_dequantize q4_0_vec_dot q4_1_quantize q4_1_dequantize q4_1_vec_dot
+    q5_0_quantize q5_0_dequantize q5_0_vec_dot q5_1_quantize q5_1_dequantize q5_1_vec_dot
+    q8_0_quantize q8_0_dequantize q8_0_vec_dot q8_1_quantize q8_k_quantize q2_k_quantize
+    q2_k_dequantize q2_k_vec_dot q6_k_quantize q6_k_dequantize q6_k_vec_dot q4_k_quantize
+    q4_k_dequantize q4_k_vec_dot q5_k_quantize q5_k_dequantize q5_k_vec_dot q3_k_quantize
+    q3_k_dequantize q3_k_vec_dot q1_0_quantize q1_0_dequantize q1_0_vec_dot mxfp4_quantize
+    mxfp4_dequantize mxfp4_vec_dot nvfp4_quantize nvfp4_dequantize nvfp4_vec_dot iq4_nl_quantize
+    iq4_nl_dequantize iq4_nl_vec_dot iq4_xs_quantize iq4_xs_dequantize iq4_xs_vec_dot tq1_0_quantize
+    tq1_0_dequantize tq1_0_vec_dot tq2_0_quantize tq2_0_dequantize tq2_0_vec_dot iq2_xxs_dequantize
+    iq2_xxs_vec_dot iq2_xs_dequantize iq2_xs_vec_dot iq2_s_quantize iq2_s_dequantize iq2_s_vec_dot
+    iq3_xxs_quantize iq3_xxs_dequantize iq3_xxs_vec_dot iq3_s_quantize iq3_s_dequantize
+    iq3_s_vec_dot iq1_s_dequantize iq1_s_vec_dot iq1_m_dequantize iq1_m_vec_dot f16_quantize
+    f16_dequantize bf16_quantize bf16_dequantize q8_0_4x4_quantize_mat q8_0_4x8_quantize_mat
+    q8_k_4x4_quantize_mat q8_k_4x8_quantize_mat q4_0_4x4_gemv q4_0_4x8_gemv q4_0_8x8_gemv
+    q8_0_4x4_gemv q8_0_4x8_gemv q4_0_4x4_gemm q4_0_4x8_gemm q4_0_8x8_gemm q8_0_4x4_gemm
+    q8_0_4x8_gemm iq4_nl_4x4_gemv iq4_nl_8x8_gemv mxfp4_4x4_gemv mxfp4_8x8_gemv iq4_nl_4x4_gemm
+    iq4_nl_8x8_gemm mxfp4_4x4_gemm mxfp4_8x8_gemm q4_k_8x4_gemv q4_k_8x8_gemv q4_k_8x4_gemm
+    q4_k_8x8_gemm q5_k_8x4_gemv q5_k_8x8_gemv q5_k_8x4_gemm q5_k_8x8_gemm q6_k_8x4_gemv
+    q6_k_8x8_gemv q6_k_8x4_gemm q6_k_8x8_gemm q2_k_8x8_gemv q2_k_8x8_gemm ggml::ggml
+)
+target_include_directories(ggml_quants_halide PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+
+# Standalone correctness checks against GGML's own reference, run before (and
+# independently of) wiring these into kernel-bench as providers.
+foreach (t IN ITEMS
+    q4_0
+    q4_1
+    q5_0
+    q5_1
+    q8_0
+    q8_1
+    q8_k
+    q2_k
+    q6_k
+    q4_k
+    q5_k
+    q3_k
+    q1_0
+    mxfp4
+    nvfp4
+    iq4_nl
+    iq4_xs
+    tq1_0
+    tq2_0
+    iq2_xxs
+    iq2_xs
+    iq2_s
+    iq3_xxs
+    iq3_s
+    iq1_s
+    iq1_m
+    f16
+    bf16
+)
+    add_executable(test_${t} test_${t}.cpp)
+    target_link_libraries(test_${t} PRIVATE ggml_quants_halide ggml::ggml)
+    target_include_directories(test_${t} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../src)
+    add_test(NAME ${t}_roundtrip COMMAND test_${t})
+    set_tests_properties(${t}_roundtrip PROPERTIES PASS_REGULAR_EXPRESSION "Success!")
+endforeach ()
diff --git a/apps/ggml/halide/bf16_generators.cpp b/apps/ggml/halide/bf16_generators.cpp
new file mode 100644
index 000000000000..bab89572cb5f
--- /dev/null
+++ b/apps/ggml/halide/bf16_generators.cpp
@@ -0,0 +1,58 @@
+// From-scratch Halide reimplementation of GGML's BF16 quantize/dequantize
+// "kernels" (see src/ggml-impl.h: ggml_compute_fp32_to_bf16 /
+// ggml_compute_bf16_to_fp32 upstream, as of GGML v0.15.3). Like F16, BF16
+// isn't really a quantized format -- it's a 1-element/block cast, no
+// header/payload split -- so this is just Halide's native bfloat16_t cast
+// in both directions.
+//
+// GGML's bf16 encode is round-to-nearest-even truncation of the top 16 bits
+// of the IEEE binary32 representation (with NaNs forced quiet); decode is
+// a plain `bits << 16` reinterpretation. Halide's cast<bfloat16_t>/cast<float>
+// compile to the same IEEE-mandated round-to-nearest-even truncation and
+// zero-extension, so this matches bit-for-bit for all finite inputs (the
+// only divergence possible is NaN payload/quieting, which the synthetic
+// benchmark/test data never produces).
+//
+// This is intentionally unscheduled -- scheduling for performance is a
+// later step.
+
+#include "Halide.h"
+
+using namespace Halide;
+
+namespace {
+
+class BF16DequantizeGenerator : public Generator<BF16DequantizeGenerator> {
+public:
+    // Raw bf16 bit patterns, one uint16 per element (block size 1).
+    Input<Buffer<uint16_t, 1>> x_{"x"};
+    Output<Buffer<float, 1>> y_{"y"};
+
+    void generate() {
+        Var i("i");
+        y_(i) = cast<float>(reinterpret<bfloat16_t>(x_(i)));
+
+        x_.dim(0).set_min(0);
+        y_.dim(0).set_min(0);
+    }
+};
+
+class BF16QuantizeGenerator : public Generator<BF16QuantizeGenerator> {
+public:
+    Input<Buffer<float, 1>> x_{"x"};
+    // Raw bf16 bit patterns, one uint16 per element (block size 1).
+    Output<Buffer<uint16_t, 1>> y_{"y"};
+
+    void generate() {
+        Var i("i");
+        y_(i) = reinterpret<uint16_t>(cast<bfloat16_t>(x_(i)));
+
+        x_.dim(0).set_min(0);
+        y_.dim(0).set_min(0);
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(BF16DequantizeGenerator, bf16_dequantize)
+HALIDE_REGISTER_GENERATOR(BF16QuantizeGenerator, bf16_quantize)
diff --git a/apps/ggml/halide/codec_generator_base.h b/apps/ggml/halide/codec_generator_base.h
new file mode 100644
index 000000000000..1974b3430505
--- /dev/null
+++ b/apps/ggml/halide/codec_generator_base.h
@@ -0,0 +1,119 @@
+#pragma once
+
+// Shared configure()/generate() scaffolding for every *_quant_generators.cpp
+// file's Direction-templated Generator (SymmetricCodecGenerator,
+// LookupTableCodecGenerator, KQuantCodecGenerator): all three build the
+// exact same "real ImageParam -> approximate_by -> compute_offline -> adopt
+// one half as a port" pipeline in configure(), differing only in how their
+// SchemeAndBytes gets built. This factors that shared body out via CRTP
+// (Derived::build_scheme()) -- the same static-polymorphism idiom
+// Halide::Generator<T> itself already uses (see its own `T` template
+// parameter), not a virtual method: the concrete type is always known at
+// compile time, so there's no reason to pay for a vtable. Confirmed safe to
+// insert as a base class between a leaf Generator and Halide::Generator<T>:
+// GeneratorParam/Input/Output discovery is address-range-based (see
+// Generator.cpp's ObjectInstanceRegistry::register_instance/
+// instances_in_range), not declaration-order or hierarchy-position based,
+// so it doesn't matter which class in the chain declares them.
+//
+// Usage:
+//   class FooCodecGenerator : public CodecGeneratorBase<FooCodecGenerator<dir>, dir> {
+//   public:
+//       GeneratorParam<...> whatever{...};
+//       SchemeAndBytes build_scheme() const { return ::build_scheme(whatever); }
+//   };
+
+#include "Halide.h"
+
+#include "quant_components.h"
+
+namespace ggml_halide {
+
+enum class Direction { Quantize,
+                       Dequantize };
+
+// SchemeAndBytes itself now lives in quant_components.h (its `scheme` is
+// held as a polymorphic owning handle -- a single leaf, a Compose, or a
+// TrustedInverse, whichever the format is; see the make_*() factories
+// there) -- moved there so those factories can return it directly instead
+// of every Generator switch hand-summing a byte count alongside a bare
+// scheme.
+
+template<typename Derived, Direction dir>
+class CodecGeneratorBase : public Halide::Generator<Derived> {
+public:
+    void configure() {
+        using namespace Halide;
+        SchemeAndBytes sb = static_cast<Derived *>(this)->build_scheme();
+
+        // A structured scheme's encoded form is a first-class 1-D Type::Struct
+        // block (one struct per block index); an unported one is the flat 2-D
+        // (byte, blk) UInt(8) buffer. block_type.bytes() is the on-disk width in
+        // the struct case -- no separately-threaded block_bytes needed.
+        const bool structured = sb.block_type.is_struct();
+
+        // The "obvious" identity: a real ImageParam (never a placeholder --
+        // that's what lets *both* directions share this one call below)
+        // flowing through unchanged.
+        Var x("x");
+        ImageParam input(Float(32), 1, "x");
+        Func identity("y");
+        identity(x) = input(x);
+
+        ApproximationResult r = Func(input).approximate_by(*sb.scheme, {identity});
+        for (Func h : r.handles) {
+            h.compute_root();
+        }
+
+        // Bind compute_offline() to a properly-named ImageParam of the packed
+        // block's shape up front, instead of letting it mint one named after
+        // whatever internal Func produced r.encoded[0]. Only Dequantize below
+        // adopts it as a port (named "blocks_in" rather than reusing Quantize's
+        // output name "blocks_out"); the two are never both real ports at once,
+        // but both objects always exist.
+        ImageParam blocks_in = structured ? ImageParam(sb.block_type, 1, "blocks_in") : ImageParam(UInt(8), 2, "blocks_in");
+
+        // Severs `identity` from `input`/encode() entirely: `q.offline`
+        // recomputes r.encoded (quantize) from `input`, while `identity`
+        // (post-severance) instead reads from `blocks_in` (dequantize).
+        ComputeOfflineResult q = Pipeline({identity}).compute_offline(r.encoded, {blocks_in});
+
+        if constexpr (dir == Direction::Quantize) {
+            input.dim(0).set_min(0);
+
+            // A thin renamed passthrough gives the compiled Output a clean name
+            // (the way `blocks_in` did for the Input side); Halide inlines it.
+            Func blocks_out("blocks_out");
+            Var byte("byte"), blk("blk");
+            if (structured) {
+                blocks_out(blk) = q.offline.outputs()[0](blk);
+                blocks_out.output_buffer().dim(0).set_min(0);
+            } else {
+                blocks_out(byte, blk) = q.offline.outputs()[0](byte, blk);
+                blocks_out.output_buffer().dim(0).set_bounds(0, sb.block_bytes);
+                blocks_out.output_buffer().dim(1).set_min(0);
+            }
+
+            this->add_input(input);
+            this->add_output(blocks_out);
+        } else {
+            if (structured) {
+                blocks_in.dim(0).set_min(0);
+            } else {
+                blocks_in.dim(0).set_bounds(0, sb.block_bytes);
+                blocks_in.dim(1).set_min(0);
+            }
+            identity.output_buffer().dim(0).set_min(0);
+
+            this->add_input(blocks_in);
+            this->add_output(identity);
+        }
+    }
+
+    void generate() {
+        // Nothing left to do: configure() already built (and, via
+        // add_input/add_output, wired up) the whole pipeline.
+    }
+};
+
+}  // namespace ggml_halide
diff --git a/apps/ggml/halide/f16_generators.cpp b/apps/ggml/halide/f16_generators.cpp
new file mode 100644
index 000000000000..9e5431df20f6
--- /dev/null
+++ b/apps/ggml/halide/f16_generators.cpp
@@ -0,0 +1,55 @@
+// From-scratch Halide reimplementation of GGML's F16 quantize/dequantize
+// "kernels" (see src/ggml.c: ggml_fp32_to_fp16_row / ggml_fp16_to_fp32_row
+// upstream, as of GGML v0.15.3). F16 isn't really a quantized format -- it's
+// a 1-element/block plain IEEE-754 binary16 cast, no header/payload split at
+// all -- so unlike every other type here, this is just Halide's native
+// float16_t cast in both directions.
+//
+// GGML's own conversion (GGML_COMPUTE_FP32_TO_FP16 / _FP16_TO_FP32 in
+// src/ggml-impl.h) is a correctly-rounded (round-to-nearest-even) software
+// IEEE binary16 <-> binary32 conversion; Halide's cast<float16_t>/cast<float>
+// compiles to the same IEEE-mandated conversion, so this matches bit-for-bit.
+//
+// This is intentionally unscheduled -- scheduling for performance is a
+// later step.
+
+#include "Halide.h"
+
+using namespace Halide;
+
+namespace {
+
+class F16DequantizeGenerator : public Generator<F16DequantizeGenerator> {
+public:
+    // Raw fp16 bit patterns, one uint16 per element (block size 1).
+    Input<Buffer<uint16_t, 1>> x_{"x"};
+    Output<Buffer<float, 1>> y_{"y"};
+
+    void generate() {
+        Var i("i");
+        y_(i) = cast<float>(reinterpret<float16_t>(x_(i)));
+
+        x_.dim(0).set_min(0);
+        y_.dim(0).set_min(0);
+    }
+};
+
+class F16QuantizeGenerator : public Generator<F16QuantizeGenerator> {
+public:
+    Input<Buffer<float, 1>> x_{"x"};
+    // Raw fp16 bit patterns, one uint16 per element (block size 1).
+    Output<Buffer<uint16_t, 1>> y_{"y"};
+
+    void generate() {
+        Var i("i");
+        y_(i) = reinterpret<uint16_t>(cast<float16_t>(x_(i)));
+
+        x_.dim(0).set_min(0);
+        y_.dim(0).set_min(0);
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(F16DequantizeGenerator, f16_dequantize)
+HALIDE_REGISTER_GENERATOR(F16QuantizeGenerator, f16_quantize)
diff --git a/apps/ggml/halide/ggml_extern_quantize.cpp b/apps/ggml/halide/ggml_extern_quantize.cpp
new file mode 100644
index 000000000000..519b8fd29878
--- /dev/null
+++ b/apps/ggml/halide/ggml_extern_quantize.cpp
@@ -0,0 +1,116 @@
+// Extern-stage scaffolding for K-quant quantize kernels.
+//
+// GGML's reference quantizer for the K-quant super-block formats (Q2_K,
+// Q3_K, Q4_K, Q5_K, Q6_K) isn't a closed-form scale computation like every
+// other type in this directory -- it runs an iterative, per-sub-block
+// error-minimizing search over ~19 candidate scale factors (see
+// src/ggml-quants.c: make_qx_quants / make_qkx1_quants / make_qkx2_quants /
+// make_q3_quants). Porting that search to Halide is deferred; per the
+// project's current phase, this sets up the Halide extern-stage plumbing
+// now (a Func whose realization is computed by an external C function) and
+// simply calls out to GGML's own public from_float_ref for the actual
+// computation. Dequantize (a pure unpacking operation, no search) is
+// implemented natively in Halide for these types -- see qX_k_generators.cpp.
+//
+// This is the one file in halide/ that depends on GGML's public API --
+// every generator's *body* stays GGML-independent, but this scaffold
+// deliberately borrows GGML's own reference computation for now, to be
+// replaced with a from-scratch Halide search later.
+//
+// Extern-stage ABI: a plain C function taking one halide_buffer_t* per
+// Func argument/output, returning 0 on success. Halide calls it twice per
+// realization: once in "bounds query" mode (host pointers null, dimensions
+// need to be filled in based on the output's already-concrete request) and
+// once for real (host pointers valid, actually compute the data). See
+// test/correctness/extern_bounds_inference.cpp for the reference pattern.
+
+#include <HalideRuntime.h>
+
+#include <ggml.h>
+
+namespace {
+
+int quantize_via_ggml_reference(ggml_type type, halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    if (x_buf->is_bounds_query()) {
+        // out_buf already carries the concrete requested region (its dim[1]
+        // extent is the number of blocks); the input row needed is exactly
+        // that many blocks' worth of elements.
+        const int64_t nb = out_buf->dim[1].extent;
+        x_buf->dim[0].min = 0;
+        x_buf->dim[0].extent = static_cast<int32_t>(nb * ggml_blck_size(type));
+        return 0;
+    }
+
+    const float *x = reinterpret_cast<const float *>(x_buf->host);
+    void *y = reinterpret_cast<void *>(out_buf->host);
+    const int64_t k = x_buf->dim[0].extent;
+
+    ggml_get_type_traits(type)->from_float_ref(x, y, k);
+    return 0;
+}
+
+}  // namespace
+
+extern "C" int q2_k_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_Q2_K, x_buf, out_buf);
+}
+
+extern "C" int q3_k_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_Q3_K, x_buf, out_buf);
+}
+
+extern "C" int q4_k_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_Q4_K, x_buf, out_buf);
+}
+
+extern "C" int q5_k_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_Q5_K, x_buf, out_buf);
+}
+
+extern "C" int q6_k_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_Q6_K, x_buf, out_buf);
+}
+
+// The remaining types below use this same scaffolding for different
+// reasons than the K-quants: MXFP4/NVFP4 derive their scale via a
+// transcendental (log2) or rounding-sensitive fixed-point float format not
+// guaranteed to be bit-reproducible from scratch; IQ4_NL/IQ4_XS run a
+// per-block/sub-block nearest-codeword search with scale refinement; TQ1_0/
+// TQ2_0's byte-packing, while closed-form, is fiddly to unroll in Halide's
+// functional style. All are deferred the same way, for now.
+
+extern "C" int mxfp4_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_MXFP4, x_buf, out_buf);
+}
+
+extern "C" int nvfp4_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_NVFP4, x_buf, out_buf);
+}
+
+extern "C" int iq4_nl_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_IQ4_NL, x_buf, out_buf);
+}
+
+extern "C" int iq4_xs_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_IQ4_XS, x_buf, out_buf);
+}
+
+extern "C" int tq1_0_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_TQ1_0, x_buf, out_buf);
+}
+
+extern "C" int tq2_0_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_TQ2_0, x_buf, out_buf);
+}
+
+extern "C" int iq3_xxs_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_IQ3_XXS, x_buf, out_buf);
+}
+
+extern "C" int iq3_s_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_IQ3_S, x_buf, out_buf);
+}
+
+extern "C" int iq2_s_quantize_via_ggml(halide_buffer_t *x_buf, halide_buffer_t *out_buf) {
+    return quantize_via_ggml_reference(GGML_TYPE_IQ2_S, x_buf, out_buf);
+}
diff --git a/apps/ggml/halide/ggml_quants.cpp b/apps/ggml/halide/ggml_quants.cpp
new file mode 100644
index 000000000000..18acc52c35aa
--- /dev/null
+++ b/apps/ggml/halide/ggml_quants.cpp
@@ -0,0 +1,1783 @@
+#include "ggml_quants.h"
+
+#include <cstdio>
+
+#include "HalideBuffer.h"
+#include "bf16_dequantize.h"
+#include "bf16_quantize.h"
+#include "f16_dequantize.h"
+#include "f16_quantize.h"
+#include "iq1_m_dequantize.h"
+#include "iq1_m_vec_dot.h"
+#include "iq1_s_dequantize.h"
+#include "iq1_s_vec_dot.h"
+#include "iq2_s_dequantize.h"
+#include "iq2_s_quantize.h"
+#include "iq2_s_vec_dot.h"
+#include "iq2_xs_dequantize.h"
+#include "iq2_xs_vec_dot.h"
+#include "iq2_xxs_dequantize.h"
+#include "iq2_xxs_vec_dot.h"
+#include "iq3_s_dequantize.h"
+#include "iq3_s_quantize.h"
+#include "iq3_s_vec_dot.h"
+#include "iq3_xxs_dequantize.h"
+#include "iq3_xxs_quantize.h"
+#include "iq3_xxs_vec_dot.h"
+#include "iq4_nl_4x4_gemm.h"
+#include "iq4_nl_4x4_gemv.h"
+#include "iq4_nl_8x8_gemm.h"
+#include "iq4_nl_8x8_gemv.h"
+#include "iq4_nl_dequantize.h"
+#include "iq4_nl_quantize.h"
+#include "iq4_nl_vec_dot.h"
+#include "iq4_xs_dequantize.h"
+#include "iq4_xs_quantize.h"
+#include "iq4_xs_vec_dot.h"
+#include "mxfp4_4x4_gemm.h"
+#include "mxfp4_4x4_gemv.h"
+#include "mxfp4_8x8_gemm.h"
+#include "mxfp4_8x8_gemv.h"
+#include "mxfp4_dequantize.h"
+#include "mxfp4_quantize.h"
+#include "mxfp4_vec_dot.h"
+#include "nvfp4_dequantize.h"
+#include "nvfp4_quantize.h"
+#include "nvfp4_vec_dot.h"
+#include "q1_0_dequantize.h"
+#include "q1_0_quantize.h"
+#include "q1_0_vec_dot.h"
+#include "q2_k_8x8_gemm.h"
+#include "q2_k_8x8_gemv.h"
+#include "q2_k_dequantize.h"
+#include "q2_k_quantize.h"
+#include "q2_k_vec_dot.h"
+#include "q3_k_dequantize.h"
+#include "q3_k_quantize.h"
+#include "q3_k_vec_dot.h"
+#include "q4_0_4x4_gemm.h"
+#include "q4_0_4x4_gemv.h"
+#include "q4_0_4x8_gemm.h"
+#include "q4_0_4x8_gemv.h"
+#include "q4_0_8x8_gemm.h"
+#include "q4_0_8x8_gemv.h"
+#include "q4_0_dequantize.h"
+#include "q4_0_quantize.h"
+#include "q4_0_vec_dot.h"
+#include "q4_1_dequantize.h"
+#include "q4_1_quantize.h"
+#include "q4_1_vec_dot.h"
+#include "q4_k_8x4_gemm.h"
+#include "q4_k_8x4_gemv.h"
+#include "q4_k_8x8_gemm.h"
+#include "q4_k_8x8_gemv.h"
+#include "q4_k_dequantize.h"
+#include "q4_k_quantize.h"
+#include "q4_k_vec_dot.h"
+#include "q5_0_dequantize.h"
+#include "q5_0_quantize.h"
+#include "q5_0_vec_dot.h"
+#include "q5_1_dequantize.h"
+#include "q5_1_quantize.h"
+#include "q5_1_vec_dot.h"
+#include "q5_k_8x4_gemm.h"
+#include "q5_k_8x4_gemv.h"
+#include "q5_k_8x8_gemm.h"
+#include "q5_k_8x8_gemv.h"
+#include "q5_k_dequantize.h"
+#include "q5_k_quantize.h"
+#include "q5_k_vec_dot.h"
+#include "q6_k_8x4_gemm.h"
+#include "q6_k_8x4_gemv.h"
+#include "q6_k_8x8_gemm.h"
+#include "q6_k_8x8_gemv.h"
+#include "q6_k_dequantize.h"
+#include "q6_k_quantize.h"
+#include "q6_k_vec_dot.h"
+#include "q8_0_4x4_gemm.h"
+#include "q8_0_4x4_gemv.h"
+#include "q8_0_4x4_quantize_mat.h"
+#include "q8_0_4x8_gemm.h"
+#include "q8_0_4x8_gemv.h"
+#include "q8_0_4x8_quantize_mat.h"
+#include "q8_0_dequantize.h"
+#include "q8_0_quantize.h"
+#include "q8_0_vec_dot.h"
+#include "q8_1_quantize.h"
+#include "q8_k_4x4_quantize_mat.h"
+#include "q8_k_4x8_quantize_mat.h"
+#include "q8_k_quantize.h"
+#include "tq1_0_dequantize.h"
+#include "tq1_0_quantize.h"
+#include "tq1_0_vec_dot.h"
+#include "tq2_0_dequantize.h"
+#include "tq2_0_quantize.h"
+#include "tq2_0_vec_dot.h"
+
+using Halide::Runtime::Buffer;
+
+namespace {
+
+void check(int result, const char *what) {
+    if (result != 0) {
+        std::fprintf(stderr, "ggml_quants_halide: %s failed (%d)\n", what, result);
+    }
+}
+
+// A 1-D Type::Struct block buffer over `nb` blocks of `block_bytes` each,
+// wrapping GGML's raw packed bytes at `data`. The struct's ABI tag is
+// {halide_type_struct, bits=8, reserved=block_bytes}; each element is one whole
+// block, dim-0 stride 1 in struct units (see Halide::Type::to_abi()). This is
+// how a Phase-3 struct-typed codec kernel receives GGML's byte layout unchanged.
+Halide::Runtime::Buffer<void, 1> struct_block_buffer(const void *data, int nb, int block_bytes) {
+    halide_type_t ty(halide_type_struct, 8);
+    ty.reserved = static_cast<uint16_t>(block_bytes);
+    halide_dimension_t shape[1] = {{0, nb, 1}};
+    return Halide::Runtime::Buffer<void, 1>(ty, const_cast<void *>(data), 1, shape);
+}
+
+// vec_dot is called once per output element of a matvec, so the argument
+// marshalling is not amortized over anything: at the row lengths GGML actually
+// uses, building three Halide::Runtime::Buffers costs about as much as the dot
+// product itself. These kernels know their shapes exactly, so the vec_dot
+// wrappers below fill a halide_buffer_t in place instead. (The quantize /
+// dequantize / gemv wrappers stay on Buffer -- they run over whole rows or
+// tiles, where the difference is noise.)
+struct StackBuffer {
+    halide_buffer_t buf{};
+    halide_dimension_t dims[2]{};
+
+    // Packed blocks as a 1-D Type::Struct buffer: one struct per block, with the
+    // block width in the type's `reserved` field (see Halide::Type::to_abi()).
+    halide_buffer_t *blocks_struct(const void *data, int nb, int block_bytes) {
+        buf.type = halide_type_t(halide_type_struct, 8);
+        buf.type.reserved = static_cast<uint16_t>(block_bytes);
+        dims[0] = {0, nb, 1, 0};
+        return init(data, 1);
+    }
+
+    // Packed blocks as a 2-D (byte, block) UInt(8) buffer.
+    halide_buffer_t *blocks_bytes(const void *data, int nb, int block_bytes) {
+        buf.type = halide_type_t(halide_type_uint, 8);
+        dims[0] = {0, block_bytes, 1, 0};
+        dims[1] = {0, nb, block_bytes, 0};
+        return init(data, 2);
+    }
+
+    // A gathered 1-D fp16 view of one field within each packed block -- e.g.
+    // Q8_1's stored `s` (scaled code sum) at byte_offset 2 of its 36-byte block.
+    // Zero-copy: the field repeats every block_bytes, so the fp16 stride is
+    // block_bytes/2. Used to sever a stored per-block quantity into a vec_dot.
+    halide_buffer_t *blocks_field_f16(const void *base, int nb, int byte_offset, int block_bytes) {
+        buf.type = halide_type_t(halide_type_float, 16);
+        dims[0] = {0, nb, block_bytes / 2, 0};  // stride in fp16 units
+        return init(static_cast<const uint8_t *>(base) + byte_offset, 1);
+    }
+
+    // The 0-D float32 result.
+    halide_buffer_t *scalar_f32(float *data) {
+        buf.type = halide_type_t(halide_type_float, 32);
+        return init(data, 0);
+    }
+
+private:
+    halide_buffer_t *init(const void *data, int dimensions) {
+        buf.host = const_cast<uint8_t *>(static_cast<const uint8_t *>(data));
+        buf.dimensions = dimensions;
+        buf.dim = dimensions ? dims : nullptr;
+        return &buf;
+    }
+};
+
+}  // namespace
+
+extern "C" {
+
+//
+// Q4_0 -- block size 32, 18 bytes/block (2 delta + 16 packed nibbles).
+//
+
+void ggml_quants_halide_quantize_q4_0(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 2 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    auto blocks = struct_block_buffer(y, nb, kBlockBytes);
+    check(q4_0_quantize(xb, blocks), "q4_0_quantize");
+}
+
+void ggml_quants_halide_dequantize_q4_0(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 2 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    auto blocks = struct_block_buffer(x, nb, kBlockBytes);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q4_0_dequantize(blocks, yb), "q4_0_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q4_0_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 32, kBlockBytesX = 2 + kQK / 2, kBlockBytesY = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    StackBuffer xb, yb, result;
+    check(q4_0_vec_dot(xb.blocks_struct(vx, nb, kBlockBytesX),  // weight: struct-typed
+                       yb.blocks_bytes(vy, nb, kBlockBytesY),
+                       result.scalar_f32(s)),
+          "q4_0_vec_dot");
+}
+
+//
+// Q4_1 -- block size 32, 20 bytes/block (2 delta + 2 min + 16 packed nibbles).
+//
+
+void ggml_quants_halide_quantize_q4_1(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 4 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q4_1_quantize(xb, blocks), "q4_1_quantize");
+}
+
+void ggml_quants_halide_dequantize_q4_1(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 4 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q4_1_dequantize(blocks, yb), "q4_1_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q4_1_q8_1(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 32, kBlockBytesX = 4 + kQK / 2, kBlockBytesY = 4 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    StackBuffer xb, yb, sb, result;
+    check(q4_1_vec_dot(xb.blocks_bytes(vx, nb, kBlockBytesX),
+                       yb.blocks_bytes(vy, nb, kBlockBytesY),
+                       sb.blocks_field_f16(vy, nb, 2, kBlockBytesY),  // Q8_1 stored `s`
+                       result.scalar_f32(s)),
+          "q4_1_vec_dot");
+}
+
+//
+// Q5_0 -- block size 32, 22 bytes/block (2 delta + 4 qh + 16 packed nibbles).
+//
+
+void ggml_quants_halide_quantize_q5_0(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 2 + 4 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    auto blocks = struct_block_buffer(y, nb, kBlockBytes);
+    check(q5_0_quantize(xb, blocks), "q5_0_quantize");
+}
+
+void ggml_quants_halide_dequantize_q5_0(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 2 + 4 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    auto blocks = struct_block_buffer(x, nb, kBlockBytes);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q5_0_dequantize(blocks, yb), "q5_0_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q5_0_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 32, kBlockBytesX = 2 + 4 + kQK / 2, kBlockBytesY = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    StackBuffer xb, yb, result;
+    check(q5_0_vec_dot(xb.blocks_struct(vx, nb, kBlockBytesX),
+                       yb.blocks_bytes(vy, nb, kBlockBytesY),
+                       result.scalar_f32(s)),
+          "q5_0_vec_dot");
+}
+
+//
+// Q5_1 -- block size 32, 24 bytes/block (2 delta + 2 min + 4 qh + 16 packed nibbles).
+//
+
+void ggml_quants_halide_quantize_q5_1(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 4 + 4 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    auto blocks = struct_block_buffer(y, nb, kBlockBytes);
+    check(q5_1_quantize(xb, blocks), "q5_1_quantize");
+}
+
+void ggml_quants_halide_dequantize_q5_1(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 4 + 4 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    auto blocks = struct_block_buffer(x, nb, kBlockBytes);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q5_1_dequantize(blocks, yb), "q5_1_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q5_1_q8_1(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 32, kBlockBytesX = 4 + 4 + kQK / 2, kBlockBytesY = 4 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    StackBuffer xb, yb, sb, result;
+    check(q5_1_vec_dot(xb.blocks_struct(vx, nb, kBlockBytesX),
+                       yb.blocks_bytes(vy, nb, kBlockBytesY),
+                       sb.blocks_field_f16(vy, nb, 2, kBlockBytesY),  // Q8_1 stored `s`
+                       result.scalar_f32(s)),
+          "q5_1_vec_dot");
+}
+
+//
+// Q8_0 -- block size 32, 34 bytes/block (2 delta + 32 int8 values).
+//
+
+void ggml_quants_halide_quantize_q8_0(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 2 + kQK;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    auto blocks = struct_block_buffer(y, nb, kBlockBytes);
+    check(q8_0_quantize(xb, blocks), "q8_0_quantize");
+}
+
+void ggml_quants_halide_dequantize_q8_0(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    auto blocks = struct_block_buffer(x, nb, kBlockBytes);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q8_0_dequantize(blocks, yb), "q8_0_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q8_0_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 32, kBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    StackBuffer xb, yb, result;
+    check(q8_0_vec_dot(xb.blocks_struct(vx, nb, kBlockBytes),  // weight: struct-typed
+                       yb.blocks_bytes(vy, nb, kBlockBytes),
+                       result.scalar_f32(s)),
+          "q8_0_vec_dot");
+}
+
+//
+// Q8_1 -- block size 32, 36 bytes/block (2 delta + 2 sum + 32 int8 values).
+// Quantize only -- GGML has no public dequantize for this activation-only format.
+//
+
+void ggml_quants_halide_quantize_q8_1(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 4 + kQK;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q8_1_quantize(xb, blocks), "q8_1_quantize");
+}
+
+//
+// Q8_K -- superblock size 256, 292 bytes/block (4 float32 delta + 256 int8
+// values + 16 int16 bsums). Quantize only -- GGML has no public dequantize
+// for this activation-only format.
+//
+
+void ggml_quants_halide_quantize_q8_k(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 4 + kQK + (kQK / 16) * 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q8_k_quantize(xb, blocks), "q8_k_quantize");
+}
+
+//
+// Q2_K -- superblock size 256, 84 bytes/block (16 scale/min nibble bytes +
+// 64 packed-2-bit bytes + 2 delta + 2 dmin). Dequantize is native Halide;
+// quantize calls out to GGML's own reference (see ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_q2_k(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 16 + kQK / 4 + 4;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q2_k_quantize(xb, blocks), "q2_k_quantize");
+}
+
+void ggml_quants_halide_dequantize_q2_k(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 16 + kQK / 4 + 4;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q2_k_dequantize(blocks, yb), "q2_k_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q2_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 16 + kQK / 4 + 4, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(q2_k_vec_dot(xb, yb, result), "q2_k_vec_dot");
+}
+
+//
+// Q6_K -- superblock size 256, 210 bytes/block (128 ql + 64 qh + 16 signed
+// int8 scales + 2 delta). Dequantize is native Halide; quantize calls out
+// to GGML's own reference (see ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_q6_k(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = kQK / 2 + kQK / 4 + kQK / 16 + 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q6_k_quantize(xb, blocks), "q6_k_quantize");
+}
+
+void ggml_quants_halide_dequantize_q6_k(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = kQK / 2 + kQK / 4 + kQK / 16 + 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q6_k_dequantize(blocks, yb), "q6_k_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q6_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = kQK / 2 + kQK / 4 + kQK / 16 + 2,
+                  kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(q6_k_vec_dot(xb, yb, result), "q6_k_vec_dot");
+}
+
+//
+// Q4_K -- superblock size 256, 144 bytes/block (2 delta + 2 dmin + 12 packed
+// scale/min bytes + 128 packed-4-bit bytes). Dequantize is native Halide;
+// quantize calls out to GGML's own reference (see ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_q4_k(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 4 + 12 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q4_k_quantize(xb, blocks), "q4_k_quantize");
+}
+
+void ggml_quants_halide_dequantize_q4_k(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 4 + 12 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q4_k_dequantize(blocks, yb), "q4_k_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q4_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 4 + 12 + kQK / 2, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(q4_k_vec_dot(xb, yb, result), "q4_k_vec_dot");
+}
+
+//
+// Q5_K -- superblock size 256, 176 bytes/block (2 delta + 2 dmin + 12 packed
+// scale/min bytes + 32 high-bit bytes + 128 packed-4-bit bytes). Dequantize
+// is native Halide; quantize calls out to GGML's own reference (see
+// ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_q5_k(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 4 + 12 + kQK / 8 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q5_k_quantize(xb, blocks), "q5_k_quantize");
+}
+
+void ggml_quants_halide_dequantize_q5_k(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 4 + 12 + kQK / 8 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q5_k_dequantize(blocks, yb), "q5_k_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q5_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 4 + 12 + kQK / 8 + kQK / 2, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(q5_k_vec_dot(xb, yb, result), "q5_k_vec_dot");
+}
+
+//
+// Q3_K -- superblock size 256, 110 bytes/block (32 hmask + 64 packed-2-bit
+// bytes + 12 packed scale bytes + 2 delta). Dequantize is native Halide;
+// quantize calls out to GGML's own reference (see ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_q3_k(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = kQK / 8 + kQK / 4 + 12 + 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(q3_k_quantize(xb, blocks), "q3_k_quantize");
+}
+
+void ggml_quants_halide_dequantize_q3_k(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = kQK / 8 + kQK / 4 + 12 + 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q3_k_dequantize(blocks, yb), "q3_k_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q3_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = kQK / 8 + kQK / 4 + 12 + 2, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(q3_k_vec_dot(xb, yb, result), "q3_k_vec_dot");
+}
+
+//
+// Q1_0 -- block size 128, 18 bytes/block (2 delta + 16 sign-bit bytes).
+// Closed-form both directions, fully native.
+//
+
+void ggml_quants_halide_quantize_q1_0(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 128, kBlockBytes = 2 + kQK / 8;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    auto blocks = struct_block_buffer(y, nb, kBlockBytes);
+    check(q1_0_quantize(xb, blocks), "q1_0_quantize");
+}
+
+void ggml_quants_halide_dequantize_q1_0(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 128, kBlockBytes = 2 + kQK / 8;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    auto blocks = struct_block_buffer(x, nb, kBlockBytes);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(q1_0_dequantize(blocks, yb), "q1_0_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_q1_0_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc) {
+    // Q1_0's own block size (128) differs from Q8_0's (32) -- unlike the
+    // same-granularity types above, nb must be computed separately per side.
+    constexpr int kQKX = 128, kBlockBytesX = 2 + kQKX / 8;
+    constexpr int kQKY = 32, kBlockBytesY = 2 + kQKY;
+    const int32_t nbx = static_cast<int32_t>(n / kQKX);
+    const int32_t nby = static_cast<int32_t>(n / kQKY);
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nby, kBlockBytesY}};
+    auto xb = struct_block_buffer(vx, nbx, kBlockBytesX);  // weight: struct-typed
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(q1_0_vec_dot(xb, yb, result), "q1_0_vec_dot");
+}
+
+//
+// MXFP4 -- block size 32, 17 bytes/block (1 E8M0 exponent + 16 packed-4-bit
+// codebook-index bytes). Dequantize is native Halide; quantize calls out to
+// GGML's own reference (see ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_mxfp4(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 1 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(mxfp4_quantize(xb, blocks), "mxfp4_quantize");
+}
+
+void ggml_quants_halide_dequantize_mxfp4(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 1 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(mxfp4_dequantize(blocks, yb), "mxfp4_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_mxfp4_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc) {
+    constexpr int kQK = 32, kBlockBytesX = 1 + kQK / 2, kBlockBytesY = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(mxfp4_vec_dot(xb, yb, result), "mxfp4_vec_dot");
+}
+
+//
+// NVFP4 -- block size 64, 36 bytes/block (4 UE4M3 scales + 32 packed-4-bit
+// codebook-index bytes). Dequantize is native Halide; quantize calls out to
+// GGML's own reference (see ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_nvfp4(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 64, kSub = 16, kBlockBytes = kQK / kSub + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(nvfp4_quantize(xb, blocks), "nvfp4_quantize");
+}
+
+void ggml_quants_halide_dequantize_nvfp4(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 64, kSub = 16, kBlockBytes = kQK / kSub + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(nvfp4_dequantize(blocks, yb), "nvfp4_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_nvfp4_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc) {
+    // NVFP4's own block size (64) differs from Q8_0's (32) -- see Q1_0's
+    // vec_dot wrapper above for why nb must be computed separately per side.
+    constexpr int kQKX = 64, kSub = 16, kBlockBytesX = kQKX / kSub + kQKX / 2;
+    constexpr int kQKY = 32, kBlockBytesY = 2 + kQKY;
+    const int32_t nbx = static_cast<int32_t>(n / kQKX);
+    const int32_t nby = static_cast<int32_t>(n / kQKY);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nbx, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nby, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(nvfp4_vec_dot(xb, yb, result), "nvfp4_vec_dot");
+}
+
+//
+// IQ4_NL -- block size 32, 18 bytes/block (2 delta + 16 packed-4-bit
+// codebook-index bytes). Dequantize is native Halide; quantize calls out to
+// GGML's own reference (see ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_iq4_nl(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 2 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(iq4_nl_quantize(xb, blocks), "iq4_nl_quantize");
+}
+
+void ggml_quants_halide_dequantize_iq4_nl(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 2 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq4_nl_dequantize(blocks, yb), "iq4_nl_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq4_nl_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                            size_t by, int nrc) {
+    constexpr int kQK = 32, kBlockBytesX = 2 + kQK / 2, kBlockBytesY = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq4_nl_vec_dot(xb, yb, result), "iq4_nl_vec_dot");
+}
+
+//
+// IQ4_XS -- superblock size 256, 136 bytes/block (2 delta + 2 scales_h + 4
+// scales_l + 128 packed-4-bit codebook-index bytes). Dequantize is native
+// Halide; quantize calls out to GGML's own reference (see
+// ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_iq4_xs(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + 2 + 4 + kQK / 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(iq4_xs_quantize(xb, blocks), "iq4_xs_quantize");
+}
+
+void ggml_quants_halide_dequantize_iq4_xs(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + 2 + 4 + kQK / 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq4_xs_dequantize(blocks, yb), "iq4_xs_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq4_xs_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                            size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 2 + 2 + 4 + kQK / 2, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq4_xs_vec_dot(xb, yb, result), "iq4_xs_vec_dot");
+}
+
+//
+// TQ1_0 -- superblock size 256, 54 bytes/block (48 base-3-packed qs + 4
+// base-3-packed qh + 2 delta). Dequantize is native Halide; quantize calls
+// out to GGML's own reference (see ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_tq1_0(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 54;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(tq1_0_quantize(xb, blocks), "tq1_0_quantize");
+}
+
+void ggml_quants_halide_dequantize_tq1_0(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 54;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(tq1_0_dequantize(blocks, yb), "tq1_0_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_tq1_0_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 54, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(tq1_0_vec_dot(xb, yb, result), "tq1_0_vec_dot");
+}
+
+//
+// TQ2_0 -- superblock size 256, 66 bytes/block (64 packed-2-bit qs + 2
+// delta -- qs before d, unlike every other type here). Dequantize is native
+// Halide; quantize calls out to GGML's own reference (see
+// ggml_extern_quantize.cpp).
+//
+
+void ggml_quants_halide_quantize_tq2_0(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = kQK / 4 + 2;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(tq2_0_quantize(xb, blocks), "tq2_0_quantize");
+}
+
+void ggml_quants_halide_dequantize_tq2_0(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = kQK / 4 + 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(tq2_0_dequantize(blocks, yb), "tq2_0_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_tq2_0_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = kQK / 4 + 2, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(tq2_0_vec_dot(xb, yb, result), "tq2_0_vec_dot");
+}
+
+//
+// IQ2_XXS -- superblock size 256, 66 bytes/block (2 delta + 64 packed qs).
+// Dequantize only -- see ggml_quants.h for why there's no quantize here.
+//
+
+void ggml_quants_halide_dequantize_iq2_xxs(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + kQK / 8 * 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq2_xxs_dequantize(blocks, yb), "iq2_xxs_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq2_xxs_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                             size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 2 + kQK / 8 * 2, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq2_xxs_vec_dot(xb, yb, result), "iq2_xxs_vec_dot");
+}
+
+//
+// IQ2_XS -- superblock size 256, 74 bytes/block. Dequantize only.
+//
+
+void ggml_quants_halide_dequantize_iq2_xs(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + kQK / 8 * 2 + kQK / 32;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq2_xs_dequantize(blocks, yb), "iq2_xs_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq2_xs_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                            size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 2 + kQK / 8 * 2 + kQK / 32, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq2_xs_vec_dot(xb, yb, result), "iq2_xs_vec_dot");
+}
+
+//
+// IQ2_S -- superblock size 256, 82 bytes/block. Dequantize is native
+// Halide; quantize calls out to GGML's own reference.
+//
+
+void ggml_quants_halide_quantize_iq2_s(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + kQK / 8 + kQK / 8 + kQK / 32 + kQK / 32;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(iq2_s_quantize(xb, blocks), "iq2_s_quantize");
+}
+
+void ggml_quants_halide_dequantize_iq2_s(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + kQK / 8 + kQK / 8 + kQK / 32 + kQK / 32;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq2_s_dequantize(blocks, yb), "iq2_s_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq2_s_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 2 + kQK / 8 + kQK / 8 + kQK / 32 + kQK / 32,
+                  kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq2_s_vec_dot(xb, yb, result), "iq2_s_vec_dot");
+}
+
+//
+// IQ3_XXS -- superblock size 256, 98 bytes/block. Dequantize is native
+// Halide; quantize calls out to GGML's own reference.
+//
+
+void ggml_quants_halide_quantize_iq3_xxs(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + 3 * kQK / 8;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(iq3_xxs_quantize(xb, blocks), "iq3_xxs_quantize");
+}
+
+void ggml_quants_halide_dequantize_iq3_xxs(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + 3 * kQK / 8;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq3_xxs_dequantize(blocks, yb), "iq3_xxs_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq3_xxs_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                             size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 2 + 3 * kQK / 8, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq3_xxs_vec_dot(xb, yb, result), "iq3_xxs_vec_dot");
+}
+
+//
+// IQ3_S -- superblock size 256, 110 bytes/block. Dequantize is native
+// Halide; quantize calls out to GGML's own reference.
+//
+
+void ggml_quants_halide_quantize_iq3_s(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + kQK / 4 + kQK / 32 + kQK / 8 + kQK / 64;
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, shape);
+    check(iq3_s_quantize(xb, blocks), "iq3_s_quantize");
+}
+
+void ggml_quants_halide_dequantize_iq3_s(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + kQK / 4 + kQK / 32 + kQK / 8 + kQK / 64;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq3_s_dequantize(blocks, yb), "iq3_s_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq3_s_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 2 + kQK / 4 + kQK / 32 + kQK / 8 + kQK / 64,
+                  kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq3_s_vec_dot(xb, yb, result), "iq3_s_vec_dot");
+}
+
+//
+// IQ1_S -- superblock size 256, 50 bytes/block. Dequantize only.
+//
+
+void ggml_quants_halide_dequantize_iq1_s(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 2 + kQK / 8 + kQK / 16;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq1_s_dequantize(blocks, yb), "iq1_s_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq1_s_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = 2 + kQK / 8 + kQK / 16, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq1_s_vec_dot(xb, yb, result), "iq1_s_vec_dot");
+}
+
+//
+// IQ1_M -- superblock size 256, 56 bytes/block. Dequantize only.
+//
+
+void ggml_quants_halide_dequantize_iq1_m(const void *x, float *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = kQK / 8 + kQK / 16 + kQK / 32;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t shape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(const_cast<uint8_t *>(static_cast<const uint8_t *>(x)), 2, shape);
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(iq1_m_dequantize(blocks, yb), "iq1_m_dequantize");
+}
+
+void ggml_quants_halide_vec_dot_iq1_m_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc) {
+    constexpr int kQK = 256, kBlockBytesX = kQK / 8 + kQK / 16 + kQK / 32, kBlockBytesY = 4 + kQK + (kQK / 16) * 2;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    halide_dimension_t xshape[2] = {{0, kBlockBytesX, 1}, {0, nb, kBlockBytesX}};
+    halide_dimension_t yshape[2] = {{0, kBlockBytesY, 1}, {0, nb, kBlockBytesY}};
+    Buffer<uint8_t, 2> xb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 2, xshape);
+    Buffer<uint8_t, 2> yb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, yshape);
+    Buffer<float, 0> result = Buffer<float, 0>::make_scalar(s);
+    check(iq1_m_vec_dot(xb, yb, result), "iq1_m_vec_dot");
+}
+
+//
+// F16 -- block size 1, 2 bytes/element (plain IEEE binary16 cast, no header).
+//
+
+void ggml_quants_halide_quantize_f16(const float *x, void *y, int64_t k) {
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    Buffer<uint16_t, 1> yb(static_cast<uint16_t *>(y), static_cast<int>(k));
+    check(f16_quantize(xb, yb), "f16_quantize");
+}
+
+void ggml_quants_halide_dequantize_f16(const void *x, float *y, int64_t k) {
+    Buffer<uint16_t, 1> xb(const_cast<uint16_t *>(static_cast<const uint16_t *>(x)), static_cast<int>(k));
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(f16_dequantize(xb, yb), "f16_dequantize");
+}
+
+//
+// BF16 -- block size 1, 2 bytes/element (plain bfloat16 cast, no header).
+//
+
+void ggml_quants_halide_quantize_bf16(const float *x, void *y, int64_t k) {
+    Buffer<float, 1> xb(const_cast<float *>(x), static_cast<int>(k));
+    Buffer<uint16_t, 1> yb(static_cast<uint16_t *>(y), static_cast<int>(k));
+    check(bf16_quantize(xb, yb), "bf16_quantize");
+}
+
+void ggml_quants_halide_dequantize_bf16(const void *x, float *y, int64_t k) {
+    Buffer<uint16_t, 1> xb(const_cast<uint16_t *>(static_cast<const uint16_t *>(x)), static_cast<int>(k));
+    Buffer<float, 1> yb(y, static_cast<int>(k));
+    check(bf16_dequantize(xb, yb), "bf16_dequantize");
+}
+
+//
+// Repack quantize_mat -- interleaves 4 contiguous rows of `k` floats (row r
+// at x[r*k .. r*k+k)) into one packed activation-format block per chunk.
+// `x` is wrapped as a 2-D buffer (dim 0: column-within-row, extent k; dim 1:
+// row, extent 4, stride k) so the generator can address x_(col, row)
+// directly instead of hand-computing `row*k + col`.
+//
+
+void ggml_quants_halide_repack_quantize_mat_q8_0_4x4(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 4 * 2 + kQK * 4;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t xshape[2] = {{0, static_cast<int32_t>(k), 1}, {0, 4, static_cast<int32_t>(k)}};
+    Buffer<float, 2> xb(const_cast<float *>(x), 2, xshape);
+    halide_dimension_t yshape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, yshape);
+    check(q8_0_4x4_quantize_mat(xb, blocks), "q8_0_4x4_quantize_mat");
+}
+
+void ggml_quants_halide_repack_quantize_mat_q8_0_4x8(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 32, kBlockBytes = 4 * 2 + kQK * 4;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t xshape[2] = {{0, static_cast<int32_t>(k), 1}, {0, 4, static_cast<int32_t>(k)}};
+    Buffer<float, 2> xb(const_cast<float *>(x), 2, xshape);
+    halide_dimension_t yshape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, yshape);
+    check(q8_0_4x8_quantize_mat(xb, blocks), "q8_0_4x8_quantize_mat");
+}
+
+void ggml_quants_halide_repack_quantize_mat_q8_k_4x4(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 4 * 4 + kQK * 4 + (kQK / 16) * 4 * 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t xshape[2] = {{0, static_cast<int32_t>(k), 1}, {0, 4, static_cast<int32_t>(k)}};
+    Buffer<float, 2> xb(const_cast<float *>(x), 2, xshape);
+    halide_dimension_t yshape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, yshape);
+    check(q8_k_4x4_quantize_mat(xb, blocks), "q8_k_4x4_quantize_mat");
+}
+
+void ggml_quants_halide_repack_quantize_mat_q8_k_4x8(const float *x, void *y, int64_t k) {
+    constexpr int kQK = 256, kBlockBytes = 4 * 4 + kQK * 4 + (kQK / 16) * 4 * 2;
+    const int32_t nb = static_cast<int32_t>(k / kQK);
+    halide_dimension_t xshape[2] = {{0, static_cast<int32_t>(k), 1}, {0, 4, static_cast<int32_t>(k)}};
+    Buffer<float, 2> xb(const_cast<float *>(x), 2, xshape);
+    halide_dimension_t yshape[2] = {{0, kBlockBytes, 1}, {0, nb, kBlockBytes}};
+    Buffer<uint8_t, 2> blocks(static_cast<uint8_t *>(y), 2, yshape);
+    check(q8_k_4x8_quantize_mat(xb, blocks), "q8_k_4x8_quantize_mat");
+}
+
+//
+// Repack gemv/gemm: dot a repack-interleaved weight matrix against a Q8_0
+// activation row (gemv) or `nr` rows packed 4 at a time (gemm). See
+// repack_gemv_generators.cpp/repack_gemm_generators.cpp for the packed
+// buffer layouts these shapes describe; `bs` (gemm's output row stride) is
+// unused by gemv, matching gemx_fn_t's own nr == 1 convention.
+//
+
+void ggml_quants_halide_repack_gemv_q4_0_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q4_0_4x4_gemv(wb, ab, sb), "q4_0_4x4_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_q4_0_4x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q4_0_4x8_gemv(wb, ab, sb), "q4_0_4x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_q4_0_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 8, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q4_0_8x8_gemv(wb, ab, sb), "q4_0_8x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_q8_0_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 8) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q8_0_4x4_gemv(wb, ab, sb), "q8_0_4x4_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_q8_0_4x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 8) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q8_0_4x8_gemv(wb, ab, sb), "q8_0_4x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemm_q4_0_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q4_0_4x4_gemm(wb, ab, sb), "q4_0_4x4_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_q4_0_4x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q4_0_4x8_gemm(wb, ab, sb), "q4_0_4x8_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_q4_0_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 8, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q4_0_8x8_gemm(wb, ab, sb), "q4_0_8x8_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_q8_0_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 8) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q8_0_4x4_gemm(wb, ab, sb), "q8_0_4x4_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_q8_0_4x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 8) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q8_0_4x8_gemm(wb, ab, sb), "q8_0_4x8_gemm");
+}
+
+//
+// IQ4_NL/MXFP4 repack gemv/gemm: same 3-D packed-buffer shapes as Q4_0's
+// above, but MXFP4's weight header is N bytes (1 E8M0 exponent per column)
+// instead of 2*N (an fp16 delta per column) -- see repack_gemv_generators.cpp.
+//
+
+void ggml_quants_halide_repack_gemv_iq4_nl_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                    int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(iq4_nl_4x4_gemv(wb, ab, sb), "iq4_nl_4x4_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_iq4_nl_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                    int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 8, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(iq4_nl_8x8_gemv(wb, ab, sb), "iq4_nl_8x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_mxfp4_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                   int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(mxfp4_4x4_gemv(wb, ab, sb), "mxfp4_4x4_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_mxfp4_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                   int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 32, kNCols = 8, kWeightBlockBytes = kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 2 + kQK;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(mxfp4_8x8_gemv(wb, ab, sb), "mxfp4_8x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemm_iq4_nl_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                    int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(iq4_nl_4x4_gemm(wb, ab, sb), "iq4_nl_4x4_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_iq4_nl_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                    int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 8, kWeightBlockBytes = 2 * kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(iq4_nl_8x8_gemm(wb, ab, sb), "iq4_nl_8x8_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_mxfp4_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                   int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 4, kWeightBlockBytes = kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(mxfp4_4x4_gemm(wb, ab, sb), "mxfp4_4x4_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_mxfp4_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                   int nr, int nc) {
+    constexpr int kQK = 32, kNCols = 8, kWeightBlockBytes = kNCols + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytes = 2 * kActNRows + kQK * kActNRows;
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {
+        {0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}, {0, nr_groups, kActBlockBytes * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(mxfp4_8x8_gemm(wb, ab, sb), "mxfp4_8x8_gemm");
+}
+
+//
+// Q4_K repack gemv/gemm: 256-element superblocks, always 8 interleaved
+// columns, paired with Q8_K activations. See repack_gemv_generators.cpp/
+// repack_gemm_generators.cpp for the packed buffer layouts.
+//
+
+void ggml_quants_halide_repack_gemv_q4_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 256, kNCols = 8, kWeightBlockBytes = 16 + 16 + 96 + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 4 + kQK + 2 * (kQK / 16);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q4_k_8x4_gemv(wb, ab, sb), "q4_k_8x4_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_q4_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 256, kNCols = 8, kWeightBlockBytes = 16 + 16 + 96 + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 4 + kQK + 2 * (kQK / 16);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q4_k_8x8_gemv(wb, ab, sb), "q4_k_8x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemm_q4_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 256, kNCols = 8, kWeightBlockBytes = 16 + 16 + 96 + (kQK * kNCols * 4) / 8;
+    // block_q8_Kx4's real per-block size is 1168 bytes (16 header + 1024 qs +
+    // 128 bsums) -- that's the actual stride between consecutive blocks in
+    // memory, even though the Halide kernel only ever reads the first 1040
+    // bytes (header + qs; bsums are unused, see repack_gemm_generators.cpp).
+    constexpr int kActNRows = 4, kActBlockBytesUsed = 4 * kActNRows + kQK * kActNRows;
+    constexpr int kActBlockBytesReal = kActBlockBytesUsed + 2 * (kQK / 4);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {{0, kActBlockBytesUsed, 1},
+                                    {0, nb, kActBlockBytesReal},
+                                    {0, nr_groups, kActBlockBytesReal * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q4_k_8x4_gemm(wb, ab, sb), "q4_k_8x4_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_q4_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 256, kNCols = 8, kWeightBlockBytes = 16 + 16 + 96 + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytesUsed = 4 * kActNRows + kQK * kActNRows;
+    constexpr int kActBlockBytesReal = kActBlockBytesUsed + 2 * (kQK / 4);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {{0, kActBlockBytesUsed, 1},
+                                    {0, nb, kActBlockBytesReal},
+                                    {0, nr_groups, kActBlockBytesReal * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q4_k_8x8_gemm(wb, ab, sb), "q4_k_8x8_gemm");
+}
+
+//
+// Q5_K repack gemv/gemm: same 256-element superblock/8-column structure as
+// Q4_K, plus a 256-byte qh (5th bit) array between scales and qs.
+//
+
+void ggml_quants_halide_repack_gemv_q5_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 16 + 96 + (kQK * kNCols) / 8 + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 4 + kQK + 2 * (kQK / 16);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q5_k_8x4_gemv(wb, ab, sb), "q5_k_8x4_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_q5_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 16 + 96 + (kQK * kNCols) / 8 + (kQK * kNCols * 4) / 8;
+    constexpr int kActBlockBytes = 4 + kQK + 2 * (kQK / 16);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q5_k_8x8_gemv(wb, ab, sb), "q5_k_8x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemm_q5_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 16 + 96 + (kQK * kNCols) / 8 + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytesUsed = 4 * kActNRows + kQK * kActNRows;
+    constexpr int kActBlockBytesReal = kActBlockBytesUsed + 2 * (kQK / 4);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {{0, kActBlockBytesUsed, 1},
+                                    {0, nb, kActBlockBytesReal},
+                                    {0, nr_groups, kActBlockBytesReal * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q5_k_8x4_gemm(wb, ab, sb), "q5_k_8x4_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_q5_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 16 + 96 + (kQK * kNCols) / 8 + (kQK * kNCols * 4) / 8;
+    constexpr int kActNRows = 4, kActBlockBytesUsed = 4 * kActNRows + kQK * kActNRows;
+    constexpr int kActBlockBytesReal = kActBlockBytesUsed + 2 * (kQK / 4);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {{0, kActBlockBytesUsed, 1},
+                                    {0, nb, kActBlockBytesReal},
+                                    {0, nr_groups, kActBlockBytesReal * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q5_k_8x8_gemm(wb, ab, sb), "q5_k_8x8_gemm");
+}
+
+//
+// Q6_K repack gemv/gemm: plain signed-int8-per-sub-group scales, no compact
+// bit packing -- see repack_gemv_generators.cpp/repack_gemm_generators.cpp.
+//
+
+void ggml_quants_halide_repack_gemv_q6_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 128 + (kQK * kNCols * 4) / 8 + (kQK * kNCols * 2) / 8;
+    constexpr int kActBlockBytes = 4 + kQK + 2 * (kQK / 16);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q6_k_8x4_gemv(wb, ab, sb), "q6_k_8x4_gemv");
+}
+
+void ggml_quants_halide_repack_gemv_q6_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 128 + (kQK * kNCols * 4) / 8 + (kQK * kNCols * 2) / 8;
+    constexpr int kActBlockBytes = 4 + kQK + 2 * (kQK / 16);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q6_k_8x8_gemv(wb, ab, sb), "q6_k_8x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemm_q6_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 128 + (kQK * kNCols * 4) / 8 + (kQK * kNCols * 2) / 8;
+    constexpr int kActNRows = 4, kActBlockBytesUsed = 4 * kActNRows + kQK * kActNRows;
+    constexpr int kActBlockBytesReal = kActBlockBytesUsed + 2 * (kQK / 4);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {{0, kActBlockBytesUsed, 1},
+                                    {0, nb, kActBlockBytesReal},
+                                    {0, nr_groups, kActBlockBytesReal * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q6_k_8x4_gemm(wb, ab, sb), "q6_k_8x4_gemm");
+}
+
+void ggml_quants_halide_repack_gemm_q6_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 128 + (kQK * kNCols * 4) / 8 + (kQK * kNCols * 2) / 8;
+    constexpr int kActNRows = 4, kActBlockBytesUsed = 4 * kActNRows + kQK * kActNRows;
+    constexpr int kActBlockBytesReal = kActBlockBytesUsed + 2 * (kQK / 4);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {{0, kActBlockBytesUsed, 1},
+                                    {0, nb, kActBlockBytesReal},
+                                    {0, nr_groups, kActBlockBytesReal * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q6_k_8x8_gemm(wb, ab, sb), "q6_k_8x8_gemm");
+}
+
+//
+// Q2_K repack gemv/gemm: only one registered variant (8x8). See
+// repack_gemv_generators.cpp/repack_gemm_generators.cpp.
+//
+
+void ggml_quants_halide_repack_gemv_q2_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    (void)bs;
+    (void)nr;
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 16 + 128 + (kQK * kNCols * 2) / 8;
+    constexpr int kActBlockBytes = 4 + kQK + 2 * (kQK / 16);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[2] = {{0, kActBlockBytes, 1}, {0, nb, kActBlockBytes}};
+    Buffer<uint8_t, 2> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 2, ashape);
+    halide_dimension_t sshape[2] = {{0, kNCols, 1}, {0, nc_groups, kNCols}};
+    Buffer<float, 2> sb(s, 2, sshape);
+    check(q2_k_8x8_gemv(wb, ab, sb), "q2_k_8x8_gemv");
+}
+
+void ggml_quants_halide_repack_gemm_q2_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                  int nr, int nc) {
+    constexpr int kQK = 256, kNCols = 8;
+    constexpr int kWeightBlockBytes = 16 + 16 + 128 + (kQK * kNCols * 2) / 8;
+    constexpr int kActNRows = 4, kActBlockBytesUsed = 4 * kActNRows + kQK * kActNRows;
+    constexpr int kActBlockBytesReal = kActBlockBytesUsed + 2 * (kQK / 4);
+    const int32_t nb = static_cast<int32_t>(n / kQK);
+    const int32_t nc_groups = static_cast<int32_t>(nc / kNCols);
+    const int32_t nr_groups = static_cast<int32_t>(nr / kActNRows);
+    const int32_t bs32 = static_cast<int32_t>(bs);
+    halide_dimension_t wshape[3] = {
+        {0, kWeightBlockBytes, 1}, {0, nb, kWeightBlockBytes}, {0, nc_groups, kWeightBlockBytes * nb}};
+    Buffer<uint8_t, 3> wb(const_cast<uint8_t *>(static_cast<const uint8_t *>(vx)), 3, wshape);
+    halide_dimension_t ashape[3] = {{0, kActBlockBytesUsed, 1},
+                                    {0, nb, kActBlockBytesReal},
+                                    {0, nr_groups, kActBlockBytesReal * nb}};
+    Buffer<uint8_t, 3> ab(const_cast<uint8_t *>(static_cast<const uint8_t *>(vy)), 3, ashape);
+    halide_dimension_t sshape[4] = {
+        {0, kNCols, 1}, {0, nc_groups, kNCols}, {0, kActNRows, bs32}, {0, nr_groups, kActNRows * bs32}};
+    Buffer<float, 4> sb(s, 4, sshape);
+    check(q2_k_8x8_gemm(wb, ab, sb), "q2_k_8x8_gemm");
+}
+
+}  // extern "C"
diff --git a/apps/ggml/halide/ggml_quants.h b/apps/ggml/halide/ggml_quants.h
new file mode 100644
index 000000000000..84704cbda070
--- /dev/null
+++ b/apps/ggml/halide/ggml_quants.h
@@ -0,0 +1,276 @@
+#pragma once
+
+// Plain C ABI for the Halide-generated quantize/dequantize/vec_dot kernels,
+// matching apps/ggml/include/kernel_registry.h's quantize_fn_t/
+// dequantize_fn_t/vec_dot_fn_t signatures exactly (by signature
+// compatibility alone -- no shared header needed between this library and
+// the benchmark harness).
+//
+// Q8_1 has no dequantize entry: it's an activation-only format (GGML itself
+// has no public to_float for it), so there is nothing to implement.
+//
+// Every vec_dot_<x>_<y> function computes the dot product between a row of
+// weight-type x and a row of activation-type y (y is x's GGML vec_dot_type).
+// Both operands flow through the Approximation framework: the generic
+// symmetric/lookup_table/k_quant vec_dot generators splice weight and
+// activation codecs from quant_components.h via approximate_by/compute_offline
+// (see vec_dot_generator_base.h).
+
+#include <cstddef>
+#include <cstdint>
+
+extern "C" {
+
+void ggml_quants_halide_quantize_q4_0(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q4_0(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q4_0_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q4_1(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q4_1(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q4_1_q8_1(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q5_0(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q5_0(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q5_0_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q5_1(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q5_1(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q5_1_q8_1(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q8_0(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q8_0(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q8_0_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q8_1(const float *x, void *y, int64_t k);
+
+void ggml_quants_halide_quantize_q8_k(const float *x, void *y, int64_t k);
+
+// Q2_K, Q6_K: dequantize is a from-scratch Halide implementation; quantize
+// is scaffolding that calls out to GGML's own reference (see
+// ggml_extern_quantize.cpp) pending a from-scratch port of GGML's iterative
+// scale search. vec_dot is from-scratch (against Q8_K activations).
+void ggml_quants_halide_quantize_q2_k(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q2_k(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q2_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q6_k(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q6_k(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q6_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q4_k(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q4_k(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q4_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q5_k(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q5_k(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q5_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+void ggml_quants_halide_quantize_q3_k(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q3_k(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q3_k_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+// Q1_0: closed-form both directions, fully native.
+void ggml_quants_halide_quantize_q1_0(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_q1_0(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_q1_0_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                          size_t by, int nrc);
+
+// MXFP4, NVFP4, IQ4_NL, IQ4_XS, TQ1_0, TQ2_0: dequantize is native Halide;
+// quantize calls out to GGML's own reference (see ggml_extern_quantize.cpp).
+void ggml_quants_halide_quantize_mxfp4(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_mxfp4(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_mxfp4_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc);
+
+void ggml_quants_halide_quantize_nvfp4(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_nvfp4(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_nvfp4_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc);
+
+void ggml_quants_halide_quantize_iq4_nl(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_iq4_nl(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq4_nl_q8_0(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                            size_t by, int nrc);
+
+void ggml_quants_halide_quantize_iq4_xs(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_iq4_xs(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq4_xs_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                            size_t by, int nrc);
+
+void ggml_quants_halide_quantize_tq1_0(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_tq1_0(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_tq1_0_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc);
+
+void ggml_quants_halide_quantize_tq2_0(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_tq2_0(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_tq2_0_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc);
+
+// IQ2_XXS: dequantize only. GGML has no public from_float_ref for this
+// importance-matrix-only codebook type (only a private whole-matrix
+// quantizer -- see providers/ggml_internal_abi.h), so there is no
+// from-scratch quantizer to write against it. vec_dot is still implemented
+// (GGML's own reference quantizer is used to produce test/benchmark input).
+void ggml_quants_halide_dequantize_iq2_xxs(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq2_xxs_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                             size_t by, int nrc);
+
+// IQ2_XS: dequantize only (same reason as IQ2_XXS above).
+void ggml_quants_halide_dequantize_iq2_xs(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq2_xs_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                            size_t by, int nrc);
+
+// IQ2_S, IQ3_XXS, IQ3_S: dequantize is native Halide; quantize calls out to
+// GGML's own reference (see ggml_extern_quantize.cpp) -- these three do
+// have a public from_float_ref, unlike IQ2_XXS/IQ2_XS/IQ1_S/IQ1_M.
+void ggml_quants_halide_quantize_iq2_s(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_iq2_s(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq2_s_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc);
+
+void ggml_quants_halide_quantize_iq3_xxs(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_iq3_xxs(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq3_xxs_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                             size_t by, int nrc);
+
+void ggml_quants_halide_quantize_iq3_s(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_iq3_s(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq3_s_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc);
+
+// IQ1_S, IQ1_M: dequantize only (same reason as IQ2_XXS above).
+void ggml_quants_halide_dequantize_iq1_s(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq1_s_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc);
+
+void ggml_quants_halide_dequantize_iq1_m(const void *x, float *y, int64_t k);
+void ggml_quants_halide_vec_dot_iq1_m_q8_k(int n, float *s, size_t bs, const void *vx, size_t bx, const void *vy,
+                                           size_t by, int nrc);
+
+// F16, BF16: not really "quantized" types -- block size 1, a plain per-
+// element float cast. Both directions are fully native Halide (Halide's
+// built-in float16_t/bfloat16_t casts implement the same IEEE round-to-
+// nearest-even conversions GGML's own reference uses). No vec_dot: not part
+// of the quantized-format vec_dot sweep this directory otherwise covers.
+void ggml_quants_halide_quantize_f16(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_f16(const void *x, float *y, int64_t k);
+
+void ggml_quants_halide_quantize_bf16(const float *x, void *y, int64_t k);
+void ggml_quants_halide_dequantize_bf16(const void *x, float *y, int64_t k);
+
+// Repack quantize_mat kernels: interleave 4 contiguous rows of `k` floats
+// into one packed activation-format block per `k`-sized chunk (see
+// repack_quantize_mat_generators.cpp). Signature-compatible with
+// quantize_fn_t (same as every other quantize_* above) -- these are just
+// keyed by activation format + interleave width rather than weight type,
+// since GGML itself only has 4 distinct quantize_mat implementations shared
+// across every repack weight type (see k_repack_entries in
+// providers/ggml_provider.cpp and this library's registration in
+// providers/halide_provider.cpp).
+void ggml_quants_halide_repack_quantize_mat_q8_0_4x4(const float *x, void *y, int64_t k);
+void ggml_quants_halide_repack_quantize_mat_q8_0_4x8(const float *x, void *y, int64_t k);
+void ggml_quants_halide_repack_quantize_mat_q8_k_4x4(const float *x, void *y, int64_t k);
+void ggml_quants_halide_repack_quantize_mat_q8_k_4x8(const float *x, void *y, int64_t k);
+
+// Repack gemv/gemm: dot a repack-interleaved weight matrix (see
+// repack_quantize_mat_generators.cpp/repack_gemv_generators.cpp for the
+// packed layout) against, respectively, one plain-Q8_0 activation row
+// (gemv, matching gemx_fn_t's nr == 1 case) or `nr` activation rows packed 4
+// at a time by the matching repack_quantize_mat_* kernel above (gemm).
+// Q4_0's own two variants (4x4, 4x8) share one packed-weight byte layout
+// (see repack_gemv_generators.cpp); 8x8 uses a wider one. All 3 use
+// activations quantized by the matching-blocklen quantize_mat kernel, same
+// as ggml_provider.cpp's own k_repack_entries table.
+void ggml_quants_halide_repack_gemv_q4_0_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemv_q4_0_4x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemv_q4_0_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemv_q8_0_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemv_q8_0_4x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+
+void ggml_quants_halide_repack_gemm_q4_0_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q4_0_4x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q4_0_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q8_0_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q8_0_4x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+
+// IQ4_NL/MXFP4: same repack-interleave scheme as Q4_0 (nibble/halves split),
+// but a plain codebook lookup (no XOR trick) for the weight value, and only
+// two interleave widths each (4x4, 8x8 -- no 4x8), matching GGML's own
+// k_repack_entries.
+void ggml_quants_halide_repack_gemv_iq4_nl_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                    int nr, int nc);
+void ggml_quants_halide_repack_gemv_iq4_nl_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                    int nr, int nc);
+void ggml_quants_halide_repack_gemv_mxfp4_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                   int nr, int nc);
+void ggml_quants_halide_repack_gemv_mxfp4_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                   int nr, int nc);
+
+void ggml_quants_halide_repack_gemm_iq4_nl_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                    int nr, int nc);
+void ggml_quants_halide_repack_gemm_iq4_nl_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                    int nr, int nc);
+void ggml_quants_halide_repack_gemm_mxfp4_4x4_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                   int nr, int nc);
+void ggml_quants_halide_repack_gemm_mxfp4_8x8_q8_0(int n, float *s, size_t bs, const void *vx, const void *vy,
+                                                   int nr, int nc);
+
+// Q4_K: 256-element superblocks, always 8 interleaved columns, paired with
+// Q8_K activations.
+void ggml_quants_halide_repack_gemv_q4_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemv_q4_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q4_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q4_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+
+// Q5_K: same 256-element superblock/8-column structure as Q4_K, plus a 5th
+// (high) bit array.
+void ggml_quants_halide_repack_gemv_q5_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemv_q5_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q5_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q5_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+
+// Q6_K: plain signed-int8-per-sub-group scales (no compact bit packing).
+void ggml_quants_halide_repack_gemv_q6_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemv_q6_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q6_k_8x4_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q6_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+
+// Q2_K: only one registered variant (8x8 -- GGML has no ARM 8x4 path).
+void ggml_quants_halide_repack_gemv_q2_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+void ggml_quants_halide_repack_gemm_q2_k_8x8_q8_k(int n, float *s, size_t bs, const void *vx, const void *vy, int nr,
+                                                  int nc);
+}
diff --git a/apps/ggml/halide/iq_grids_data.h b/apps/ggml/halide/iq_grids_data.h
new file mode 100644
index 000000000000..9076fe4ec162
--- /dev/null
+++ b/apps/ggml/halide/iq_grids_data.h
@@ -0,0 +1,4770 @@
+// Constant lookup tables copied verbatim from GGML's src/ggml-common.h
+// (commit eced84c86f8b012c752c016f7fe789adea168e1e / v0.15.3), used by the
+// codebook-based IQ2/IQ3/IQ1 dequantize generators. These are fixed,
+// published codebook constants, not derived logic -- transcribed exactly,
+// not "included" from GGML (no GGML headers are used elsewhere in this
+// directory's generators).
+#pragma once
+#include <cstdint>
+namespace iq_grids {
+constexpr uint8_t kmask_iq2xs[8] = {
+    1,
+    2,
+    4,
+    8,
+    16,
+    32,
+    64,
+    128,
+};
+constexpr uint8_t ksigns_iq2xs[128] = {
+    0,
+    129,
+    130,
+    3,
+    132,
+    5,
+    6,
+    135,
+    136,
+    9,
+    10,
+    139,
+    12,
+    141,
+    142,
+    15,
+    144,
+    17,
+    18,
+    147,
+    20,
+    149,
+    150,
+    23,
+    24,
+    153,
+    154,
+    27,
+    156,
+    29,
+    30,
+    159,
+    160,
+    33,
+    34,
+    163,
+    36,
+    165,
+    166,
+    39,
+    40,
+    169,
+    170,
+    43,
+    172,
+    45,
+    46,
+    175,
+    48,
+    177,
+    178,
+    51,
+    180,
+    53,
+    54,
+    183,
+    184,
+    57,
+    58,
+    187,
+    60,
+    189,
+    190,
+    63,
+    192,
+    65,
+    66,
+    195,
+    68,
+    197,
+    198,
+    71,
+    72,
+    201,
+    202,
+    75,
+    204,
+    77,
+    78,
+    207,
+    80,
+    209,
+    210,
+    83,
+    212,
+    85,
+    86,
+    215,
+    216,
+    89,
+    90,
+    219,
+    92,
+    221,
+    222,
+    95,
+    96,
+    225,
+    226,
+    99,
+    228,
+    101,
+    102,
+    231,
+    232,
+    105,
+    106,
+    235,
+    108,
+    237,
+    238,
+    111,
+    240,
+    113,
+    114,
+    243,
+    116,
+    245,
+    246,
+    119,
+    120,
+    249,
+    250,
+    123,
+    252,
+    125,
+    126,
+    255,
+};
+constexpr uint64_t iq2xxs_grid[256] = {
+    0x0808080808080808,
+    0x080808080808082b,
+    0x0808080808081919,
+    0x0808080808082b08,
+    0x0808080808082b2b,
+    0x0808080808190819,
+    0x0808080808191908,
+    0x08080808082b0808,
+    0x08080808082b082b,
+    0x08080808082b2b08,
+    0x08080808082b2b2b,
+    0x0808080819080819,
+    0x0808080819081908,
+    0x0808080819190808,
+    0x0808080819192b08,
+    0x08080808192b0819,
+    0x08080808192b1908,
+    0x080808082b080808,
+    0x080808082b08082b,
+    0x080808082b082b2b,
+    0x080808082b2b082b,
+    0x0808081908080819,
+    0x0808081908081908,
+    0x0808081908190808,
+    0x0808081908191919,
+    0x0808081919080808,
+    0x080808192b081908,
+    0x080808192b192b08,
+    0x0808082b08080808,
+    0x0808082b0808082b,
+    0x0808082b082b082b,
+    0x0808082b2b08082b,
+    0x0808190808080819,
+    0x0808190808081908,
+    0x0808190808190808,
+    0x08081908082b0819,
+    0x08081908082b1908,
+    0x0808190819080808,
+    0x080819081908082b,
+    0x0808190819082b08,
+    0x08081908192b0808,
+    0x080819082b080819,
+    0x080819082b081908,
+    0x080819082b190808,
+    0x080819082b2b1908,
+    0x0808191908080808,
+    0x080819190808082b,
+    0x0808191908082b08,
+    0x08081919082b0808,
+    0x080819191908192b,
+    0x08081919192b2b19,
+    0x080819192b080808,
+    0x080819192b190819,
+    0x0808192b08082b19,
+    0x0808192b08190808,
+    0x0808192b19080808,
+    0x0808192b2b081908,
+    0x0808192b2b2b1908,
+    0x08082b0808080808,
+    0x08082b0808081919,
+    0x08082b0808082b08,
+    0x08082b0808191908,
+    0x08082b08082b2b08,
+    0x08082b0819080819,
+    0x08082b0819081908,
+    0x08082b0819190808,
+    0x08082b081919082b,
+    0x08082b082b082b08,
+    0x08082b1908081908,
+    0x08082b1919080808,
+    0x08082b2b0808082b,
+    0x08082b2b08191908,
+    0x0819080808080819,
+    0x0819080808081908,
+    0x0819080808190808,
+    0x08190808082b0819,
+    0x0819080819080808,
+    0x08190808192b0808,
+    0x081908082b081908,
+    0x081908082b190808,
+    0x081908082b191919,
+    0x0819081908080808,
+    0x0819081908082b08,
+    0x08190819082b0808,
+    0x0819081919190808,
+    0x0819081919192b2b,
+    0x081908192b080808,
+    0x0819082b082b1908,
+    0x0819082b19081919,
+    0x0819190808080808,
+    0x0819190808082b08,
+    0x08191908082b0808,
+    0x08191908082b1919,
+    0x0819190819082b19,
+    0x081919082b080808,
+    0x0819191908192b08,
+    0x08191919192b082b,
+    0x0819192b08080808,
+    0x0819192b0819192b,
+    0x08192b0808080819,
+    0x08192b0808081908,
+    0x08192b0808190808,
+    0x08192b0819080808,
+    0x08192b082b080819,
+    0x08192b1908080808,
+    0x08192b1908081919,
+    0x08192b192b2b0808,
+    0x08192b2b19190819,
+    0x082b080808080808,
+    0x082b08080808082b,
+    0x082b080808082b2b,
+    0x082b080819081908,
+    0x082b0808192b0819,
+    0x082b08082b080808,
+    0x082b08082b08082b,
+    0x082b0819082b2b19,
+    0x082b081919082b08,
+    0x082b082b08080808,
+    0x082b082b0808082b,
+    0x082b190808080819,
+    0x082b190808081908,
+    0x082b190808190808,
+    0x082b190819080808,
+    0x082b19081919192b,
+    0x082b191908080808,
+    0x082b191919080819,
+    0x082b1919192b1908,
+    0x082b192b2b190808,
+    0x082b2b0808082b08,
+    0x082b2b08082b0808,
+    0x082b2b082b191908,
+    0x082b2b2b19081908,
+    0x1908080808080819,
+    0x1908080808081908,
+    0x1908080808190808,
+    0x1908080808192b08,
+    0x19080808082b0819,
+    0x19080808082b1908,
+    0x1908080819080808,
+    0x1908080819082b08,
+    0x190808081919192b,
+    0x19080808192b0808,
+    0x190808082b080819,
+    0x190808082b081908,
+    0x190808082b190808,
+    0x1908081908080808,
+    0x19080819082b0808,
+    0x19080819192b0819,
+    0x190808192b080808,
+    0x190808192b081919,
+    0x1908082b08080819,
+    0x1908082b08190808,
+    0x1908082b19082b08,
+    0x1908082b1919192b,
+    0x1908082b192b2b08,
+    0x1908190808080808,
+    0x1908190808082b08,
+    0x19081908082b0808,
+    0x190819082b080808,
+    0x190819082b192b19,
+    0x190819190819082b,
+    0x19081919082b1908,
+    0x1908192b08080808,
+    0x19082b0808080819,
+    0x19082b0808081908,
+    0x19082b0808190808,
+    0x19082b0819080808,
+    0x19082b0819081919,
+    0x19082b1908080808,
+    0x19082b1919192b08,
+    0x19082b19192b0819,
+    0x19082b192b08082b,
+    0x19082b2b19081919,
+    0x19082b2b2b190808,
+    0x1919080808080808,
+    0x1919080808082b08,
+    0x1919080808190819,
+    0x1919080808192b19,
+    0x19190808082b0808,
+    0x191908082b080808,
+    0x191908082b082b08,
+    0x1919081908081908,
+    0x191908191908082b,
+    0x191908192b2b1908,
+    0x1919082b2b190819,
+    0x191919082b190808,
+    0x191919082b19082b,
+    0x1919191908082b2b,
+    0x1919192b08080819,
+    0x1919192b19191908,
+    0x19192b0808080808,
+    0x19192b0808190819,
+    0x19192b0808192b19,
+    0x19192b08192b1908,
+    0x19192b1919080808,
+    0x19192b2b08082b08,
+    0x192b080808081908,
+    0x192b080808190808,
+    0x192b080819080808,
+    0x192b0808192b2b08,
+    0x192b081908080808,
+    0x192b081919191919,
+    0x192b082b08192b08,
+    0x192b082b192b0808,
+    0x192b190808080808,
+    0x192b190808081919,
+    0x192b191908190808,
+    0x192b19190819082b,
+    0x192b19192b081908,
+    0x192b2b081908082b,
+    0x2b08080808080808,
+    0x2b0808080808082b,
+    0x2b08080808082b2b,
+    0x2b08080819080819,
+    0x2b0808082b08082b,
+    0x2b08081908081908,
+    0x2b08081908192b08,
+    0x2b08081919080808,
+    0x2b08082b08190819,
+    0x2b08190808080819,
+    0x2b08190808081908,
+    0x2b08190808190808,
+    0x2b08190808191919,
+    0x2b08190819080808,
+    0x2b081908192b0808,
+    0x2b08191908080808,
+    0x2b0819191908192b,
+    0x2b0819192b191908,
+    0x2b08192b08082b19,
+    0x2b08192b19080808,
+    0x2b08192b192b0808,
+    0x2b082b080808082b,
+    0x2b082b1908081908,
+    0x2b082b2b08190819,
+    0x2b19080808081908,
+    0x2b19080808190808,
+    0x2b190808082b1908,
+    0x2b19080819080808,
+    0x2b1908082b2b0819,
+    0x2b1908190819192b,
+    0x2b1908192b080808,
+    0x2b19082b19081919,
+    0x2b19190808080808,
+    0x2b191908082b082b,
+    0x2b19190819081908,
+    0x2b19191919190819,
+    0x2b192b082b080819,
+    0x2b192b19082b0808,
+    0x2b2b08080808082b,
+    0x2b2b080819190808,
+    0x2b2b08082b081919,
+    0x2b2b081908082b19,
+    0x2b2b082b08080808,
+    0x2b2b190808192b08,
+    0x2b2b2b0819190808,
+    0x2b2b2b1908081908,
+};
+constexpr uint64_t iq2xs_grid[512] = {
+    0x0808080808080808,
+    0x080808080808082b,
+    0x0808080808081919,
+    0x0808080808082b08,
+    0x0808080808082b2b,
+    0x0808080808190819,
+    0x0808080808191908,
+    0x080808080819192b,
+    0x0808080808192b19,
+    0x08080808082b0808,
+    0x08080808082b082b,
+    0x08080808082b1919,
+    0x08080808082b2b08,
+    0x0808080819080819,
+    0x0808080819081908,
+    0x080808081908192b,
+    0x0808080819082b19,
+    0x0808080819190808,
+    0x080808081919082b,
+    0x0808080819191919,
+    0x0808080819192b08,
+    0x08080808192b0819,
+    0x08080808192b1908,
+    0x080808082b080808,
+    0x080808082b08082b,
+    0x080808082b081919,
+    0x080808082b082b08,
+    0x080808082b190819,
+    0x080808082b191908,
+    0x080808082b192b19,
+    0x080808082b2b0808,
+    0x0808081908080819,
+    0x0808081908081908,
+    0x080808190808192b,
+    0x0808081908082b19,
+    0x0808081908190808,
+    0x080808190819082b,
+    0x0808081908191919,
+    0x0808081908192b08,
+    0x0808081908192b2b,
+    0x08080819082b0819,
+    0x08080819082b1908,
+    0x0808081919080808,
+    0x080808191908082b,
+    0x0808081919081919,
+    0x0808081919082b08,
+    0x0808081919190819,
+    0x0808081919191908,
+    0x08080819192b0808,
+    0x08080819192b2b08,
+    0x080808192b080819,
+    0x080808192b081908,
+    0x080808192b190808,
+    0x0808082b08080808,
+    0x0808082b0808082b,
+    0x0808082b08081919,
+    0x0808082b08082b08,
+    0x0808082b08190819,
+    0x0808082b08191908,
+    0x0808082b082b0808,
+    0x0808082b19080819,
+    0x0808082b19081908,
+    0x0808082b19190808,
+    0x0808082b19191919,
+    0x0808082b2b080808,
+    0x0808082b2b082b2b,
+    0x0808190808080819,
+    0x0808190808081908,
+    0x080819080808192b,
+    0x0808190808082b19,
+    0x0808190808190808,
+    0x080819080819082b,
+    0x0808190808191919,
+    0x0808190808192b08,
+    0x08081908082b0819,
+    0x08081908082b1908,
+    0x0808190819080808,
+    0x080819081908082b,
+    0x0808190819081919,
+    0x0808190819082b08,
+    0x0808190819190819,
+    0x0808190819191908,
+    0x080819081919192b,
+    0x08081908192b0808,
+    0x080819082b080819,
+    0x080819082b081908,
+    0x080819082b190808,
+    0x0808191908080808,
+    0x080819190808082b,
+    0x0808191908081919,
+    0x0808191908082b08,
+    0x0808191908190819,
+    0x0808191908191908,
+    0x08081919082b0808,
+    0x0808191919080819,
+    0x0808191919081908,
+    0x0808191919190808,
+    0x08081919192b0819,
+    0x080819192b080808,
+    0x0808192b08080819,
+    0x0808192b08081908,
+    0x0808192b08190808,
+    0x0808192b082b192b,
+    0x0808192b19080808,
+    0x0808192b1908082b,
+    0x0808192b2b081908,
+    0x08082b0808080808,
+    0x08082b080808082b,
+    0x08082b0808081919,
+    0x08082b0808082b08,
+    0x08082b0808082b2b,
+    0x08082b0808190819,
+    0x08082b0808191908,
+    0x08082b08082b0808,
+    0x08082b08082b1919,
+    0x08082b0819080819,
+    0x08082b0819081908,
+    0x08082b0819190808,
+    0x08082b0819192b08,
+    0x08082b082b080808,
+    0x08082b082b2b0808,
+    0x08082b082b2b2b2b,
+    0x08082b1908080819,
+    0x08082b1908081908,
+    0x08082b1908190808,
+    0x08082b1919080808,
+    0x08082b192b080819,
+    0x08082b192b082b19,
+    0x08082b2b08080808,
+    0x08082b2b082b0808,
+    0x08082b2b082b2b08,
+    0x08082b2b2b19192b,
+    0x08082b2b2b2b0808,
+    0x0819080808080819,
+    0x0819080808081908,
+    0x081908080808192b,
+    0x0819080808082b19,
+    0x0819080808190808,
+    0x081908080819082b,
+    0x0819080808191919,
+    0x0819080808192b08,
+    0x08190808082b0819,
+    0x08190808082b1908,
+    0x0819080819080808,
+    0x081908081908082b,
+    0x0819080819081919,
+    0x0819080819082b08,
+    0x0819080819190819,
+    0x0819080819191908,
+    0x08190808192b0808,
+    0x08190808192b2b2b,
+    0x081908082b080819,
+    0x081908082b081908,
+    0x081908082b190808,
+    0x0819081908080808,
+    0x081908190808082b,
+    0x0819081908081919,
+    0x0819081908082b08,
+    0x0819081908190819,
+    0x0819081908191908,
+    0x08190819082b0808,
+    0x0819081919080819,
+    0x0819081919081908,
+    0x0819081919190808,
+    0x081908192b080808,
+    0x081908192b191908,
+    0x081908192b19192b,
+    0x0819082b08080819,
+    0x0819082b08081908,
+    0x0819082b0808192b,
+    0x0819082b08190808,
+    0x0819082b19080808,
+    0x0819082b192b0808,
+    0x0819190808080808,
+    0x081919080808082b,
+    0x0819190808081919,
+    0x0819190808082b08,
+    0x0819190808190819,
+    0x0819190808191908,
+    0x08191908082b0808,
+    0x0819190819080819,
+    0x0819190819081908,
+    0x0819190819082b19,
+    0x0819190819190808,
+    0x08191908192b1908,
+    0x081919082b080808,
+    0x0819191908080819,
+    0x0819191908081908,
+    0x0819191908190808,
+    0x0819191919080808,
+    0x0819192b08080808,
+    0x0819192b08191908,
+    0x0819192b19082b19,
+    0x08192b0808080819,
+    0x08192b0808081908,
+    0x08192b0808190808,
+    0x08192b080819082b,
+    0x08192b0819080808,
+    0x08192b0819191908,
+    0x08192b082b08192b,
+    0x08192b1908080808,
+    0x08192b1908081919,
+    0x08192b19192b192b,
+    0x08192b2b19190819,
+    0x08192b2b2b2b2b19,
+    0x082b080808080808,
+    0x082b08080808082b,
+    0x082b080808081919,
+    0x082b080808082b08,
+    0x082b080808082b2b,
+    0x082b080808190819,
+    0x082b080808191908,
+    0x082b0808082b0808,
+    0x082b080819080819,
+    0x082b080819081908,
+    0x082b080819190808,
+    0x082b08082b080808,
+    0x082b08082b2b0808,
+    0x082b081908080819,
+    0x082b081908081908,
+    0x082b081908190808,
+    0x082b081919080808,
+    0x082b081919082b08,
+    0x082b0819192b1919,
+    0x082b082b08080808,
+    0x082b082b082b082b,
+    0x082b082b2b080808,
+    0x082b082b2b2b2b08,
+    0x082b190808080819,
+    0x082b190808081908,
+    0x082b190808190808,
+    0x082b1908082b2b19,
+    0x082b190819080808,
+    0x082b191908080808,
+    0x082b191919080819,
+    0x082b19191919082b,
+    0x082b19192b192b19,
+    0x082b192b08080819,
+    0x082b192b08192b2b,
+    0x082b192b2b2b192b,
+    0x082b2b0808080808,
+    0x082b2b0808082b08,
+    0x082b2b0808082b2b,
+    0x082b2b08082b0808,
+    0x082b2b0819191919,
+    0x082b2b082b082b08,
+    0x082b2b082b2b082b,
+    0x082b2b19192b2b08,
+    0x082b2b192b190808,
+    0x082b2b2b08082b08,
+    0x082b2b2b082b0808,
+    0x082b2b2b2b08082b,
+    0x082b2b2b2b082b08,
+    0x082b2b2b2b082b2b,
+    0x1908080808080819,
+    0x1908080808081908,
+    0x190808080808192b,
+    0x1908080808082b19,
+    0x1908080808190808,
+    0x190808080819082b,
+    0x1908080808191919,
+    0x1908080808192b08,
+    0x19080808082b0819,
+    0x19080808082b1908,
+    0x1908080819080808,
+    0x190808081908082b,
+    0x1908080819081919,
+    0x1908080819082b08,
+    0x1908080819082b2b,
+    0x1908080819190819,
+    0x1908080819191908,
+    0x19080808192b0808,
+    0x19080808192b1919,
+    0x190808082b080819,
+    0x190808082b081908,
+    0x190808082b190808,
+    0x1908081908080808,
+    0x190808190808082b,
+    0x1908081908081919,
+    0x1908081908082b08,
+    0x1908081908190819,
+    0x1908081908191908,
+    0x19080819082b0808,
+    0x1908081919080819,
+    0x1908081919081908,
+    0x1908081919190808,
+    0x190808192b080808,
+    0x190808192b081919,
+    0x190808192b2b082b,
+    0x1908082b08080819,
+    0x1908082b08081908,
+    0x1908082b08190808,
+    0x1908082b0819082b,
+    0x1908082b082b2b19,
+    0x1908082b19080808,
+    0x1908190808080808,
+    0x190819080808082b,
+    0x1908190808081919,
+    0x1908190808082b08,
+    0x1908190808190819,
+    0x1908190808191908,
+    0x1908190808192b19,
+    0x19081908082b0808,
+    0x1908190819080819,
+    0x1908190819081908,
+    0x1908190819190808,
+    0x190819082b080808,
+    0x190819082b191908,
+    0x1908191908080819,
+    0x1908191908081908,
+    0x1908191908190808,
+    0x19081919082b1908,
+    0x1908191919080808,
+    0x190819192b192b2b,
+    0x1908192b08080808,
+    0x1908192b08082b2b,
+    0x1908192b19081908,
+    0x1908192b19190808,
+    0x19082b0808080819,
+    0x19082b0808081908,
+    0x19082b0808190808,
+    0x19082b0819080808,
+    0x19082b0819081919,
+    0x19082b0819191908,
+    0x19082b08192b082b,
+    0x19082b1908080808,
+    0x19082b1908190819,
+    0x19082b1919081908,
+    0x19082b1919190808,
+    0x19082b19192b2b19,
+    0x19082b2b08081908,
+    0x1919080808080808,
+    0x191908080808082b,
+    0x1919080808081919,
+    0x1919080808082b08,
+    0x1919080808190819,
+    0x1919080808191908,
+    0x19190808082b0808,
+    0x19190808082b2b08,
+    0x1919080819080819,
+    0x1919080819081908,
+    0x1919080819190808,
+    0x191908082b080808,
+    0x1919081908080819,
+    0x1919081908081908,
+    0x1919081908190808,
+    0x1919081908191919,
+    0x1919081919080808,
+    0x191908191908082b,
+    0x1919082b08080808,
+    0x1919082b19081908,
+    0x1919082b2b2b2b2b,
+    0x1919190808080819,
+    0x1919190808081908,
+    0x1919190808190808,
+    0x19191908082b0819,
+    0x1919190819080808,
+    0x19191908192b0808,
+    0x191919082b080819,
+    0x191919082b2b0819,
+    0x1919191908080808,
+    0x1919191908082b08,
+    0x191919192b080808,
+    0x191919192b082b08,
+    0x1919192b082b0819,
+    0x1919192b192b2b08,
+    0x1919192b2b2b0819,
+    0x19192b0808080808,
+    0x19192b0808191908,
+    0x19192b0819080819,
+    0x19192b0819190808,
+    0x19192b082b192b19,
+    0x19192b1908192b2b,
+    0x19192b1919080808,
+    0x19192b191908082b,
+    0x19192b2b2b081919,
+    0x192b080808080819,
+    0x192b080808081908,
+    0x192b080808190808,
+    0x192b080819080808,
+    0x192b080819191908,
+    0x192b0808192b082b,
+    0x192b08082b08192b,
+    0x192b08082b2b2b19,
+    0x192b081908080808,
+    0x192b082b082b1908,
+    0x192b082b19082b2b,
+    0x192b082b2b19082b,
+    0x192b190808080808,
+    0x192b19080819192b,
+    0x192b191908190808,
+    0x192b191919080808,
+    0x192b191919081919,
+    0x192b19192b2b1908,
+    0x192b2b0808080819,
+    0x192b2b08192b2b2b,
+    0x192b2b19082b1919,
+    0x192b2b2b0808192b,
+    0x192b2b2b19191908,
+    0x192b2b2b192b082b,
+    0x2b08080808080808,
+    0x2b0808080808082b,
+    0x2b08080808081919,
+    0x2b08080808082b08,
+    0x2b08080808190819,
+    0x2b08080808191908,
+    0x2b080808082b0808,
+    0x2b080808082b2b2b,
+    0x2b08080819080819,
+    0x2b08080819081908,
+    0x2b08080819190808,
+    0x2b0808082b080808,
+    0x2b0808082b08082b,
+    0x2b0808082b2b2b08,
+    0x2b0808082b2b2b2b,
+    0x2b08081908080819,
+    0x2b08081908081908,
+    0x2b0808190808192b,
+    0x2b08081908190808,
+    0x2b08081919080808,
+    0x2b08081919190819,
+    0x2b08081919192b19,
+    0x2b08082b08080808,
+    0x2b08082b082b0808,
+    0x2b08082b2b080808,
+    0x2b08082b2b08082b,
+    0x2b08082b2b2b0808,
+    0x2b08082b2b2b2b08,
+    0x2b08190808080819,
+    0x2b08190808081908,
+    0x2b08190808190808,
+    0x2b0819080819082b,
+    0x2b08190808191919,
+    0x2b08190819080808,
+    0x2b081908192b0808,
+    0x2b0819082b082b19,
+    0x2b08191908080808,
+    0x2b08191919081908,
+    0x2b0819192b2b1919,
+    0x2b08192b08192b08,
+    0x2b08192b192b2b2b,
+    0x2b082b0808080808,
+    0x2b082b0808082b08,
+    0x2b082b08082b1919,
+    0x2b082b0819192b2b,
+    0x2b082b082b080808,
+    0x2b082b082b08082b,
+    0x2b082b082b2b2b08,
+    0x2b082b190808192b,
+    0x2b082b2b082b082b,
+    0x2b082b2b2b080808,
+    0x2b082b2b2b082b08,
+    0x2b082b2b2b19192b,
+    0x2b082b2b2b2b2b08,
+    0x2b19080808080819,
+    0x2b19080808081908,
+    0x2b19080808190808,
+    0x2b19080819080808,
+    0x2b1908081919192b,
+    0x2b1908082b081908,
+    0x2b19081908080808,
+    0x2b190819082b082b,
+    0x2b190819192b1908,
+    0x2b19082b1919192b,
+    0x2b19082b2b082b19,
+    0x2b19190808080808,
+    0x2b19190808081919,
+    0x2b19190819081908,
+    0x2b19190819190808,
+    0x2b19190819192b08,
+    0x2b191919082b2b19,
+    0x2b1919192b190808,
+    0x2b1919192b19082b,
+    0x2b19192b19080819,
+    0x2b192b0819190819,
+    0x2b192b082b2b192b,
+    0x2b192b1919082b19,
+    0x2b192b2b08191919,
+    0x2b192b2b192b0808,
+    0x2b2b080808080808,
+    0x2b2b08080808082b,
+    0x2b2b080808082b08,
+    0x2b2b080808082b2b,
+    0x2b2b0808082b0808,
+    0x2b2b0808082b2b2b,
+    0x2b2b08082b2b0808,
+    0x2b2b081919190819,
+    0x2b2b081919192b19,
+    0x2b2b08192b2b192b,
+    0x2b2b082b08080808,
+    0x2b2b082b0808082b,
+    0x2b2b082b08082b08,
+    0x2b2b082b082b2b2b,
+    0x2b2b082b2b080808,
+    0x2b2b082b2b2b0808,
+    0x2b2b190819080808,
+    0x2b2b19082b191919,
+    0x2b2b192b192b1919,
+    0x2b2b192b2b192b08,
+    0x2b2b2b0808082b2b,
+    0x2b2b2b08082b0808,
+    0x2b2b2b08082b082b,
+    0x2b2b2b08082b2b08,
+    0x2b2b2b082b2b0808,
+    0x2b2b2b082b2b2b08,
+    0x2b2b2b1908081908,
+    0x2b2b2b192b081908,
+    0x2b2b2b192b08192b,
+    0x2b2b2b2b082b2b08,
+    0x2b2b2b2b082b2b2b,
+    0x2b2b2b2b2b190819,
+    0x2b2b2b2b2b2b2b2b,
+};
+constexpr uint64_t iq2s_grid[1024] = {
+    0x0808080808080808,
+    0x080808080808082b,
+    0x0808080808081919,
+    0x0808080808082b08,
+    0x0808080808082b2b,
+    0x0808080808190819,
+    0x0808080808191908,
+    0x080808080819192b,
+    0x0808080808192b19,
+    0x08080808082b0808,
+    0x08080808082b082b,
+    0x08080808082b1919,
+    0x08080808082b2b08,
+    0x0808080819080819,
+    0x0808080819081908,
+    0x080808081908192b,
+    0x0808080819082b19,
+    0x0808080819190808,
+    0x080808081919082b,
+    0x0808080819191919,
+    0x0808080819192b08,
+    0x08080808192b0819,
+    0x08080808192b1908,
+    0x08080808192b192b,
+    0x08080808192b2b19,
+    0x080808082b080808,
+    0x080808082b08082b,
+    0x080808082b081919,
+    0x080808082b082b08,
+    0x080808082b190819,
+    0x080808082b191908,
+    0x080808082b2b0808,
+    0x080808082b2b1919,
+    0x080808082b2b2b2b,
+    0x0808081908080819,
+    0x0808081908081908,
+    0x080808190808192b,
+    0x0808081908082b19,
+    0x0808081908190808,
+    0x080808190819082b,
+    0x0808081908191919,
+    0x0808081908192b08,
+    0x08080819082b0819,
+    0x08080819082b1908,
+    0x0808081919080808,
+    0x080808191908082b,
+    0x0808081919081919,
+    0x0808081919082b08,
+    0x0808081919190819,
+    0x0808081919191908,
+    0x080808191919192b,
+    0x0808081919192b19,
+    0x08080819192b0808,
+    0x08080819192b1919,
+    0x08080819192b2b08,
+    0x080808192b080819,
+    0x080808192b081908,
+    0x080808192b190808,
+    0x080808192b19082b,
+    0x080808192b191919,
+    0x080808192b2b0819,
+    0x080808192b2b1908,
+    0x0808082b08080808,
+    0x0808082b0808082b,
+    0x0808082b08081919,
+    0x0808082b08082b08,
+    0x0808082b08190819,
+    0x0808082b08191908,
+    0x0808082b082b0808,
+    0x0808082b082b2b2b,
+    0x0808082b19080819,
+    0x0808082b19081908,
+    0x0808082b1908192b,
+    0x0808082b19082b19,
+    0x0808082b19190808,
+    0x0808082b19191919,
+    0x0808082b2b080808,
+    0x0808082b2b081919,
+    0x0808082b2b082b2b,
+    0x0808082b2b191908,
+    0x0808082b2b2b082b,
+    0x0808190808080819,
+    0x0808190808081908,
+    0x080819080808192b,
+    0x0808190808082b19,
+    0x0808190808190808,
+    0x080819080819082b,
+    0x0808190808191919,
+    0x0808190808192b08,
+    0x08081908082b0819,
+    0x08081908082b1908,
+    0x08081908082b192b,
+    0x08081908082b2b19,
+    0x0808190819080808,
+    0x080819081908082b,
+    0x0808190819081919,
+    0x0808190819082b08,
+    0x0808190819082b2b,
+    0x0808190819190819,
+    0x0808190819191908,
+    0x080819081919192b,
+    0x0808190819192b19,
+    0x08081908192b0808,
+    0x08081908192b082b,
+    0x08081908192b1919,
+    0x080819082b080819,
+    0x080819082b081908,
+    0x080819082b08192b,
+    0x080819082b082b19,
+    0x080819082b190808,
+    0x080819082b191919,
+    0x080819082b192b08,
+    0x080819082b2b0819,
+    0x080819082b2b1908,
+    0x0808191908080808,
+    0x080819190808082b,
+    0x0808191908081919,
+    0x0808191908082b08,
+    0x0808191908082b2b,
+    0x0808191908190819,
+    0x0808191908191908,
+    0x080819190819192b,
+    0x0808191908192b19,
+    0x08081919082b0808,
+    0x08081919082b1919,
+    0x08081919082b2b08,
+    0x0808191919080819,
+    0x0808191919081908,
+    0x080819191908192b,
+    0x0808191919082b19,
+    0x0808191919190808,
+    0x080819191919082b,
+    0x0808191919191919,
+    0x0808191919192b08,
+    0x08081919192b0819,
+    0x08081919192b1908,
+    0x080819192b080808,
+    0x080819192b08082b,
+    0x080819192b081919,
+    0x080819192b082b08,
+    0x080819192b190819,
+    0x080819192b191908,
+    0x080819192b2b0808,
+    0x0808192b08080819,
+    0x0808192b08081908,
+    0x0808192b0808192b,
+    0x0808192b08082b19,
+    0x0808192b08190808,
+    0x0808192b08191919,
+    0x0808192b19080808,
+    0x0808192b19081919,
+    0x0808192b19082b08,
+    0x0808192b19190819,
+    0x0808192b19191908,
+    0x0808192b192b0808,
+    0x0808192b2b080819,
+    0x0808192b2b081908,
+    0x0808192b2b190808,
+    0x08082b0808080808,
+    0x08082b080808082b,
+    0x08082b0808081919,
+    0x08082b0808082b08,
+    0x08082b0808190819,
+    0x08082b0808191908,
+    0x08082b080819192b,
+    0x08082b0808192b19,
+    0x08082b08082b0808,
+    0x08082b08082b1919,
+    0x08082b08082b2b2b,
+    0x08082b0819080819,
+    0x08082b0819081908,
+    0x08082b081908192b,
+    0x08082b0819082b19,
+    0x08082b0819190808,
+    0x08082b081919082b,
+    0x08082b0819191919,
+    0x08082b0819192b08,
+    0x08082b08192b0819,
+    0x08082b08192b1908,
+    0x08082b082b080808,
+    0x08082b082b081919,
+    0x08082b082b191908,
+    0x08082b082b2b2b2b,
+    0x08082b1908080819,
+    0x08082b1908081908,
+    0x08082b1908190808,
+    0x08082b190819082b,
+    0x08082b1908191919,
+    0x08082b1908192b08,
+    0x08082b19082b0819,
+    0x08082b1919080808,
+    0x08082b1919081919,
+    0x08082b1919082b08,
+    0x08082b1919190819,
+    0x08082b1919191908,
+    0x08082b19192b0808,
+    0x08082b192b080819,
+    0x08082b192b190808,
+    0x08082b2b08080808,
+    0x08082b2b08190819,
+    0x08082b2b08191908,
+    0x08082b2b082b082b,
+    0x08082b2b082b2b08,
+    0x08082b2b082b2b2b,
+    0x08082b2b19190808,
+    0x08082b2b2b192b19,
+    0x0819080808080819,
+    0x0819080808081908,
+    0x081908080808192b,
+    0x0819080808082b19,
+    0x0819080808190808,
+    0x081908080819082b,
+    0x0819080808191919,
+    0x0819080808192b08,
+    0x08190808082b0819,
+    0x08190808082b1908,
+    0x08190808082b192b,
+    0x0819080819080808,
+    0x081908081908082b,
+    0x0819080819081919,
+    0x0819080819082b08,
+    0x0819080819190819,
+    0x0819080819191908,
+    0x081908081919192b,
+    0x0819080819192b19,
+    0x08190808192b0808,
+    0x08190808192b082b,
+    0x08190808192b1919,
+    0x08190808192b2b08,
+    0x081908082b080819,
+    0x081908082b081908,
+    0x081908082b08192b,
+    0x081908082b190808,
+    0x081908082b191919,
+    0x081908082b192b08,
+    0x081908082b2b0819,
+    0x081908082b2b1908,
+    0x0819081908080808,
+    0x081908190808082b,
+    0x0819081908081919,
+    0x0819081908082b08,
+    0x0819081908082b2b,
+    0x0819081908190819,
+    0x0819081908191908,
+    0x081908190819192b,
+    0x0819081908192b19,
+    0x08190819082b0808,
+    0x08190819082b082b,
+    0x08190819082b1919,
+    0x08190819082b2b08,
+    0x0819081919080819,
+    0x0819081919081908,
+    0x081908191908192b,
+    0x0819081919082b19,
+    0x0819081919190808,
+    0x081908191919082b,
+    0x0819081919191919,
+    0x0819081919192b08,
+    0x08190819192b0819,
+    0x08190819192b1908,
+    0x081908192b080808,
+    0x081908192b08082b,
+    0x081908192b081919,
+    0x081908192b082b08,
+    0x081908192b190819,
+    0x081908192b191908,
+    0x0819082b08080819,
+    0x0819082b08081908,
+    0x0819082b08082b19,
+    0x0819082b08190808,
+    0x0819082b08191919,
+    0x0819082b082b0819,
+    0x0819082b082b1908,
+    0x0819082b19080808,
+    0x0819082b19081919,
+    0x0819082b19190819,
+    0x0819082b19191908,
+    0x0819082b2b080819,
+    0x0819082b2b081908,
+    0x0819082b2b190808,
+    0x0819190808080808,
+    0x081919080808082b,
+    0x0819190808081919,
+    0x0819190808082b08,
+    0x0819190808190819,
+    0x0819190808191908,
+    0x081919080819192b,
+    0x0819190808192b19,
+    0x08191908082b0808,
+    0x08191908082b1919,
+    0x08191908082b2b08,
+    0x0819190819080819,
+    0x0819190819081908,
+    0x081919081908192b,
+    0x0819190819082b19,
+    0x0819190819190808,
+    0x081919081919082b,
+    0x0819190819191919,
+    0x0819190819192b08,
+    0x08191908192b0819,
+    0x08191908192b1908,
+    0x081919082b080808,
+    0x081919082b08082b,
+    0x081919082b081919,
+    0x081919082b082b08,
+    0x081919082b190819,
+    0x081919082b191908,
+    0x081919082b2b0808,
+    0x0819191908080819,
+    0x0819191908081908,
+    0x081919190808192b,
+    0x0819191908082b19,
+    0x0819191908190808,
+    0x081919190819082b,
+    0x0819191908191919,
+    0x0819191908192b08,
+    0x08191919082b0819,
+    0x08191919082b1908,
+    0x0819191919080808,
+    0x081919191908082b,
+    0x0819191919081919,
+    0x0819191919082b08,
+    0x0819191919190819,
+    0x0819191919191908,
+    0x08191919192b0808,
+    0x081919192b080819,
+    0x081919192b081908,
+    0x081919192b190808,
+    0x0819192b08080808,
+    0x0819192b08081919,
+    0x0819192b08082b08,
+    0x0819192b08190819,
+    0x0819192b08191908,
+    0x0819192b082b0808,
+    0x0819192b19080819,
+    0x0819192b19081908,
+    0x0819192b19190808,
+    0x0819192b2b080808,
+    0x0819192b2b2b2b2b,
+    0x08192b0808080819,
+    0x08192b0808081908,
+    0x08192b080808192b,
+    0x08192b0808082b19,
+    0x08192b0808190808,
+    0x08192b0808191919,
+    0x08192b0808192b08,
+    0x08192b08082b0819,
+    0x08192b0819080808,
+    0x08192b081908082b,
+    0x08192b0819081919,
+    0x08192b0819082b08,
+    0x08192b0819190819,
+    0x08192b0819191908,
+    0x08192b08192b0808,
+    0x08192b082b080819,
+    0x08192b082b081908,
+    0x08192b1908080808,
+    0x08192b190808082b,
+    0x08192b1908081919,
+    0x08192b1908082b08,
+    0x08192b1908190819,
+    0x08192b1908191908,
+    0x08192b19082b0808,
+    0x08192b1919080819,
+    0x08192b1919081908,
+    0x08192b1919190808,
+    0x08192b19192b2b19,
+    0x08192b192b2b082b,
+    0x08192b2b08081908,
+    0x08192b2b08190808,
+    0x08192b2b19080808,
+    0x08192b2b1919192b,
+    0x082b080808080808,
+    0x082b08080808082b,
+    0x082b080808081919,
+    0x082b080808082b08,
+    0x082b080808190819,
+    0x082b080808191908,
+    0x082b08080819192b,
+    0x082b080808192b19,
+    0x082b0808082b0808,
+    0x082b0808082b1919,
+    0x082b0808082b2b2b,
+    0x082b080819080819,
+    0x082b080819081908,
+    0x082b080819190808,
+    0x082b08081919082b,
+    0x082b080819191919,
+    0x082b0808192b1908,
+    0x082b08082b080808,
+    0x082b08082b082b2b,
+    0x082b08082b191908,
+    0x082b08082b2b2b2b,
+    0x082b081908080819,
+    0x082b081908081908,
+    0x082b081908190808,
+    0x082b08190819082b,
+    0x082b081908191919,
+    0x082b0819082b0819,
+    0x082b081919080808,
+    0x082b08191908082b,
+    0x082b081919081919,
+    0x082b081919190819,
+    0x082b081919191908,
+    0x082b0819192b0808,
+    0x082b08192b080819,
+    0x082b08192b081908,
+    0x082b08192b190808,
+    0x082b082b08080808,
+    0x082b082b08082b2b,
+    0x082b082b082b082b,
+    0x082b082b082b2b08,
+    0x082b082b082b2b2b,
+    0x082b082b19081908,
+    0x082b082b19190808,
+    0x082b082b2b082b08,
+    0x082b082b2b082b2b,
+    0x082b082b2b2b2b08,
+    0x082b190808080819,
+    0x082b190808081908,
+    0x082b19080808192b,
+    0x082b190808082b19,
+    0x082b190808190808,
+    0x082b190808191919,
+    0x082b190808192b08,
+    0x082b1908082b0819,
+    0x082b1908082b1908,
+    0x082b190819080808,
+    0x082b19081908082b,
+    0x082b190819081919,
+    0x082b190819082b08,
+    0x082b190819190819,
+    0x082b190819191908,
+    0x082b1908192b0808,
+    0x082b19082b080819,
+    0x082b19082b081908,
+    0x082b19082b190808,
+    0x082b191908080808,
+    0x082b191908081919,
+    0x082b191908082b08,
+    0x082b191908190819,
+    0x082b191908191908,
+    0x082b1919082b0808,
+    0x082b191919080819,
+    0x082b191919081908,
+    0x082b191919190808,
+    0x082b1919192b192b,
+    0x082b19192b080808,
+    0x082b192b08080819,
+    0x082b192b08081908,
+    0x082b192b08190808,
+    0x082b192b19080808,
+    0x082b192b19192b19,
+    0x082b2b0808080808,
+    0x082b2b0808081919,
+    0x082b2b0808190819,
+    0x082b2b0808191908,
+    0x082b2b0819080819,
+    0x082b2b0819081908,
+    0x082b2b0819190808,
+    0x082b2b082b082b2b,
+    0x082b2b082b2b2b2b,
+    0x082b2b1908080819,
+    0x082b2b1908081908,
+    0x082b2b1908190808,
+    0x082b2b192b191919,
+    0x082b2b2b08082b2b,
+    0x082b2b2b082b082b,
+    0x082b2b2b192b1908,
+    0x082b2b2b2b082b08,
+    0x082b2b2b2b082b2b,
+    0x1908080808080819,
+    0x1908080808081908,
+    0x190808080808192b,
+    0x1908080808082b19,
+    0x1908080808190808,
+    0x190808080819082b,
+    0x1908080808191919,
+    0x1908080808192b08,
+    0x1908080808192b2b,
+    0x19080808082b0819,
+    0x19080808082b1908,
+    0x19080808082b192b,
+    0x1908080819080808,
+    0x190808081908082b,
+    0x1908080819081919,
+    0x1908080819082b08,
+    0x1908080819082b2b,
+    0x1908080819190819,
+    0x1908080819191908,
+    0x190808081919192b,
+    0x1908080819192b19,
+    0x19080808192b0808,
+    0x19080808192b082b,
+    0x19080808192b1919,
+    0x190808082b080819,
+    0x190808082b081908,
+    0x190808082b190808,
+    0x190808082b191919,
+    0x190808082b192b08,
+    0x190808082b2b0819,
+    0x190808082b2b1908,
+    0x1908081908080808,
+    0x190808190808082b,
+    0x1908081908081919,
+    0x1908081908082b08,
+    0x1908081908190819,
+    0x1908081908191908,
+    0x190808190819192b,
+    0x1908081908192b19,
+    0x19080819082b0808,
+    0x19080819082b082b,
+    0x19080819082b1919,
+    0x1908081919080819,
+    0x1908081919081908,
+    0x190808191908192b,
+    0x1908081919082b19,
+    0x1908081919190808,
+    0x190808191919082b,
+    0x1908081919191919,
+    0x1908081919192b08,
+    0x19080819192b0819,
+    0x19080819192b1908,
+    0x190808192b080808,
+    0x190808192b08082b,
+    0x190808192b081919,
+    0x190808192b082b08,
+    0x190808192b190819,
+    0x190808192b191908,
+    0x190808192b2b0808,
+    0x1908082b08080819,
+    0x1908082b08081908,
+    0x1908082b08190808,
+    0x1908082b0819082b,
+    0x1908082b08191919,
+    0x1908082b08192b08,
+    0x1908082b082b1908,
+    0x1908082b19080808,
+    0x1908082b19081919,
+    0x1908082b19082b08,
+    0x1908082b19190819,
+    0x1908082b19191908,
+    0x1908082b192b0808,
+    0x1908082b2b080819,
+    0x1908082b2b081908,
+    0x1908190808080808,
+    0x190819080808082b,
+    0x1908190808081919,
+    0x1908190808082b08,
+    0x1908190808082b2b,
+    0x1908190808190819,
+    0x1908190808191908,
+    0x190819080819192b,
+    0x1908190808192b19,
+    0x19081908082b0808,
+    0x19081908082b082b,
+    0x19081908082b1919,
+    0x19081908082b2b08,
+    0x1908190819080819,
+    0x1908190819081908,
+    0x190819081908192b,
+    0x1908190819082b19,
+    0x1908190819190808,
+    0x190819081919082b,
+    0x1908190819191919,
+    0x1908190819192b08,
+    0x19081908192b0819,
+    0x19081908192b1908,
+    0x190819082b080808,
+    0x190819082b08082b,
+    0x190819082b081919,
+    0x190819082b082b08,
+    0x190819082b190819,
+    0x190819082b191908,
+    0x190819082b2b0808,
+    0x1908191908080819,
+    0x1908191908081908,
+    0x190819190808192b,
+    0x1908191908082b19,
+    0x1908191908190808,
+    0x190819190819082b,
+    0x1908191908191919,
+    0x1908191908192b08,
+    0x19081919082b0819,
+    0x19081919082b1908,
+    0x1908191919080808,
+    0x190819191908082b,
+    0x1908191919081919,
+    0x1908191919082b08,
+    0x1908191919190819,
+    0x1908191919191908,
+    0x19081919192b0808,
+    0x19081919192b2b2b,
+    0x190819192b080819,
+    0x190819192b081908,
+    0x190819192b190808,
+    0x1908192b08080808,
+    0x1908192b0808082b,
+    0x1908192b08081919,
+    0x1908192b08082b08,
+    0x1908192b08190819,
+    0x1908192b08191908,
+    0x1908192b082b0808,
+    0x1908192b19080819,
+    0x1908192b19081908,
+    0x1908192b19190808,
+    0x1908192b2b080808,
+    0x1908192b2b2b1919,
+    0x19082b0808080819,
+    0x19082b0808081908,
+    0x19082b0808082b19,
+    0x19082b0808190808,
+    0x19082b080819082b,
+    0x19082b0808191919,
+    0x19082b0808192b08,
+    0x19082b08082b0819,
+    0x19082b08082b1908,
+    0x19082b0819080808,
+    0x19082b081908082b,
+    0x19082b0819081919,
+    0x19082b0819082b08,
+    0x19082b0819190819,
+    0x19082b0819191908,
+    0x19082b08192b0808,
+    0x19082b082b081908,
+    0x19082b082b190808,
+    0x19082b1908080808,
+    0x19082b190808082b,
+    0x19082b1908081919,
+    0x19082b1908082b08,
+    0x19082b1908190819,
+    0x19082b1908191908,
+    0x19082b19082b0808,
+    0x19082b1919080819,
+    0x19082b1919081908,
+    0x19082b1919190808,
+    0x19082b192b080808,
+    0x19082b192b19192b,
+    0x19082b2b08080819,
+    0x19082b2b08081908,
+    0x19082b2b08190808,
+    0x19082b2b19080808,
+    0x1919080808080808,
+    0x191908080808082b,
+    0x1919080808081919,
+    0x1919080808082b08,
+    0x1919080808190819,
+    0x1919080808191908,
+    0x191908080819192b,
+    0x1919080808192b19,
+    0x19190808082b0808,
+    0x19190808082b082b,
+    0x19190808082b1919,
+    0x19190808082b2b08,
+    0x1919080819080819,
+    0x1919080819081908,
+    0x191908081908192b,
+    0x1919080819082b19,
+    0x1919080819190808,
+    0x191908081919082b,
+    0x1919080819191919,
+    0x1919080819192b08,
+    0x19190808192b0819,
+    0x19190808192b1908,
+    0x191908082b080808,
+    0x191908082b08082b,
+    0x191908082b081919,
+    0x191908082b082b08,
+    0x191908082b190819,
+    0x191908082b191908,
+    0x1919081908080819,
+    0x1919081908081908,
+    0x191908190808192b,
+    0x1919081908082b19,
+    0x1919081908190808,
+    0x191908190819082b,
+    0x1919081908191919,
+    0x1919081908192b08,
+    0x19190819082b0819,
+    0x19190819082b1908,
+    0x1919081919080808,
+    0x191908191908082b,
+    0x1919081919081919,
+    0x1919081919082b08,
+    0x1919081919190819,
+    0x1919081919191908,
+    0x19190819192b0808,
+    0x191908192b080819,
+    0x191908192b081908,
+    0x191908192b190808,
+    0x1919082b08080808,
+    0x1919082b08081919,
+    0x1919082b08082b08,
+    0x1919082b08190819,
+    0x1919082b08191908,
+    0x1919082b082b0808,
+    0x1919082b19080819,
+    0x1919082b19081908,
+    0x1919082b19190808,
+    0x1919082b192b2b19,
+    0x1919082b2b080808,
+    0x1919190808080819,
+    0x1919190808081908,
+    0x191919080808192b,
+    0x1919190808082b19,
+    0x1919190808190808,
+    0x191919080819082b,
+    0x1919190808191919,
+    0x1919190808192b08,
+    0x19191908082b0819,
+    0x19191908082b1908,
+    0x1919190819080808,
+    0x191919081908082b,
+    0x1919190819081919,
+    0x1919190819082b08,
+    0x1919190819190819,
+    0x1919190819191908,
+    0x19191908192b0808,
+    0x191919082b080819,
+    0x191919082b081908,
+    0x191919082b190808,
+    0x1919191908080808,
+    0x191919190808082b,
+    0x1919191908081919,
+    0x1919191908082b08,
+    0x1919191908190819,
+    0x1919191908191908,
+    0x19191919082b0808,
+    0x1919191919080819,
+    0x1919191919081908,
+    0x1919191919190808,
+    0x191919192b080808,
+    0x1919192b08080819,
+    0x1919192b08081908,
+    0x1919192b08190808,
+    0x1919192b082b192b,
+    0x1919192b19080808,
+    0x19192b0808080808,
+    0x19192b080808082b,
+    0x19192b0808081919,
+    0x19192b0808082b08,
+    0x19192b0808190819,
+    0x19192b0808191908,
+    0x19192b08082b0808,
+    0x19192b0819080819,
+    0x19192b0819081908,
+    0x19192b0819190808,
+    0x19192b0819192b2b,
+    0x19192b082b080808,
+    0x19192b1908080819,
+    0x19192b1908081908,
+    0x19192b1908190808,
+    0x19192b1919080808,
+    0x19192b2b08080808,
+    0x19192b2b08192b19,
+    0x19192b2b2b081919,
+    0x19192b2b2b2b2b08,
+    0x192b080808080819,
+    0x192b080808081908,
+    0x192b08080808192b,
+    0x192b080808190808,
+    0x192b08080819082b,
+    0x192b080808191919,
+    0x192b080808192b08,
+    0x192b0808082b0819,
+    0x192b0808082b1908,
+    0x192b080819080808,
+    0x192b080819081919,
+    0x192b080819082b08,
+    0x192b080819190819,
+    0x192b080819191908,
+    0x192b0808192b0808,
+    0x192b08082b081908,
+    0x192b08082b190808,
+    0x192b081908080808,
+    0x192b08190808082b,
+    0x192b081908081919,
+    0x192b081908082b08,
+    0x192b081908190819,
+    0x192b081908191908,
+    0x192b0819082b0808,
+    0x192b081919080819,
+    0x192b081919081908,
+    0x192b081919190808,
+    0x192b08192b080808,
+    0x192b08192b192b19,
+    0x192b082b08081908,
+    0x192b082b08190808,
+    0x192b082b19080808,
+    0x192b082b1919192b,
+    0x192b082b2b2b0819,
+    0x192b190808080808,
+    0x192b190808081919,
+    0x192b190808082b08,
+    0x192b190808190819,
+    0x192b190808191908,
+    0x192b1908082b0808,
+    0x192b190819080819,
+    0x192b190819081908,
+    0x192b190819190808,
+    0x192b19082b080808,
+    0x192b191908080819,
+    0x192b191908081908,
+    0x192b191908190808,
+    0x192b191919080808,
+    0x192b191919082b2b,
+    0x192b1919192b2b08,
+    0x192b19192b19082b,
+    0x192b192b08080808,
+    0x192b192b2b191908,
+    0x192b2b0808080819,
+    0x192b2b0808081908,
+    0x192b2b0808190808,
+    0x192b2b08192b1919,
+    0x192b2b082b192b08,
+    0x192b2b1908080808,
+    0x192b2b19082b2b2b,
+    0x192b2b2b1908082b,
+    0x192b2b2b2b2b0819,
+    0x2b08080808080808,
+    0x2b0808080808082b,
+    0x2b08080808081919,
+    0x2b08080808082b08,
+    0x2b08080808190819,
+    0x2b08080808191908,
+    0x2b08080808192b19,
+    0x2b080808082b0808,
+    0x2b080808082b1919,
+    0x2b08080819080819,
+    0x2b08080819081908,
+    0x2b08080819190808,
+    0x2b0808081919082b,
+    0x2b08080819191919,
+    0x2b08080819192b08,
+    0x2b080808192b0819,
+    0x2b0808082b080808,
+    0x2b0808082b081919,
+    0x2b0808082b190819,
+    0x2b0808082b191908,
+    0x2b08081908080819,
+    0x2b08081908081908,
+    0x2b08081908082b19,
+    0x2b08081908190808,
+    0x2b0808190819082b,
+    0x2b08081908191919,
+    0x2b08081908192b08,
+    0x2b080819082b0819,
+    0x2b080819082b1908,
+    0x2b08081919080808,
+    0x2b0808191908082b,
+    0x2b08081919081919,
+    0x2b08081919082b08,
+    0x2b08081919190819,
+    0x2b08081919191908,
+    0x2b0808192b080819,
+    0x2b0808192b081908,
+    0x2b0808192b190808,
+    0x2b0808192b2b2b19,
+    0x2b08082b08080808,
+    0x2b08082b08081919,
+    0x2b08082b08082b2b,
+    0x2b08082b08190819,
+    0x2b08082b08191908,
+    0x2b08082b19080819,
+    0x2b08082b19081908,
+    0x2b08082b19190808,
+    0x2b08190808080819,
+    0x2b08190808081908,
+    0x2b0819080808192b,
+    0x2b08190808082b19,
+    0x2b08190808190808,
+    0x2b0819080819082b,
+    0x2b08190808191919,
+    0x2b08190808192b08,
+    0x2b081908082b0819,
+    0x2b08190819080808,
+    0x2b0819081908082b,
+    0x2b08190819081919,
+    0x2b08190819082b08,
+    0x2b08190819190819,
+    0x2b08190819191908,
+    0x2b081908192b0808,
+    0x2b0819082b080819,
+    0x2b0819082b081908,
+    0x2b0819082b190808,
+    0x2b08191908080808,
+    0x2b0819190808082b,
+    0x2b08191908081919,
+    0x2b08191908082b08,
+    0x2b08191908190819,
+    0x2b08191908191908,
+    0x2b081919082b0808,
+    0x2b08191919080819,
+    0x2b08191919081908,
+    0x2b08191919190808,
+    0x2b0819192b080808,
+    0x2b0819192b082b2b,
+    0x2b08192b08080819,
+    0x2b08192b08081908,
+    0x2b08192b08190808,
+    0x2b08192b082b2b19,
+    0x2b08192b19080808,
+    0x2b082b0808080808,
+    0x2b082b0808081919,
+    0x2b082b0808190819,
+    0x2b082b0808191908,
+    0x2b082b0819080819,
+    0x2b082b0819081908,
+    0x2b082b0819190808,
+    0x2b082b082b2b082b,
+    0x2b082b1908080819,
+    0x2b082b1908081908,
+    0x2b082b1919080808,
+    0x2b082b19192b1919,
+    0x2b082b2b082b082b,
+    0x2b082b2b19192b08,
+    0x2b082b2b19192b2b,
+    0x2b082b2b2b08082b,
+    0x2b082b2b2b2b082b,
+    0x2b19080808080819,
+    0x2b19080808081908,
+    0x2b19080808082b19,
+    0x2b19080808190808,
+    0x2b1908080819082b,
+    0x2b19080808191919,
+    0x2b19080808192b08,
+    0x2b190808082b1908,
+    0x2b19080819080808,
+    0x2b1908081908082b,
+    0x2b19080819081919,
+    0x2b19080819082b08,
+    0x2b19080819190819,
+    0x2b19080819191908,
+    0x2b190808192b0808,
+    0x2b1908082b080819,
+    0x2b1908082b081908,
+    0x2b1908082b190808,
+    0x2b19081908080808,
+    0x2b19081908081919,
+    0x2b19081908190819,
+    0x2b19081908191908,
+    0x2b19081919080819,
+    0x2b19081919081908,
+    0x2b19081919190808,
+    0x2b19081919192b2b,
+    0x2b19082b08080819,
+    0x2b19082b08081908,
+    0x2b19082b08190808,
+    0x2b19082b19080808,
+    0x2b19082b2b2b192b,
+    0x2b19190808080808,
+    0x2b1919080808082b,
+    0x2b19190808081919,
+    0x2b19190808082b08,
+    0x2b19190808190819,
+    0x2b19190808191908,
+    0x2b191908082b0808,
+    0x2b19190819080819,
+    0x2b19190819081908,
+    0x2b19190819190808,
+    0x2b1919082b080808,
+    0x2b1919082b19192b,
+    0x2b19191908080819,
+    0x2b19191908081908,
+    0x2b19191908190808,
+    0x2b19191919080808,
+    0x2b1919192b192b08,
+    0x2b1919192b2b0819,
+    0x2b19192b08080808,
+    0x2b19192b1908192b,
+    0x2b19192b192b1908,
+    0x2b192b0808080819,
+    0x2b192b0808081908,
+    0x2b192b0808190808,
+    0x2b192b08082b192b,
+    0x2b192b0819080808,
+    0x2b192b082b2b2b19,
+    0x2b192b1908080808,
+    0x2b192b1919082b19,
+    0x2b192b191919082b,
+    0x2b192b2b2b190808,
+    0x2b2b080808080808,
+    0x2b2b080808081919,
+    0x2b2b080808082b2b,
+    0x2b2b080808191908,
+    0x2b2b0808082b082b,
+    0x2b2b0808082b2b2b,
+    0x2b2b080819080819,
+    0x2b2b080819081908,
+    0x2b2b080819190808,
+    0x2b2b08082b2b082b,
+    0x2b2b08082b2b2b2b,
+    0x2b2b081919080808,
+    0x2b2b0819192b1919,
+    0x2b2b082b0808082b,
+    0x2b2b082b08082b2b,
+    0x2b2b082b082b082b,
+    0x2b2b082b082b2b08,
+    0x2b2b082b082b2b2b,
+    0x2b2b082b2b08082b,
+    0x2b2b082b2b082b08,
+    0x2b2b082b2b082b2b,
+    0x2b2b082b2b2b2b08,
+    0x2b2b190808080819,
+    0x2b2b190808081908,
+    0x2b2b190808190808,
+    0x2b2b190819080808,
+    0x2b2b19082b082b19,
+    0x2b2b19082b2b1908,
+    0x2b2b191908080808,
+    0x2b2b191908192b19,
+    0x2b2b192b19190819,
+    0x2b2b2b0808082b2b,
+    0x2b2b2b08082b2b08,
+    0x2b2b2b082b2b082b,
+    0x2b2b2b1919191908,
+    0x2b2b2b192b08192b,
+    0x2b2b2b2b08082b08,
+    0x2b2b2b2b08082b2b,
+    0x2b2b2b2b082b0808,
+    0x2b2b2b2b082b082b,
+    0x2b2b2b2b082b2b08,
+    0x2b2b2b2b2b082b08,
+    0x2b2b2b2b2b2b2b2b,
+};
+constexpr uint32_t iq3xxs_grid[256] = {
+    0x04040404,
+    0x04040414,
+    0x04040424,
+    0x04040c0c,
+    0x04040c1c,
+    0x04040c3e,
+    0x04041404,
+    0x04041414,
+    0x04041c0c,
+    0x04042414,
+    0x04043e1c,
+    0x04043e2c,
+    0x040c040c,
+    0x040c041c,
+    0x040c0c04,
+    0x040c0c14,
+    0x040c140c,
+    0x040c142c,
+    0x040c1c04,
+    0x040c1c14,
+    0x040c240c,
+    0x040c2c24,
+    0x040c3e04,
+    0x04140404,
+    0x04140414,
+    0x04140424,
+    0x04140c0c,
+    0x04141404,
+    0x04141414,
+    0x04141c0c,
+    0x04141c1c,
+    0x04141c3e,
+    0x04142c0c,
+    0x04142c3e,
+    0x04143e2c,
+    0x041c040c,
+    0x041c043e,
+    0x041c0c04,
+    0x041c0c14,
+    0x041c142c,
+    0x041c3e04,
+    0x04240c1c,
+    0x04241c3e,
+    0x04242424,
+    0x04242c3e,
+    0x04243e1c,
+    0x04243e2c,
+    0x042c040c,
+    0x042c043e,
+    0x042c1c14,
+    0x042c2c14,
+    0x04341c2c,
+    0x04343424,
+    0x043e0c04,
+    0x043e0c24,
+    0x043e0c34,
+    0x043e241c,
+    0x043e340c,
+    0x0c04040c,
+    0x0c04041c,
+    0x0c040c04,
+    0x0c040c14,
+    0x0c04140c,
+    0x0c04141c,
+    0x0c041c04,
+    0x0c041c14,
+    0x0c041c24,
+    0x0c04243e,
+    0x0c042c04,
+    0x0c0c0404,
+    0x0c0c0414,
+    0x0c0c0c0c,
+    0x0c0c1404,
+    0x0c0c1414,
+    0x0c14040c,
+    0x0c14041c,
+    0x0c140c04,
+    0x0c140c14,
+    0x0c14140c,
+    0x0c141c04,
+    0x0c143e14,
+    0x0c1c0404,
+    0x0c1c0414,
+    0x0c1c1404,
+    0x0c1c1c0c,
+    0x0c1c2434,
+    0x0c1c3434,
+    0x0c24040c,
+    0x0c24042c,
+    0x0c242c04,
+    0x0c2c1404,
+    0x0c2c1424,
+    0x0c2c2434,
+    0x0c2c3e0c,
+    0x0c34042c,
+    0x0c3e1414,
+    0x0c3e2404,
+    0x14040404,
+    0x14040414,
+    0x14040c0c,
+    0x14040c1c,
+    0x14041404,
+    0x14041414,
+    0x14041434,
+    0x14041c0c,
+    0x14042414,
+    0x140c040c,
+    0x140c041c,
+    0x140c042c,
+    0x140c0c04,
+    0x140c0c14,
+    0x140c140c,
+    0x140c1c04,
+    0x140c341c,
+    0x140c343e,
+    0x140c3e04,
+    0x14140404,
+    0x14140414,
+    0x14140c0c,
+    0x14140c3e,
+    0x14141404,
+    0x14141414,
+    0x14141c3e,
+    0x14142404,
+    0x14142c2c,
+    0x141c040c,
+    0x141c0c04,
+    0x141c0c24,
+    0x141c3e04,
+    0x141c3e24,
+    0x14241c2c,
+    0x14242c1c,
+    0x142c041c,
+    0x142c143e,
+    0x142c240c,
+    0x142c3e24,
+    0x143e040c,
+    0x143e041c,
+    0x143e0c34,
+    0x143e242c,
+    0x1c04040c,
+    0x1c040c04,
+    0x1c040c14,
+    0x1c04140c,
+    0x1c04141c,
+    0x1c042c04,
+    0x1c04342c,
+    0x1c043e14,
+    0x1c0c0404,
+    0x1c0c0414,
+    0x1c0c1404,
+    0x1c0c1c0c,
+    0x1c0c2424,
+    0x1c0c2434,
+    0x1c14040c,
+    0x1c14041c,
+    0x1c140c04,
+    0x1c14142c,
+    0x1c142c14,
+    0x1c143e14,
+    0x1c1c0c0c,
+    0x1c1c1c1c,
+    0x1c241c04,
+    0x1c24243e,
+    0x1c243e14,
+    0x1c2c0404,
+    0x1c2c0434,
+    0x1c2c1414,
+    0x1c2c2c2c,
+    0x1c340c24,
+    0x1c341c34,
+    0x1c34341c,
+    0x1c3e1c1c,
+    0x1c3e3404,
+    0x24040424,
+    0x24040c3e,
+    0x24041c2c,
+    0x24041c3e,
+    0x24042c1c,
+    0x24042c3e,
+    0x240c3e24,
+    0x24141404,
+    0x24141c3e,
+    0x24142404,
+    0x24143404,
+    0x24143434,
+    0x241c043e,
+    0x241c242c,
+    0x24240424,
+    0x24242c0c,
+    0x24243424,
+    0x242c142c,
+    0x242c241c,
+    0x242c3e04,
+    0x243e042c,
+    0x243e0c04,
+    0x243e0c14,
+    0x243e1c04,
+    0x2c040c14,
+    0x2c04240c,
+    0x2c043e04,
+    0x2c0c0404,
+    0x2c0c0434,
+    0x2c0c1434,
+    0x2c0c2c2c,
+    0x2c140c24,
+    0x2c141c14,
+    0x2c143e14,
+    0x2c1c0414,
+    0x2c1c2c1c,
+    0x2c240c04,
+    0x2c24141c,
+    0x2c24143e,
+    0x2c243e14,
+    0x2c2c0414,
+    0x2c2c1c0c,
+    0x2c342c04,
+    0x2c3e1424,
+    0x2c3e2414,
+    0x34041424,
+    0x34042424,
+    0x34042434,
+    0x34043424,
+    0x340c140c,
+    0x340c340c,
+    0x34140c3e,
+    0x34143424,
+    0x341c1c04,
+    0x341c1c34,
+    0x34242424,
+    0x342c042c,
+    0x342c2c14,
+    0x34341c1c,
+    0x343e041c,
+    0x343e140c,
+    0x3e04041c,
+    0x3e04042c,
+    0x3e04043e,
+    0x3e040c04,
+    0x3e041c14,
+    0x3e042c14,
+    0x3e0c1434,
+    0x3e0c2404,
+    0x3e140c14,
+    0x3e14242c,
+    0x3e142c14,
+    0x3e1c0404,
+    0x3e1c0c2c,
+    0x3e1c1c1c,
+    0x3e1c3404,
+    0x3e24140c,
+    0x3e24240c,
+    0x3e2c0404,
+    0x3e2c0414,
+    0x3e2c1424,
+    0x3e341c04,
+};
+constexpr uint32_t iq3s_grid[512] = {
+    0x01010101,
+    0x01010103,
+    0x01010105,
+    0x0101010b,
+    0x0101010f,
+    0x01010301,
+    0x01010303,
+    0x01010305,
+    0x01010309,
+    0x0101030d,
+    0x01010501,
+    0x01010503,
+    0x0101050b,
+    0x01010707,
+    0x01010901,
+    0x01010905,
+    0x0101090b,
+    0x0101090f,
+    0x01010b03,
+    0x01010b07,
+    0x01010d01,
+    0x01010d05,
+    0x01010f03,
+    0x01010f09,
+    0x01010f0f,
+    0x01030101,
+    0x01030103,
+    0x01030105,
+    0x01030109,
+    0x01030301,
+    0x01030303,
+    0x0103030b,
+    0x01030501,
+    0x01030507,
+    0x0103050f,
+    0x01030703,
+    0x0103070b,
+    0x01030909,
+    0x01030d03,
+    0x01030d0b,
+    0x01030f05,
+    0x01050101,
+    0x01050103,
+    0x0105010b,
+    0x0105010f,
+    0x01050301,
+    0x01050307,
+    0x0105030d,
+    0x01050503,
+    0x0105050b,
+    0x01050701,
+    0x01050709,
+    0x01050905,
+    0x0105090b,
+    0x0105090f,
+    0x01050b03,
+    0x01050b07,
+    0x01050f01,
+    0x01050f07,
+    0x01070107,
+    0x01070303,
+    0x0107030b,
+    0x01070501,
+    0x01070505,
+    0x01070703,
+    0x01070707,
+    0x0107070d,
+    0x01070909,
+    0x01070b01,
+    0x01070b05,
+    0x01070d0f,
+    0x01070f03,
+    0x01070f0b,
+    0x01090101,
+    0x01090307,
+    0x0109030f,
+    0x01090503,
+    0x01090509,
+    0x01090705,
+    0x01090901,
+    0x01090907,
+    0x01090b03,
+    0x01090f01,
+    0x010b0105,
+    0x010b0109,
+    0x010b0501,
+    0x010b0505,
+    0x010b050d,
+    0x010b0707,
+    0x010b0903,
+    0x010b090b,
+    0x010b090f,
+    0x010b0d0d,
+    0x010b0f07,
+    0x010d010d,
+    0x010d0303,
+    0x010d0307,
+    0x010d0703,
+    0x010d0b05,
+    0x010d0f03,
+    0x010f0101,
+    0x010f0105,
+    0x010f0109,
+    0x010f0501,
+    0x010f0505,
+    0x010f050d,
+    0x010f0707,
+    0x010f0b01,
+    0x010f0b09,
+    0x03010101,
+    0x03010103,
+    0x03010105,
+    0x03010109,
+    0x03010301,
+    0x03010303,
+    0x03010307,
+    0x0301030b,
+    0x0301030f,
+    0x03010501,
+    0x03010505,
+    0x03010703,
+    0x03010709,
+    0x0301070d,
+    0x03010b09,
+    0x03010b0d,
+    0x03010d03,
+    0x03010f05,
+    0x03030101,
+    0x03030103,
+    0x03030107,
+    0x0303010d,
+    0x03030301,
+    0x03030309,
+    0x03030503,
+    0x03030701,
+    0x03030707,
+    0x03030903,
+    0x03030b01,
+    0x03030b05,
+    0x03030f01,
+    0x03030f0d,
+    0x03050101,
+    0x03050305,
+    0x0305030b,
+    0x0305030f,
+    0x03050501,
+    0x03050509,
+    0x03050705,
+    0x03050901,
+    0x03050907,
+    0x03050b0b,
+    0x03050d01,
+    0x03050f05,
+    0x03070103,
+    0x03070109,
+    0x0307010f,
+    0x03070301,
+    0x03070307,
+    0x03070503,
+    0x0307050f,
+    0x03070701,
+    0x03070709,
+    0x03070903,
+    0x03070d05,
+    0x03070f01,
+    0x03090107,
+    0x0309010b,
+    0x03090305,
+    0x03090309,
+    0x03090703,
+    0x03090707,
+    0x03090905,
+    0x0309090d,
+    0x03090b01,
+    0x03090b09,
+    0x030b0103,
+    0x030b0301,
+    0x030b0307,
+    0x030b0503,
+    0x030b0701,
+    0x030b0705,
+    0x030b0b03,
+    0x030d0501,
+    0x030d0509,
+    0x030d050f,
+    0x030d0909,
+    0x030d090d,
+    0x030f0103,
+    0x030f0107,
+    0x030f0301,
+    0x030f0305,
+    0x030f0503,
+    0x030f070b,
+    0x030f0903,
+    0x030f0d05,
+    0x030f0f01,
+    0x05010101,
+    0x05010103,
+    0x05010107,
+    0x0501010b,
+    0x0501010f,
+    0x05010301,
+    0x05010305,
+    0x05010309,
+    0x0501030d,
+    0x05010503,
+    0x05010507,
+    0x0501050f,
+    0x05010701,
+    0x05010705,
+    0x05010903,
+    0x05010907,
+    0x0501090b,
+    0x05010b01,
+    0x05010b05,
+    0x05010d0f,
+    0x05010f01,
+    0x05010f07,
+    0x05010f0b,
+    0x05030101,
+    0x05030105,
+    0x05030301,
+    0x05030307,
+    0x0503030f,
+    0x05030505,
+    0x0503050b,
+    0x05030703,
+    0x05030709,
+    0x05030905,
+    0x05030b03,
+    0x05050103,
+    0x05050109,
+    0x0505010f,
+    0x05050503,
+    0x05050507,
+    0x05050701,
+    0x0505070f,
+    0x05050903,
+    0x05050b07,
+    0x05050b0f,
+    0x05050f03,
+    0x05050f09,
+    0x05070101,
+    0x05070105,
+    0x0507010b,
+    0x05070303,
+    0x05070505,
+    0x05070509,
+    0x05070703,
+    0x05070707,
+    0x05070905,
+    0x05070b01,
+    0x05070d0d,
+    0x05090103,
+    0x0509010f,
+    0x05090501,
+    0x05090507,
+    0x05090705,
+    0x0509070b,
+    0x05090903,
+    0x05090f05,
+    0x05090f0b,
+    0x050b0109,
+    0x050b0303,
+    0x050b0505,
+    0x050b070f,
+    0x050b0901,
+    0x050b0b07,
+    0x050b0f01,
+    0x050d0101,
+    0x050d0105,
+    0x050d010f,
+    0x050d0503,
+    0x050d0b0b,
+    0x050d0d03,
+    0x050f010b,
+    0x050f0303,
+    0x050f050d,
+    0x050f0701,
+    0x050f0907,
+    0x050f0b01,
+    0x07010105,
+    0x07010303,
+    0x07010307,
+    0x0701030b,
+    0x0701030f,
+    0x07010505,
+    0x07010703,
+    0x07010707,
+    0x0701070b,
+    0x07010905,
+    0x07010909,
+    0x0701090f,
+    0x07010b03,
+    0x07010d07,
+    0x07010f03,
+    0x07030103,
+    0x07030107,
+    0x0703010b,
+    0x07030309,
+    0x07030503,
+    0x07030507,
+    0x07030901,
+    0x07030d01,
+    0x07030f05,
+    0x07030f0d,
+    0x07050101,
+    0x07050305,
+    0x07050501,
+    0x07050705,
+    0x07050709,
+    0x07050b01,
+    0x07070103,
+    0x07070301,
+    0x07070309,
+    0x07070503,
+    0x07070507,
+    0x0707050f,
+    0x07070701,
+    0x07070903,
+    0x07070907,
+    0x0707090f,
+    0x07070b0b,
+    0x07070f07,
+    0x07090107,
+    0x07090303,
+    0x0709030d,
+    0x07090505,
+    0x07090703,
+    0x07090b05,
+    0x07090d01,
+    0x07090d09,
+    0x070b0103,
+    0x070b0301,
+    0x070b0305,
+    0x070b050b,
+    0x070b0705,
+    0x070b0909,
+    0x070b0b0d,
+    0x070b0f07,
+    0x070d030d,
+    0x070d0903,
+    0x070f0103,
+    0x070f0107,
+    0x070f0501,
+    0x070f0505,
+    0x070f070b,
+    0x09010101,
+    0x09010109,
+    0x09010305,
+    0x09010501,
+    0x09010509,
+    0x0901050f,
+    0x09010705,
+    0x09010903,
+    0x09010b01,
+    0x09010f01,
+    0x09030105,
+    0x0903010f,
+    0x09030303,
+    0x09030307,
+    0x09030505,
+    0x09030701,
+    0x0903070b,
+    0x09030907,
+    0x09030b03,
+    0x09030b0b,
+    0x09050103,
+    0x09050107,
+    0x09050301,
+    0x0905030b,
+    0x09050503,
+    0x09050707,
+    0x09050901,
+    0x09050b0f,
+    0x09050d05,
+    0x09050f01,
+    0x09070109,
+    0x09070303,
+    0x09070307,
+    0x09070501,
+    0x09070505,
+    0x09070703,
+    0x0907070b,
+    0x09090101,
+    0x09090105,
+    0x09090509,
+    0x0909070f,
+    0x09090901,
+    0x09090f03,
+    0x090b010b,
+    0x090b010f,
+    0x090b0503,
+    0x090b0d05,
+    0x090d0307,
+    0x090d0709,
+    0x090d0d01,
+    0x090f0301,
+    0x090f030b,
+    0x090f0701,
+    0x090f0907,
+    0x090f0b03,
+    0x0b010105,
+    0x0b010301,
+    0x0b010309,
+    0x0b010505,
+    0x0b010901,
+    0x0b010909,
+    0x0b01090f,
+    0x0b010b05,
+    0x0b010d0d,
+    0x0b010f09,
+    0x0b030103,
+    0x0b030107,
+    0x0b03010b,
+    0x0b030305,
+    0x0b030503,
+    0x0b030705,
+    0x0b030f05,
+    0x0b050101,
+    0x0b050303,
+    0x0b050507,
+    0x0b050701,
+    0x0b05070d,
+    0x0b050b07,
+    0x0b070105,
+    0x0b07010f,
+    0x0b070301,
+    0x0b07050f,
+    0x0b070909,
+    0x0b070b03,
+    0x0b070d0b,
+    0x0b070f07,
+    0x0b090103,
+    0x0b090109,
+    0x0b090501,
+    0x0b090705,
+    0x0b09090d,
+    0x0b0b0305,
+    0x0b0b050d,
+    0x0b0b0b03,
+    0x0b0b0b07,
+    0x0b0d0905,
+    0x0b0f0105,
+    0x0b0f0109,
+    0x0b0f0505,
+    0x0d010303,
+    0x0d010307,
+    0x0d01030b,
+    0x0d010703,
+    0x0d010707,
+    0x0d010d01,
+    0x0d030101,
+    0x0d030501,
+    0x0d03050f,
+    0x0d030d09,
+    0x0d050305,
+    0x0d050709,
+    0x0d050905,
+    0x0d050b0b,
+    0x0d050d05,
+    0x0d050f01,
+    0x0d070101,
+    0x0d070309,
+    0x0d070503,
+    0x0d070901,
+    0x0d09050b,
+    0x0d090907,
+    0x0d090d05,
+    0x0d0b0101,
+    0x0d0b0107,
+    0x0d0b0709,
+    0x0d0b0d01,
+    0x0d0d010b,
+    0x0d0d0901,
+    0x0d0f0303,
+    0x0d0f0307,
+    0x0f010101,
+    0x0f010109,
+    0x0f01010f,
+    0x0f010501,
+    0x0f010505,
+    0x0f01070d,
+    0x0f010901,
+    0x0f010b09,
+    0x0f010d05,
+    0x0f030105,
+    0x0f030303,
+    0x0f030509,
+    0x0f030907,
+    0x0f03090b,
+    0x0f050103,
+    0x0f050109,
+    0x0f050301,
+    0x0f05030d,
+    0x0f050503,
+    0x0f050701,
+    0x0f050b03,
+    0x0f070105,
+    0x0f070705,
+    0x0f07070b,
+    0x0f070b07,
+    0x0f090103,
+    0x0f09010b,
+    0x0f090307,
+    0x0f090501,
+    0x0f090b01,
+    0x0f0b0505,
+    0x0f0b0905,
+    0x0f0d0105,
+    0x0f0d0703,
+    0x0f0f0101,
+};
+constexpr uint64_t iq1s_grid[2048] = {
+    0xffffffffffffffff,
+    0xffffffffffffff01,
+    0xffffffffffff0000,
+    0xffffffffffff01ff,
+    0xffffffffffff0101,
+    0xffffffffff00ff00,
+    0xffffffffff000000,
+    0xffffffffff01ffff,
+    0xffffffffff01ff01,
+    0xffffffffff0101ff,
+    0xffffffffff010101,
+    0xffffffff00ff0000,
+    0xffffffff0000ff00,
+    0xffffffff000000ff,
+    0xffffffff00000001,
+    0xffffffff00010000,
+    0xffffffff01ffffff,
+    0xffffffff01ffff01,
+    0xffffffff01ff01ff,
+    0xffffffff01ff0101,
+    0xffffffff01000000,
+    0xffffffff0101ffff,
+    0xffffffff0101ff01,
+    0xffffffff010101ff,
+    0xffffffff01010101,
+    0xffffff00ffff00ff,
+    0xffffff00ffff0000,
+    0xffffff00ff00ff00,
+    0xffffff00ff0000ff,
+    0xffffff00ff000001,
+    0xffffff00ff000100,
+    0xffffff00ff000101,
+    0xffffff00ff010000,
+    0xffffff0000ffff00,
+    0xffffff0000ff0001,
+    0xffffff0000ff0100,
+    0xffffff000000ff01,
+    0xffffff0000000000,
+    0xffffff0000000101,
+    0xffffff000001ff00,
+    0xffffff00000100ff,
+    0xffffff0000010001,
+    0xffffff00000101ff,
+    0xffffff0001ff0000,
+    0xffffff000100ff00,
+    0xffffff00010000ff,
+    0xffffff0001000001,
+    0xffffff0001010000,
+    0xffffff01ffffffff,
+    0xffffff01ffffff01,
+    0xffffff01ffff01ff,
+    0xffffff01ffff0101,
+    0xffffff01ff000000,
+    0xffffff01ff01ffff,
+    0xffffff01ff01ff01,
+    0xffffff01ff0101ff,
+    0xffffff01ff010101,
+    0xffffff0100ff0000,
+    0xffffff010000ff00,
+    0xffffff0100000100,
+    0xffffff01000100ff,
+    0xffffff0100010100,
+    0xffffff0101ffffff,
+    0xffffff0101ffff01,
+    0xffffff0101ff01ff,
+    0xffffff0101ff0101,
+    0xffffff010100ff00,
+    0xffffff0101000000,
+    0xffffff0101000100,
+    0xffffff010101ffff,
+    0xffffff010101ff01,
+    0xffffff01010101ff,
+    0xffffff0101010101,
+    0xffff00ffff00ff00,
+    0xffff00ffff0000ff,
+    0xffff00ffff000001,
+    0xffff00ffff010000,
+    0xffff00ff00ffff00,
+    0xffff00ff00ff0100,
+    0xffff00ff00000000,
+    0xffff00ff00000101,
+    0xffff00ff000100ff,
+    0xffff00ff00010000,
+    0xffff00ff0100ff00,
+    0xffff00ff01000100,
+    0xffff00ff01010000,
+    0xffff0000ffffff00,
+    0xffff0000ffff00ff,
+    0xffff0000ffff0000,
+    0xffff0000ffff0001,
+    0xffff0000ff000000,
+    0xffff0000ff0001ff,
+    0xffff0000ff000101,
+    0xffff0000ff010100,
+    0xffff000000ffffff,
+    0xffff000000ff0000,
+    0xffff000000ff0101,
+    0xffff00000000ffff,
+    0xffff00000000ff00,
+    0xffff0000000000ff,
+    0xffff000000000000,
+    0xffff000000000001,
+    0xffff000000000100,
+    0xffff00000001ffff,
+    0xffff00000001ff01,
+    0xffff000000010000,
+    0xffff0000000101ff,
+    0xffff000000010101,
+    0xffff000001ffff00,
+    0xffff00000100ff00,
+    0xffff000001000000,
+    0xffff0000010001ff,
+    0xffff000001000101,
+    0xffff00000101ff00,
+    0xffff0000010100ff,
+    0xffff000001010000,
+    0xffff000001010001,
+    0xffff000001010100,
+    0xffff0001ff0000ff,
+    0xffff0001ff000100,
+    0xffff000100ffff00,
+    0xffff000100ff00ff,
+    0xffff00010000ffff,
+    0xffff00010000ff01,
+    0xffff000100000000,
+    0xffff0001000001ff,
+    0xffff00010001ffff,
+    0xffff00010001ff00,
+    0xffff000100010001,
+    0xffff000100010100,
+    0xffff000101ff0000,
+    0xffff00010100ff00,
+    0xffff0001010000ff,
+    0xffff000101000100,
+    0xffff01ffffffffff,
+    0xffff01ffffffff01,
+    0xffff01ffffff01ff,
+    0xffff01ffffff0101,
+    0xffff01ffff000000,
+    0xffff01ffff01ffff,
+    0xffff01ffff01ff01,
+    0xffff01ffff0101ff,
+    0xffff01ffff010101,
+    0xffff01ff00ff0000,
+    0xffff01ff0000ff00,
+    0xffff01ff00000001,
+    0xffff01ff00010000,
+    0xffff01ff01ffffff,
+    0xffff01ff01ffff01,
+    0xffff01ff01ff01ff,
+    0xffff01ff01ff0101,
+    0xffff01ff01000000,
+    0xffff01ff0101ffff,
+    0xffff01ff0101ff01,
+    0xffff01ff010101ff,
+    0xffff01ff01010101,
+    0xffff0100ffff0000,
+    0xffff0100ff00ff00,
+    0xffff0100ff0000ff,
+    0xffff0100ff000100,
+    0xffff0100ff0100ff,
+    0xffff0100ff010000,
+    0xffff010000ffff00,
+    0xffff01000000ffff,
+    0xffff01000000ff00,
+    0xffff010000000000,
+    0xffff01000001ff00,
+    0xffff0100000100ff,
+    0xffff010000010100,
+    0xffff01000100ff00,
+    0xffff0100010000ff,
+    0xffff010001000001,
+    0xffff010001000100,
+    0xffff010001010000,
+    0xffff0101ffffffff,
+    0xffff0101ffffff01,
+    0xffff0101ffff01ff,
+    0xffff0101ffff0101,
+    0xffff0101ff000000,
+    0xffff0101ff01ffff,
+    0xffff0101ff01ff01,
+    0xffff0101ff0101ff,
+    0xffff0101ff010101,
+    0xffff010100ff0000,
+    0xffff01010000ff00,
+    0xffff010100000100,
+    0xffff01010001ff00,
+    0xffff010100010000,
+    0xffff010101ffffff,
+    0xffff010101ffff01,
+    0xffff010101ff0000,
+    0xffff010101ff01ff,
+    0xffff010101ff0101,
+    0xffff010101000000,
+    0xffff01010101ffff,
+    0xffff01010101ff01,
+    0xffff0101010101ff,
+    0xffff010101010101,
+    0xff00ffffff00ffff,
+    0xff00ffffff00ff00,
+    0xff00ffffff0000ff,
+    0xff00ffffff000100,
+    0xff00ffffff0100ff,
+    0xff00ffffff010000,
+    0xff00ffff00ffff00,
+    0xff00ffff00ff00ff,
+    0xff00ffff0000ffff,
+    0xff00ffff00000000,
+    0xff00ffff000001ff,
+    0xff00ffff0001ff00,
+    0xff00ffff000100ff,
+    0xff00ffff00010000,
+    0xff00ffff00010100,
+    0xff00ffff0100ff00,
+    0xff00ffff010000ff,
+    0xff00ffff01000001,
+    0xff00ffff0101ff00,
+    0xff00ffff01010000,
+    0xff00ff00ffffff00,
+    0xff00ff00ffff00ff,
+    0xff00ff00ffff0001,
+    0xff00ff00ffff0100,
+    0xff00ff00ff00ffff,
+    0xff00ff00ff00ff01,
+    0xff00ff00ff000000,
+    0xff00ff00ff0001ff,
+    0xff00ff00ff01ff00,
+    0xff00ff00ff0100ff,
+    0xff00ff00ff010100,
+    0xff00ff0000ff0000,
+    0xff00ff0000ff0101,
+    0xff00ff000000ffff,
+    0xff00ff000000ff00,
+    0xff00ff000000ff01,
+    0xff00ff00000000ff,
+    0xff00ff0000000000,
+    0xff00ff0000000001,
+    0xff00ff0000000100,
+    0xff00ff000001ffff,
+    0xff00ff0000010000,
+    0xff00ff0001ff00ff,
+    0xff00ff000100ff01,
+    0xff00ff0001000000,
+    0xff00ff000101ff00,
+    0xff00ff00010100ff,
+    0xff00ff01ff00ff00,
+    0xff00ff01ff0000ff,
+    0xff00ff01ff000001,
+    0xff00ff01ff010000,
+    0xff00ff0100ffffff,
+    0xff00ff0100ff0001,
+    0xff00ff0100ff0100,
+    0xff00ff010000ff01,
+    0xff00ff0100000000,
+    0xff00ff01000001ff,
+    0xff00ff0100000101,
+    0xff00ff01000100ff,
+    0xff00ff0100010001,
+    0xff00ff0101ff0000,
+    0xff00ff010100ff00,
+    0xff00ff01010000ff,
+    0xff00ff0101000001,
+    0xff00ff0101010000,
+    0xff0000ffffffff00,
+    0xff0000ffffff0001,
+    0xff0000ffffff0100,
+    0xff0000ffff0000ff,
+    0xff0000ffff000000,
+    0xff0000ffff0001ff,
+    0xff0000ffff000100,
+    0xff0000ffff01ff00,
+    0xff0000ffff010001,
+    0xff0000ff00ffff00,
+    0xff0000ff00ff0000,
+    0xff0000ff00ff0001,
+    0xff0000ff00ff01ff,
+    0xff0000ff00ff0101,
+    0xff0000ff0000ff00,
+    0xff0000ff000000ff,
+    0xff0000ff00000000,
+    0xff0000ff00000001,
+    0xff0000ff00000100,
+    0xff0000ff0001ff01,
+    0xff0000ff00010000,
+    0xff0000ff000101ff,
+    0xff0000ff01ff00ff,
+    0xff0000ff01ff0100,
+    0xff0000ff0100ffff,
+    0xff0000ff010000ff,
+    0xff0000ff01000000,
+    0xff0000ff010001ff,
+    0xff0000ff01000100,
+    0xff0000ff01000101,
+    0xff0000ff0101ff00,
+    0xff0000ff010100ff,
+    0xff0000ff01010000,
+    0xff0000ff01010100,
+    0xff000000ffffff01,
+    0xff000000ffff0000,
+    0xff000000ffff0101,
+    0xff000000ff00ff00,
+    0xff000000ff0000ff,
+    0xff000000ff000000,
+    0xff000000ff000001,
+    0xff000000ff000100,
+    0xff000000ff01ffff,
+    0xff000000ff01ff01,
+    0xff000000ff010000,
+    0xff000000ff0101ff,
+    0xff000000ff010101,
+    0xff00000000ffff00,
+    0xff00000000ff00ff,
+    0xff00000000ff0000,
+    0xff00000000ff0001,
+    0xff0000000000ff00,
+    0xff0000000000ff01,
+    0xff000000000000ff,
+    0xff00000000000000,
+    0xff00000000000001,
+    0xff00000000000100,
+    0xff00000000000101,
+    0xff0000000001ff00,
+    0xff000000000100ff,
+    0xff00000000010000,
+    0xff00000000010001,
+    0xff00000000010100,
+    0xff00000001ffffff,
+    0xff00000001ffff01,
+    0xff00000001ff00ff,
+    0xff00000001ff0000,
+    0xff00000001ff01ff,
+    0xff00000001ff0101,
+    0xff0000000100ffff,
+    0xff0000000100ff00,
+    0xff000000010000ff,
+    0xff00000001000000,
+    0xff00000001000001,
+    0xff00000001000100,
+    0xff00000001000101,
+    0xff0000000101ffff,
+    0xff0000000101ff01,
+    0xff00000001010000,
+    0xff000001ffffff00,
+    0xff000001ffff00ff,
+    0xff000001ffff0000,
+    0xff000001ffff0001,
+    0xff000001ff000000,
+    0xff000001ff000001,
+    0xff000001ff0001ff,
+    0xff000001ff000101,
+    0xff000001ff01ff00,
+    0xff000001ff010001,
+    0xff00000100ffffff,
+    0xff00000100ffff01,
+    0xff00000100ff00ff,
+    0xff00000100ff0000,
+    0xff00000100ff01ff,
+    0xff00000100ff0101,
+    0xff0000010000ff00,
+    0xff00000100000000,
+    0xff00000100000001,
+    0xff000001000001ff,
+    0xff00000100000100,
+    0xff0000010001ff00,
+    0xff000001000100ff,
+    0xff00000100010000,
+    0xff000001000101ff,
+    0xff00000100010100,
+    0xff00000100010101,
+    0xff00000101ff0001,
+    0xff00000101ff0101,
+    0xff0000010100ff01,
+    0xff00000101000000,
+    0xff000001010100ff,
+    0xff00000101010100,
+    0xff0001ffff00ff00,
+    0xff0001ffff000001,
+    0xff0001ffff010000,
+    0xff0001ff00ffff00,
+    0xff0001ff00ff00ff,
+    0xff0001ff00ff0001,
+    0xff0001ff00ff0100,
+    0xff0001ff0000ffff,
+    0xff0001ff00000000,
+    0xff0001ff000001ff,
+    0xff0001ff00000101,
+    0xff0001ff0001ffff,
+    0xff0001ff0001ff00,
+    0xff0001ff000100ff,
+    0xff0001ff00010001,
+    0xff0001ff00010100,
+    0xff0001ff01ff0000,
+    0xff0001ff0100ff00,
+    0xff0001ff010000ff,
+    0xff0001ff01010000,
+    0xff000100ff00ffff,
+    0xff000100ff00ff01,
+    0xff000100ff000000,
+    0xff000100ff000101,
+    0xff000100ff01ff00,
+    0xff000100ff010000,
+    0xff00010000ffff01,
+    0xff00010000ff00ff,
+    0xff00010000ff0000,
+    0xff00010000ff01ff,
+    0xff0001000000ff00,
+    0xff000100000000ff,
+    0xff00010000000000,
+    0xff00010000000001,
+    0xff00010000000100,
+    0xff00010000000101,
+    0xff0001000001ffff,
+    0xff00010000010000,
+    0xff00010000010101,
+    0xff00010001ff0100,
+    0xff0001000100ff00,
+    0xff0001000100ff01,
+    0xff00010001000000,
+    0xff000100010001ff,
+    0xff0001000101ff00,
+    0xff00010001010001,
+    0xff00010001010100,
+    0xff000101ffff0100,
+    0xff000101ff000001,
+    0xff000101ff0100ff,
+    0xff000101ff010001,
+    0xff00010100ff00ff,
+    0xff00010100ff0001,
+    0xff00010100ff0100,
+    0xff0001010000ffff,
+    0xff0001010000ff01,
+    0xff00010100000000,
+    0xff000101000001ff,
+    0xff0001010001ff00,
+    0xff00010100010001,
+    0xff00010100010100,
+    0xff00010101ff0000,
+    0xff0001010100ff00,
+    0xff00010101000001,
+    0xff00010101000101,
+    0xff01ffffffffffff,
+    0xff01ffffffffff01,
+    0xff01ffffffff01ff,
+    0xff01ffffffff0101,
+    0xff01ffffff000000,
+    0xff01ffffff01ffff,
+    0xff01ffffff01ff01,
+    0xff01ffffff010000,
+    0xff01ffffff0101ff,
+    0xff01ffffff010101,
+    0xff01ffff00ff0000,
+    0xff01ffff0000ff00,
+    0xff01ffff00000100,
+    0xff01ffff0001ff00,
+    0xff01ffff00010000,
+    0xff01ffff01ffffff,
+    0xff01ffff01ffff01,
+    0xff01ffff01ff01ff,
+    0xff01ffff01ff0101,
+    0xff01ffff01000000,
+    0xff01ffff0101ffff,
+    0xff01ffff0101ff01,
+    0xff01ffff01010000,
+    0xff01ffff010101ff,
+    0xff01ffff01010101,
+    0xff01ff00ffff0000,
+    0xff01ff00ff00ff00,
+    0xff01ff00ff0000ff,
+    0xff01ff00ff000100,
+    0xff01ff00ff010000,
+    0xff01ff0000ffff01,
+    0xff01ff0000ff00ff,
+    0xff01ff0000ff0100,
+    0xff01ff0000000000,
+    0xff01ff00000001ff,
+    0xff01ff0000000101,
+    0xff01ff000001ff00,
+    0xff01ff00000100ff,
+    0xff01ff0000010000,
+    0xff01ff0000010001,
+    0xff01ff0001ff0000,
+    0xff01ff000100ffff,
+    0xff01ff0001000001,
+    0xff01ff0001000100,
+    0xff01ff0001010000,
+    0xff01ff01ffffff00,
+    0xff01ff01ffff01ff,
+    0xff01ff01ffff0101,
+    0xff01ff01ff00ff00,
+    0xff01ff01ff000000,
+    0xff01ff01ff01ffff,
+    0xff01ff01ff01ff01,
+    0xff01ff01ff0101ff,
+    0xff01ff01ff010101,
+    0xff01ff0100ff0000,
+    0xff01ff010000ff00,
+    0xff01ff0100000001,
+    0xff01ff0100000100,
+    0xff01ff0100010000,
+    0xff01ff0101ffff00,
+    0xff01ff0101ff01ff,
+    0xff01ff0101ff0101,
+    0xff01ff010100ff00,
+    0xff01ff0101000000,
+    0xff01ff010101ffff,
+    0xff01ff010101ff01,
+    0xff01ff01010101ff,
+    0xff01ff0101010101,
+    0xff0100ffffff0000,
+    0xff0100ffff0000ff,
+    0xff0100ffff000001,
+    0xff0100ffff000100,
+    0xff0100ffff010000,
+    0xff0100ff00ff00ff,
+    0xff0100ff00ff0000,
+    0xff0100ff00ff0001,
+    0xff0100ff00ff0100,
+    0xff0100ff0000ff01,
+    0xff0100ff00000000,
+    0xff0100ff000001ff,
+    0xff0100ff00000101,
+    0xff0100ff00010001,
+    0xff0100ff01ff0000,
+    0xff0100ff0100ff00,
+    0xff0100ff010000ff,
+    0xff0100ff01000100,
+    0xff0100ff0101ff00,
+    0xff0100ff01010000,
+    0xff010000ffff0100,
+    0xff010000ff000000,
+    0xff010000ff01ff00,
+    0xff010000ff010100,
+    0xff01000000ffffff,
+    0xff01000000ff0000,
+    0xff01000000ff01ff,
+    0xff0100000000ff00,
+    0xff010000000000ff,
+    0xff01000000000000,
+    0xff01000000000100,
+    0xff0100000001ff01,
+    0xff01000000010000,
+    0xff010000000101ff,
+    0xff01000001ff0100,
+    0xff0100000100ffff,
+    0xff010000010000ff,
+    0xff01000001000000,
+    0xff010000010001ff,
+    0xff01000001000101,
+    0xff0100000101ff00,
+    0xff010000010100ff,
+    0xff01000001010001,
+    0xff01000001010100,
+    0xff010001ffff0000,
+    0xff010001ff00ffff,
+    0xff010001ff00ff01,
+    0xff010001ff000100,
+    0xff010001ff010000,
+    0xff01000100ffff00,
+    0xff01000100ff0100,
+    0xff01000100000000,
+    0xff0100010001ffff,
+    0xff0100010001ff00,
+    0xff01000100010100,
+    0xff01000101ff00ff,
+    0xff01000101ff0001,
+    0xff0100010100ffff,
+    0xff01000101000101,
+    0xff0101ffffffffff,
+    0xff0101ffffffff01,
+    0xff0101ffffff01ff,
+    0xff0101ffffff0101,
+    0xff0101ffff000000,
+    0xff0101ffff01ffff,
+    0xff0101ffff01ff01,
+    0xff0101ffff0101ff,
+    0xff0101ffff010101,
+    0xff0101ff00ff0000,
+    0xff0101ff0000ff00,
+    0xff0101ff000000ff,
+    0xff0101ff00010000,
+    0xff0101ff01ffffff,
+    0xff0101ff01ffff01,
+    0xff0101ff01ff01ff,
+    0xff0101ff01ff0101,
+    0xff0101ff0101ffff,
+    0xff0101ff0101ff01,
+    0xff0101ff010101ff,
+    0xff0101ff01010101,
+    0xff010100ffff0100,
+    0xff010100ff00ff00,
+    0xff010100ff0000ff,
+    0xff010100ff000100,
+    0xff010100ff010000,
+    0xff01010000ff0001,
+    0xff01010000ff0100,
+    0xff0101000000ff01,
+    0xff01010000000000,
+    0xff0101000001ff00,
+    0xff010100000100ff,
+    0xff01010000010001,
+    0xff01010000010100,
+    0xff01010001ff0000,
+    0xff0101000100ffff,
+    0xff01010001000001,
+    0xff01010001000100,
+    0xff010100010100ff,
+    0xff01010001010000,
+    0xff010101ffffffff,
+    0xff010101ffffff01,
+    0xff010101ffff01ff,
+    0xff010101ffff0101,
+    0xff010101ff01ffff,
+    0xff010101ff01ff01,
+    0xff010101ff0101ff,
+    0xff010101ff010101,
+    0xff01010100ff0000,
+    0xff0101010000ff00,
+    0xff01010100000001,
+    0xff01010100000100,
+    0xff01010100010000,
+    0xff01010101ffffff,
+    0xff01010101ffff01,
+    0xff01010101ff01ff,
+    0xff01010101ff0101,
+    0xff01010101000000,
+    0xff0101010101ffff,
+    0xff0101010101ff01,
+    0xff010101010101ff,
+    0xff01010101010101,
+    0x00ffffffffff0000,
+    0x00ffffffff00ff00,
+    0x00ffffffff000001,
+    0x00ffffffff010000,
+    0x00ffffff00ff0100,
+    0x00ffffff0000ff01,
+    0x00ffffff00000000,
+    0x00ffffff000001ff,
+    0x00ffffff00000101,
+    0x00ffffff0001ff00,
+    0x00ffffff000100ff,
+    0x00ffffff00010001,
+    0x00ffffff010000ff,
+    0x00ffffff01000100,
+    0x00ffffff0101ff00,
+    0x00ffffff01010001,
+    0x00ffff00ffffffff,
+    0x00ffff00ffffff00,
+    0x00ffff00ffff00ff,
+    0x00ffff00ffff0001,
+    0x00ffff00ffff0100,
+    0x00ffff00ff00ff01,
+    0x00ffff00ff000000,
+    0x00ffff00ff000001,
+    0x00ffff00ff0001ff,
+    0x00ffff00ff000101,
+    0x00ffff00ff01ff00,
+    0x00ffff00ff010001,
+    0x00ffff00ff010100,
+    0x00ffff0000ff0000,
+    0x00ffff0000ff01ff,
+    0x00ffff0000ff0101,
+    0x00ffff000000ff00,
+    0x00ffff00000000ff,
+    0x00ffff0000000000,
+    0x00ffff0000000001,
+    0x00ffff0000000100,
+    0x00ffff0000000101,
+    0x00ffff0000010000,
+    0x00ffff00000101ff,
+    0x00ffff0000010101,
+    0x00ffff0001ffff00,
+    0x00ffff0001ff00ff,
+    0x00ffff0001ff0001,
+    0x00ffff000100ffff,
+    0x00ffff000100ff01,
+    0x00ffff0001000000,
+    0x00ffff000101ffff,
+    0x00ffff000101ff00,
+    0x00ffff000101ff01,
+    0x00ffff01ffff0000,
+    0x00ffff01ff00ff00,
+    0x00ffff01ff0000ff,
+    0x00ffff01ff000001,
+    0x00ffff01ff010000,
+    0x00ffff0100ffff00,
+    0x00ffff010000ff01,
+    0x00ffff0100000000,
+    0x00ffff0100000101,
+    0x00ffff01000100ff,
+    0x00ffff0100010100,
+    0x00ffff0101ff0100,
+    0x00ffff01010000ff,
+    0x00ffff0101010000,
+    0x00ff00ffffffff00,
+    0x00ff00ffff000000,
+    0x00ff00ffff000100,
+    0x00ff00ffff010100,
+    0x00ff00ff00ff0000,
+    0x00ff00ff00ff01ff,
+    0x00ff00ff00ff0101,
+    0x00ff00ff0000ff00,
+    0x00ff00ff000000ff,
+    0x00ff00ff00000000,
+    0x00ff00ff00000001,
+    0x00ff00ff0001ff00,
+    0x00ff00ff0001ff01,
+    0x00ff00ff00010000,
+    0x00ff00ff000101ff,
+    0x00ff00ff00010101,
+    0x00ff00ff01ffff00,
+    0x00ff00ff01ff0001,
+    0x00ff00ff01ff0100,
+    0x00ff00ff0100ffff,
+    0x00ff00ff0100ff01,
+    0x00ff00ff01000000,
+    0x00ff00ff0101ffff,
+    0x00ff00ff0101ff00,
+    0x00ff00ff01010100,
+    0x00ff0000ffffff00,
+    0x00ff0000ffffff01,
+    0x00ff0000ffff0000,
+    0x00ff0000ffff0101,
+    0x00ff0000ff00ff00,
+    0x00ff0000ff0000ff,
+    0x00ff0000ff000000,
+    0x00ff0000ff000001,
+    0x00ff0000ff000100,
+    0x00ff0000ff01ffff,
+    0x00ff0000ff010000,
+    0x00ff0000ff010101,
+    0x00ff000000ffff00,
+    0x00ff000000ff00ff,
+    0x00ff000000ff0000,
+    0x00ff000000ff0001,
+    0x00ff000000ff0100,
+    0x00ff00000000ffff,
+    0x00ff00000000ff00,
+    0x00ff0000000000ff,
+    0x00ff000000000000,
+    0x00ff000000000001,
+    0x00ff0000000001ff,
+    0x00ff000000000100,
+    0x00ff00000001ff00,
+    0x00ff0000000100ff,
+    0x00ff000000010000,
+    0x00ff000000010001,
+    0x00ff000000010100,
+    0x00ff000001ffff01,
+    0x00ff000001ff00ff,
+    0x00ff000001ff0000,
+    0x00ff000001ff01ff,
+    0x00ff00000100ff00,
+    0x00ff0000010000ff,
+    0x00ff000001000000,
+    0x00ff000001000001,
+    0x00ff000001000100,
+    0x00ff000001000101,
+    0x00ff000001010000,
+    0x00ff0000010101ff,
+    0x00ff000001010101,
+    0x00ff0001ffffff00,
+    0x00ff0001ffff0000,
+    0x00ff0001ffff0100,
+    0x00ff0001ff0000ff,
+    0x00ff0001ff000000,
+    0x00ff0001ff0001ff,
+    0x00ff0001ff000101,
+    0x00ff0001ff01ff00,
+    0x00ff0001ff0100ff,
+    0x00ff0001ff010100,
+    0x00ff000100ffffff,
+    0x00ff000100ffff01,
+    0x00ff000100ff0000,
+    0x00ff000100ff01ff,
+    0x00ff00010000ffff,
+    0x00ff00010000ff00,
+    0x00ff00010000ff01,
+    0x00ff000100000000,
+    0x00ff000100000001,
+    0x00ff000100000100,
+    0x00ff00010001ff01,
+    0x00ff000100010000,
+    0x00ff0001000101ff,
+    0x00ff000101ffff00,
+    0x00ff000101ff0000,
+    0x00ff000101ff0101,
+    0x00ff0001010000ff,
+    0x00ff000101000000,
+    0x00ff00010101ff00,
+    0x00ff0001010100ff,
+    0x00ff000101010001,
+    0x00ff01ffffff0000,
+    0x00ff01ffff00ff00,
+    0x00ff01ffff000000,
+    0x00ff01ffff000101,
+    0x00ff01ffff010000,
+    0x00ff01ff00ffff01,
+    0x00ff01ff00ff0100,
+    0x00ff01ff0000ffff,
+    0x00ff01ff00000000,
+    0x00ff01ff000001ff,
+    0x00ff01ff0001ff00,
+    0x00ff01ff000100ff,
+    0x00ff01ff00010001,
+    0x00ff01ff00010100,
+    0x00ff01ff01ff0000,
+    0x00ff01ff0100ff00,
+    0x00ff01ff010000ff,
+    0x00ff01ff01000001,
+    0x00ff01ff01000100,
+    0x00ff01ff01010000,
+    0x00ff0100ffffff00,
+    0x00ff0100ffff0000,
+    0x00ff0100ffff0001,
+    0x00ff0100ffff0101,
+    0x00ff0100ff00ffff,
+    0x00ff0100ff0000ff,
+    0x00ff0100ff000000,
+    0x00ff0100ff0001ff,
+    0x00ff0100ff01ff00,
+    0x00ff0100ff0100ff,
+    0x00ff0100ff010001,
+    0x00ff010000ffffff,
+    0x00ff010000ff0000,
+    0x00ff010000ff0101,
+    0x00ff01000000ff00,
+    0x00ff01000000ff01,
+    0x00ff0100000000ff,
+    0x00ff010000000000,
+    0x00ff010000000001,
+    0x00ff010000000100,
+    0x00ff01000001ffff,
+    0x00ff01000001ff01,
+    0x00ff010000010000,
+    0x00ff010000010001,
+    0x00ff010000010101,
+    0x00ff010001ff0001,
+    0x00ff010001ff0100,
+    0x00ff01000100ff01,
+    0x00ff010001000000,
+    0x00ff010001000001,
+    0x00ff0100010001ff,
+    0x00ff01000101ff00,
+    0x00ff0100010100ff,
+    0x00ff010001010001,
+    0x00ff010001010100,
+    0x00ff0101ff000001,
+    0x00ff010100ff00ff,
+    0x00ff010100ff0001,
+    0x00ff010100ff0100,
+    0x00ff010100000000,
+    0x00ff0101000001ff,
+    0x00ff010100000101,
+    0x00ff0101000100ff,
+    0x00ff010100010100,
+    0x00ff0101010000ff,
+    0x00ff010101010000,
+    0x0000ffffffffff00,
+    0x0000ffffffff00ff,
+    0x0000ffffffff0000,
+    0x0000ffffffff0001,
+    0x0000ffffffff0100,
+    0x0000ffffff00ff01,
+    0x0000ffffff000000,
+    0x0000ffffff000101,
+    0x0000ffffff01ff00,
+    0x0000ffffff0100ff,
+    0x0000ffffff010100,
+    0x0000ffff00ffffff,
+    0x0000ffff00ff0000,
+    0x0000ffff00ff01ff,
+    0x0000ffff0000ff00,
+    0x0000ffff000000ff,
+    0x0000ffff00000000,
+    0x0000ffff00000001,
+    0x0000ffff00000100,
+    0x0000ffff00010000,
+    0x0000ffff000101ff,
+    0x0000ffff01ff0001,
+    0x0000ffff01ff0100,
+    0x0000ffff01000000,
+    0x0000ffff010001ff,
+    0x0000ffff0101ffff,
+    0x0000ffff0101ff00,
+    0x0000ffff01010001,
+    0x0000ffff01010100,
+    0x0000ff00ffff0000,
+    0x0000ff00ffff01ff,
+    0x0000ff00ffff0100,
+    0x0000ff00ffff0101,
+    0x0000ff00ff00ff00,
+    0x0000ff00ff0000ff,
+    0x0000ff00ff000000,
+    0x0000ff00ff000001,
+    0x0000ff00ff0001ff,
+    0x0000ff00ff000100,
+    0x0000ff00ff01ffff,
+    0x0000ff00ff010000,
+    0x0000ff00ff010001,
+    0x0000ff00ff0101ff,
+    0x0000ff00ff010101,
+    0x0000ff0000ffff00,
+    0x0000ff0000ff00ff,
+    0x0000ff0000ff0000,
+    0x0000ff0000ff0001,
+    0x0000ff0000ff0100,
+    0x0000ff000000ffff,
+    0x0000ff000000ff00,
+    0x0000ff000000ff01,
+    0x0000ff00000000ff,
+    0x0000ff0000000000,
+    0x0000ff0000000001,
+    0x0000ff00000001ff,
+    0x0000ff0000000100,
+    0x0000ff0000000101,
+    0x0000ff000001ff00,
+    0x0000ff00000100ff,
+    0x0000ff0000010000,
+    0x0000ff0000010001,
+    0x0000ff0000010100,
+    0x0000ff0001ffff01,
+    0x0000ff0001ff0000,
+    0x0000ff000100ff00,
+    0x0000ff00010000ff,
+    0x0000ff0001000000,
+    0x0000ff0001000001,
+    0x0000ff0001000100,
+    0x0000ff000101ffff,
+    0x0000ff0001010000,
+    0x0000ff0001010101,
+    0x0000ff01ffffff00,
+    0x0000ff01ffff0001,
+    0x0000ff01ff00ff01,
+    0x0000ff01ff000000,
+    0x0000ff01ff000101,
+    0x0000ff01ff01ff00,
+    0x0000ff01ff0100ff,
+    0x0000ff0100ffff01,
+    0x0000ff0100ff0000,
+    0x0000ff0100ff0101,
+    0x0000ff010000ff00,
+    0x0000ff01000000ff,
+    0x0000ff0100000000,
+    0x0000ff0100000001,
+    0x0000ff0100000100,
+    0x0000ff010001ff01,
+    0x0000ff0100010000,
+    0x0000ff0101ff0000,
+    0x0000ff010100ffff,
+    0x0000ff010100ff01,
+    0x0000ff0101000000,
+    0x0000ff0101000100,
+    0x0000ff0101000101,
+    0x0000ff01010100ff,
+    0x000000ffffff00ff,
+    0x000000ffffff0000,
+    0x000000ffff00ff00,
+    0x000000ffff0000ff,
+    0x000000ffff000000,
+    0x000000ffff000001,
+    0x000000ffff0001ff,
+    0x000000ffff000100,
+    0x000000ffff01ff00,
+    0x000000ffff010000,
+    0x000000ffff0101ff,
+    0x000000ffff010101,
+    0x000000ff00ffff00,
+    0x000000ff00ff00ff,
+    0x000000ff00ff0000,
+    0x000000ff00ff0001,
+    0x000000ff00ff0100,
+    0x000000ff00ff0101,
+    0x000000ff0000ffff,
+    0x000000ff0000ff00,
+    0x000000ff000000ff,
+    0x000000ff00000000,
+    0x000000ff00000001,
+    0x000000ff000001ff,
+    0x000000ff00000100,
+    0x000000ff00000101,
+    0x000000ff0001ff00,
+    0x000000ff0001ff01,
+    0x000000ff000100ff,
+    0x000000ff00010000,
+    0x000000ff00010001,
+    0x000000ff00010100,
+    0x000000ff01ffffff,
+    0x000000ff01ff01ff,
+    0x000000ff01ff0101,
+    0x000000ff0100ff00,
+    0x000000ff010000ff,
+    0x000000ff01000000,
+    0x000000ff01000001,
+    0x000000ff01000100,
+    0x000000ff0101ff00,
+    0x000000ff010100ff,
+    0x000000ff01010000,
+    0x000000ff01010101,
+    0x00000000ffffff00,
+    0x00000000ffffff01,
+    0x00000000ffff00ff,
+    0x00000000ffff0000,
+    0x00000000ffff0001,
+    0x00000000ffff0100,
+    0x00000000ff00ffff,
+    0x00000000ff00ff00,
+    0x00000000ff00ff01,
+    0x00000000ff0000ff,
+    0x00000000ff000000,
+    0x00000000ff000001,
+    0x00000000ff000100,
+    0x00000000ff000101,
+    0x00000000ff01ff00,
+    0x00000000ff0100ff,
+    0x00000000ff010000,
+    0x00000000ff010001,
+    0x00000000ff010100,
+    0x0000000000ffffff,
+    0x0000000000ffff00,
+    0x0000000000ffff01,
+    0x0000000000ff00ff,
+    0x0000000000ff0000,
+    0x0000000000ff0001,
+    0x0000000000ff01ff,
+    0x0000000000ff0100,
+    0x000000000000ffff,
+    0x000000000000ff00,
+    0x000000000000ff01,
+    0x00000000000000ff,
+    0x0000000000000000,
+    0x0000000000000001,
+    0x00000000000001ff,
+    0x0000000000000100,
+    0x0000000000000101,
+    0x000000000001ffff,
+    0x000000000001ff00,
+    0x00000000000100ff,
+    0x0000000000010000,
+    0x0000000000010001,
+    0x00000000000101ff,
+    0x0000000000010100,
+    0x0000000000010101,
+    0x0000000001ffff00,
+    0x0000000001ff00ff,
+    0x0000000001ff0000,
+    0x0000000001ff0100,
+    0x0000000001ff0101,
+    0x000000000100ffff,
+    0x000000000100ff00,
+    0x00000000010000ff,
+    0x0000000001000000,
+    0x0000000001000001,
+    0x00000000010001ff,
+    0x0000000001000100,
+    0x000000000101ff00,
+    0x00000000010100ff,
+    0x0000000001010000,
+    0x0000000001010001,
+    0x0000000001010100,
+    0x00000001ffffffff,
+    0x00000001ffffff00,
+    0x00000001ffffff01,
+    0x00000001ffff00ff,
+    0x00000001ffff0001,
+    0x00000001ffff01ff,
+    0x00000001ffff0100,
+    0x00000001ff00ff00,
+    0x00000001ff0000ff,
+    0x00000001ff000000,
+    0x00000001ff0001ff,
+    0x00000001ff000100,
+    0x00000001ff01ffff,
+    0x00000001ff01ff00,
+    0x00000001ff01ff01,
+    0x00000001ff0100ff,
+    0x00000001ff010000,
+    0x00000001ff010001,
+    0x00000001ff0101ff,
+    0x00000001ff010100,
+    0x0000000100ffff00,
+    0x0000000100ff0000,
+    0x0000000100ff0001,
+    0x0000000100ff01ff,
+    0x0000000100ff0100,
+    0x0000000100ff0101,
+    0x000000010000ffff,
+    0x000000010000ff00,
+    0x000000010000ff01,
+    0x00000001000000ff,
+    0x0000000100000000,
+    0x0000000100000001,
+    0x00000001000001ff,
+    0x0000000100000100,
+    0x0000000100000101,
+    0x000000010001ff00,
+    0x00000001000100ff,
+    0x0000000100010000,
+    0x0000000100010100,
+    0x0000000101ffff01,
+    0x0000000101ff0000,
+    0x0000000101ff0001,
+    0x0000000101ff01ff,
+    0x0000000101ff0100,
+    0x0000000101ff0101,
+    0x000000010100ff00,
+    0x0000000101000000,
+    0x0000000101000101,
+    0x000000010101ff01,
+    0x0000000101010000,
+    0x0000000101010001,
+    0x00000001010101ff,
+    0x0000000101010100,
+    0x000001ffffff00ff,
+    0x000001ffffff0000,
+    0x000001ffffff0001,
+    0x000001ffffff0100,
+    0x000001ffff00ffff,
+    0x000001ffff000000,
+    0x000001ffff0001ff,
+    0x000001ffff01ff00,
+    0x000001ffff010101,
+    0x000001ff00ff0000,
+    0x000001ff00ff01ff,
+    0x000001ff00ff0101,
+    0x000001ff0000ff00,
+    0x000001ff000000ff,
+    0x000001ff00000000,
+    0x000001ff00000001,
+    0x000001ff000001ff,
+    0x000001ff00000100,
+    0x000001ff0001ffff,
+    0x000001ff0001ff01,
+    0x000001ff000100ff,
+    0x000001ff00010000,
+    0x000001ff01ffff01,
+    0x000001ff01ff0100,
+    0x000001ff0100ffff,
+    0x000001ff0100ff01,
+    0x000001ff01000000,
+    0x000001ff010001ff,
+    0x000001ff0101ff00,
+    0x000001ff01010100,
+    0x00000100ffffff00,
+    0x00000100ffffff01,
+    0x00000100ffff0000,
+    0x00000100ffff0101,
+    0x00000100ff00ff00,
+    0x00000100ff0000ff,
+    0x00000100ff000000,
+    0x00000100ff000001,
+    0x00000100ff000100,
+    0x00000100ff010000,
+    0x0000010000ffff00,
+    0x0000010000ff00ff,
+    0x0000010000ff0000,
+    0x0000010000ff0001,
+    0x0000010000ff0100,
+    0x000001000000ffff,
+    0x000001000000ff00,
+    0x000001000000ff01,
+    0x00000100000000ff,
+    0x0000010000000000,
+    0x0000010000000001,
+    0x00000100000001ff,
+    0x0000010000000100,
+    0x0000010000000101,
+    0x000001000001ff00,
+    0x00000100000100ff,
+    0x0000010000010000,
+    0x0000010000010001,
+    0x0000010000010100,
+    0x0000010001ffff00,
+    0x0000010001ff0000,
+    0x0000010001ff0100,
+    0x000001000100ff00,
+    0x00000100010000ff,
+    0x0000010001000000,
+    0x0000010001000001,
+    0x00000100010001ff,
+    0x0000010001000100,
+    0x0000010001010000,
+    0x00000101ffff00ff,
+    0x00000101ffff01ff,
+    0x00000101ff000000,
+    0x00000101ff000101,
+    0x00000101ff01ffff,
+    0x00000101ff010000,
+    0x00000101ff010001,
+    0x00000101ff010100,
+    0x0000010100ff0000,
+    0x0000010100ff01ff,
+    0x0000010100ff0100,
+    0x000001010000ff00,
+    0x0000010100000000,
+    0x0000010100000001,
+    0x00000101000001ff,
+    0x0000010100000100,
+    0x000001010001ff01,
+    0x0000010100010000,
+    0x00000101000101ff,
+    0x0000010100010101,
+    0x0000010101ffff00,
+    0x0000010101ff0101,
+    0x000001010100ff01,
+    0x0000010101000000,
+    0x0000010101000001,
+    0x00000101010001ff,
+    0x0000010101000101,
+    0x000001010101ff00,
+    0x0001ffffffff0000,
+    0x0001ffffff0000ff,
+    0x0001ffffff000001,
+    0x0001ffffff000100,
+    0x0001ffffff010000,
+    0x0001ffff00ff00ff,
+    0x0001ffff0000ffff,
+    0x0001ffff00000000,
+    0x0001ffff00000001,
+    0x0001ffff000001ff,
+    0x0001ffff00000101,
+    0x0001ffff0001ff00,
+    0x0001ffff000100ff,
+    0x0001ffff00010001,
+    0x0001ffff00010100,
+    0x0001ffff01ffff00,
+    0x0001ffff01000001,
+    0x0001ffff01010000,
+    0x0001ff00ffffff00,
+    0x0001ff00ffff00ff,
+    0x0001ff00ffff0001,
+    0x0001ff00ffff0100,
+    0x0001ff00ff00ff01,
+    0x0001ff00ff000000,
+    0x0001ff00ff01ff00,
+    0x0001ff00ff01ff01,
+    0x0001ff00ff010001,
+    0x0001ff00ff010100,
+    0x0001ff0000ff0000,
+    0x0001ff0000ff0100,
+    0x0001ff000000ff00,
+    0x0001ff0000000000,
+    0x0001ff0000000001,
+    0x0001ff0000000100,
+    0x0001ff0000010000,
+    0x0001ff0000010001,
+    0x0001ff0000010101,
+    0x0001ff0001ff00ff,
+    0x0001ff0001ff0101,
+    0x0001ff000100ff01,
+    0x0001ff0001000000,
+    0x0001ff000101ff00,
+    0x0001ff0001010001,
+    0x0001ff0001010100,
+    0x0001ff01ff00ff00,
+    0x0001ff01ff000001,
+    0x0001ff01ff000100,
+    0x0001ff0100ffffff,
+    0x0001ff0100ffff00,
+    0x0001ff0100ff0001,
+    0x0001ff0100000000,
+    0x0001ff0100000001,
+    0x0001ff01000001ff,
+    0x0001ff010001ffff,
+    0x0001ff0101ff0000,
+    0x0001ff010100ff00,
+    0x0001ff0101000001,
+    0x0001ff0101010000,
+    0x000100ffff00ff00,
+    0x000100ffff00ff01,
+    0x000100ffff000000,
+    0x000100ffff000001,
+    0x000100ffff000101,
+    0x000100ffff01ff00,
+    0x000100ffff010001,
+    0x000100ffff010100,
+    0x000100ff00ffffff,
+    0x000100ff00ffff01,
+    0x000100ff00ff0000,
+    0x000100ff00ff01ff,
+    0x000100ff00ff0101,
+    0x000100ff0000ff00,
+    0x000100ff000000ff,
+    0x000100ff00000000,
+    0x000100ff00000001,
+    0x000100ff00000100,
+    0x000100ff00000101,
+    0x000100ff0001ffff,
+    0x000100ff0001ff01,
+    0x000100ff00010000,
+    0x000100ff01ff00ff,
+    0x000100ff01ff0000,
+    0x000100ff01ff0100,
+    0x000100ff0100ffff,
+    0x000100ff0100ff01,
+    0x000100ff010000ff,
+    0x000100ff01000000,
+    0x000100ff01000001,
+    0x000100ff010001ff,
+    0x000100ff01000101,
+    0x000100ff0101ff00,
+    0x000100ff010100ff,
+    0x000100ff01010100,
+    0x00010000ffff0000,
+    0x00010000ffff01ff,
+    0x00010000ffff0101,
+    0x00010000ff00ff00,
+    0x00010000ff000000,
+    0x00010000ff000001,
+    0x00010000ff000100,
+    0x0001000000ff00ff,
+    0x0001000000ff0000,
+    0x0001000000ff0001,
+    0x0001000000ff0100,
+    0x000100000000ffff,
+    0x000100000000ff00,
+    0x00010000000000ff,
+    0x0001000000000000,
+    0x0001000000000001,
+    0x0001000000000100,
+    0x000100000001ff00,
+    0x00010000000100ff,
+    0x0001000000010000,
+    0x0001000000010001,
+    0x0001000000010100,
+    0x0001000001ff0001,
+    0x0001000001ff0100,
+    0x0001000001ff0101,
+    0x000100000100ff00,
+    0x0001000001000000,
+    0x0001000001000001,
+    0x0001000001000100,
+    0x0001000001000101,
+    0x000100000101ff01,
+    0x0001000001010000,
+    0x0001000001010001,
+    0x00010000010101ff,
+    0x00010001ffffff01,
+    0x00010001ffff0100,
+    0x00010001ff000000,
+    0x00010001ff01ffff,
+    0x00010001ff010001,
+    0x00010001ff0101ff,
+    0x00010001ff010100,
+    0x0001000100ffffff,
+    0x0001000100ff0000,
+    0x0001000100ff01ff,
+    0x0001000100ff0101,
+    0x000100010000ff00,
+    0x00010001000000ff,
+    0x0001000100000000,
+    0x0001000100000001,
+    0x00010001000001ff,
+    0x0001000100000101,
+    0x000100010001ffff,
+    0x0001000100010000,
+    0x00010001000101ff,
+    0x0001000101ffffff,
+    0x0001000101ffff01,
+    0x0001000101ff0000,
+    0x0001000101ff0101,
+    0x00010001010000ff,
+    0x0001000101000001,
+    0x00010001010001ff,
+    0x0001000101000100,
+    0x000100010101ffff,
+    0x00010001010100ff,
+    0x0001000101010001,
+    0x0001000101010101,
+    0x000101ffff000001,
+    0x000101ffff000100,
+    0x000101ffff010000,
+    0x000101ff00ffff00,
+    0x000101ff0000ff01,
+    0x000101ff00000000,
+    0x000101ff00000101,
+    0x000101ff0001ff00,
+    0x000101ff00010100,
+    0x000101ff01ff0000,
+    0x000101ff0100ff00,
+    0x000101ff010001ff,
+    0x000101ff01010001,
+    0x00010100ffffff00,
+    0x00010100ffff00ff,
+    0x00010100ff00ffff,
+    0x00010100ff000000,
+    0x00010100ff01ff00,
+    0x00010100ff0100ff,
+    0x00010100ff010001,
+    0x00010100ff010100,
+    0x0001010000ffffff,
+    0x0001010000ffff00,
+    0x0001010000ff0000,
+    0x0001010000ff0001,
+    0x0001010000ff01ff,
+    0x000101000000ff00,
+    0x00010100000000ff,
+    0x0001010000000000,
+    0x0001010000000001,
+    0x0001010000000100,
+    0x000101000001ffff,
+    0x0001010000010000,
+    0x0001010000010101,
+    0x0001010001ffff01,
+    0x0001010001ff00ff,
+    0x0001010001ff0101,
+    0x0001010001000000,
+    0x000101000101ff00,
+    0x00010100010100ff,
+    0x0001010001010000,
+    0x0001010001010100,
+    0x00010101ff00ff00,
+    0x00010101ff000001,
+    0x00010101ff0001ff,
+    0x0001010100ffff00,
+    0x0001010100ff00ff,
+    0x0001010100ff0100,
+    0x000101010000ffff,
+    0x0001010100000000,
+    0x00010101000001ff,
+    0x0001010100000101,
+    0x00010101000100ff,
+    0x0001010100010000,
+    0x0001010100010100,
+    0x0001010101ff0001,
+    0x00010101010000ff,
+    0x00010101010001ff,
+    0x0001010101000101,
+    0x0001010101010001,
+    0x01ffffffffffffff,
+    0x01ffffffffffff01,
+    0x01ffffffffff01ff,
+    0x01ffffffffff0101,
+    0x01ffffffff01ffff,
+    0x01ffffffff01ff01,
+    0x01ffffffff0101ff,
+    0x01ffffffff010101,
+    0x01ffffff00ff0000,
+    0x01ffffff0000ffff,
+    0x01ffffff0000ff00,
+    0x01ffffff000000ff,
+    0x01ffffff00000001,
+    0x01ffffff00000100,
+    0x01ffffff00010000,
+    0x01ffffff01ffffff,
+    0x01ffffff01ffff01,
+    0x01ffffff01ff01ff,
+    0x01ffffff01ff0101,
+    0x01ffffff01000000,
+    0x01ffffff0101ffff,
+    0x01ffffff0101ff01,
+    0x01ffffff010101ff,
+    0x01ffffff01010101,
+    0x01ffff00ffff0000,
+    0x01ffff00ff00ff00,
+    0x01ffff00ff0000ff,
+    0x01ffff00ff000001,
+    0x01ffff00ff000100,
+    0x01ffff00ff010000,
+    0x01ffff0000ffff00,
+    0x01ffff0000ff00ff,
+    0x01ffff0000ff0100,
+    0x01ffff000000ffff,
+    0x01ffff000000ff01,
+    0x01ffff0000000000,
+    0x01ffff0000000001,
+    0x01ffff00000001ff,
+    0x01ffff0000000100,
+    0x01ffff00000100ff,
+    0x01ffff0000010001,
+    0x01ffff0000010100,
+    0x01ffff0001ff0000,
+    0x01ffff0001ff0100,
+    0x01ffff00010000ff,
+    0x01ffff0001000001,
+    0x01ffff0001000100,
+    0x01ffff0001010000,
+    0x01ffff01ffffffff,
+    0x01ffff01ffffff01,
+    0x01ffff01ffff01ff,
+    0x01ffff01ffff0101,
+    0x01ffff01ff000000,
+    0x01ffff01ff01ffff,
+    0x01ffff01ff01ff01,
+    0x01ffff01ff0101ff,
+    0x01ffff01ff010101,
+    0x01ffff010000ff00,
+    0x01ffff01000000ff,
+    0x01ffff0100000100,
+    0x01ffff0100010000,
+    0x01ffff0101ffffff,
+    0x01ffff0101ffff01,
+    0x01ffff0101ff01ff,
+    0x01ffff0101ff0101,
+    0x01ffff0101000000,
+    0x01ffff010101ffff,
+    0x01ffff010101ff01,
+    0x01ffff01010101ff,
+    0x01ffff0101010101,
+    0x01ff00ffff0000ff,
+    0x01ff00ffff000100,
+    0x01ff00ff00ffff00,
+    0x01ff00ff00ff00ff,
+    0x01ff00ff0000ff00,
+    0x01ff00ff00000000,
+    0x01ff00ff00000101,
+    0x01ff00ff0001ff00,
+    0x01ff00ff000100ff,
+    0x01ff00ff00010100,
+    0x01ff00ff010000ff,
+    0x01ff00ff01000100,
+    0x01ff0000ffffff00,
+    0x01ff0000ffff0100,
+    0x01ff0000ff00ff01,
+    0x01ff0000ff000000,
+    0x01ff0000ff000101,
+    0x01ff0000ff010001,
+    0x01ff0000ff010100,
+    0x01ff000000ffffff,
+    0x01ff000000ffff00,
+    0x01ff000000ff0000,
+    0x01ff000000ff01ff,
+    0x01ff00000000ff00,
+    0x01ff0000000000ff,
+    0x01ff000000000000,
+    0x01ff000000000001,
+    0x01ff000000000100,
+    0x01ff000000000101,
+    0x01ff000000010000,
+    0x01ff000000010001,
+    0x01ff0000000101ff,
+    0x01ff000000010101,
+    0x01ff000001ffff00,
+    0x01ff000001ff00ff,
+    0x01ff000001ff0001,
+    0x01ff000001ff0100,
+    0x01ff00000100ffff,
+    0x01ff00000100ff01,
+    0x01ff000001000000,
+    0x01ff0000010001ff,
+    0x01ff000001010001,
+    0x01ff0001ff00ff00,
+    0x01ff0001ff000001,
+    0x01ff0001ff000100,
+    0x01ff0001ff010000,
+    0x01ff000100ffff00,
+    0x01ff000100ff00ff,
+    0x01ff000100ff0100,
+    0x01ff000100ff0101,
+    0x01ff00010000ffff,
+    0x01ff000100000000,
+    0x01ff000100000100,
+    0x01ff000100000101,
+    0x01ff00010001ff00,
+    0x01ff000100010001,
+    0x01ff000100010101,
+    0x01ff000101ff0000,
+    0x01ff00010100ff00,
+    0x01ff000101000101,
+    0x01ff0001010100ff,
+    0x01ff01ffffffffff,
+    0x01ff01ffffffff01,
+    0x01ff01ffffff01ff,
+    0x01ff01ffffff0101,
+    0x01ff01ffff000000,
+    0x01ff01ffff01ffff,
+    0x01ff01ffff01ff01,
+    0x01ff01ffff0101ff,
+    0x01ff01ffff010101,
+    0x01ff01ff00ffff00,
+    0x01ff01ff00ff0000,
+    0x01ff01ff0000ff00,
+    0x01ff01ff000000ff,
+    0x01ff01ff00000100,
+    0x01ff01ff00010000,
+    0x01ff01ff00010100,
+    0x01ff01ff01ffffff,
+    0x01ff01ff01ffff01,
+    0x01ff01ff01ff01ff,
+    0x01ff01ff01ff0101,
+    0x01ff01ff01000000,
+    0x01ff01ff0101ffff,
+    0x01ff01ff0101ff01,
+    0x01ff01ff010101ff,
+    0x01ff01ff01010101,
+    0x01ff0100ffff0000,
+    0x01ff0100ffff0001,
+    0x01ff0100ff00ff00,
+    0x01ff0100ff0000ff,
+    0x01ff0100ff000001,
+    0x01ff0100ff010000,
+    0x01ff010000ffff00,
+    0x01ff010000ff00ff,
+    0x01ff010000ff0001,
+    0x01ff010000ff0100,
+    0x01ff01000000ffff,
+    0x01ff01000000ff01,
+    0x01ff010000000000,
+    0x01ff010000000101,
+    0x01ff01000001ff00,
+    0x01ff0100000100ff,
+    0x01ff010001ff0000,
+    0x01ff010001000001,
+    0x01ff010001000100,
+    0x01ff010001010000,
+    0x01ff0101ffffffff,
+    0x01ff0101ffffff01,
+    0x01ff0101ffff01ff,
+    0x01ff0101ffff0101,
+    0x01ff0101ff000000,
+    0x01ff0101ff01ffff,
+    0x01ff0101ff01ff01,
+    0x01ff0101ff0101ff,
+    0x01ff0101ff010101,
+    0x01ff010100ff0000,
+    0x01ff01010000ff00,
+    0x01ff0101000000ff,
+    0x01ff010100000001,
+    0x01ff010101ffffff,
+    0x01ff010101ffff01,
+    0x01ff010101ff01ff,
+    0x01ff010101ff0101,
+    0x01ff010101000000,
+    0x01ff01010101ffff,
+    0x01ff01010101ff01,
+    0x01ff0101010101ff,
+    0x01ff010101010101,
+    0x0100ffffffff0000,
+    0x0100ffffff00ff00,
+    0x0100ffffff000001,
+    0x0100ffffff0001ff,
+    0x0100ffffff000100,
+    0x0100ffffff010000,
+    0x0100ffff00ffff00,
+    0x0100ffff00ff0001,
+    0x0100ffff00ff0100,
+    0x0100ffff00000000,
+    0x0100ffff000001ff,
+    0x0100ffff00000101,
+    0x0100ffff00010100,
+    0x0100ffff00010101,
+    0x0100ffff01ff0000,
+    0x0100ffff0100ff00,
+    0x0100ffff010000ff,
+    0x0100ffff01000001,
+    0x0100ffff01000100,
+    0x0100ffff01010000,
+    0x0100ff00ffffff00,
+    0x0100ff00ffff00ff,
+    0x0100ff00ffff0001,
+    0x0100ff00ffff0100,
+    0x0100ff00ff00ffff,
+    0x0100ff00ff000000,
+    0x0100ff00ff0001ff,
+    0x0100ff00ff000101,
+    0x0100ff00ff01ff00,
+    0x0100ff00ff0100ff,
+    0x0100ff00ff010001,
+    0x0100ff00ff010100,
+    0x0100ff0000ffffff,
+    0x0100ff0000ff0000,
+    0x0100ff000000ffff,
+    0x0100ff000000ff00,
+    0x0100ff00000000ff,
+    0x0100ff0000000000,
+    0x0100ff0000000001,
+    0x0100ff0000000100,
+    0x0100ff000001ff01,
+    0x0100ff0000010000,
+    0x0100ff0001ff00ff,
+    0x0100ff0001ff0001,
+    0x0100ff000100ff01,
+    0x0100ff0001000000,
+    0x0100ff00010001ff,
+    0x0100ff000101ff00,
+    0x0100ff00010100ff,
+    0x0100ff0001010001,
+    0x0100ff0001010100,
+    0x0100ff01ffff0000,
+    0x0100ff01ff00ff00,
+    0x0100ff01ff0000ff,
+    0x0100ff01ff000100,
+    0x0100ff01ff010000,
+    0x0100ff0100ff00ff,
+    0x0100ff0100ff0001,
+    0x0100ff0100ff0100,
+    0x0100ff010000ffff,
+    0x0100ff010000ff01,
+    0x0100ff0100000000,
+    0x0100ff01000001ff,
+    0x0100ff0100010001,
+    0x0100ff0100010100,
+    0x0100ff0101ff0000,
+    0x0100ff01010000ff,
+    0x0100ff0101000001,
+    0x0100ff0101010100,
+    0x010000ffffffff00,
+    0x010000ffffff00ff,
+    0x010000ffffff0001,
+    0x010000ffff00ffff,
+    0x010000ffff000000,
+    0x010000ffff0001ff,
+    0x010000ffff010001,
+    0x010000ff00ffffff,
+    0x010000ff00ff0101,
+    0x010000ff0000ff00,
+    0x010000ff000000ff,
+    0x010000ff00000000,
+    0x010000ff00000001,
+    0x010000ff000001ff,
+    0x010000ff00000100,
+    0x010000ff0001ffff,
+    0x010000ff0001ff00,
+    0x010000ff0001ff01,
+    0x010000ff00010000,
+    0x010000ff01ff00ff,
+    0x010000ff01ff0001,
+    0x010000ff0100ff01,
+    0x010000ff010000ff,
+    0x010000ff01000000,
+    0x010000ff010001ff,
+    0x010000ff0101ff00,
+    0x010000ff01010100,
+    0x01000000ffffffff,
+    0x01000000ffff0000,
+    0x01000000ffff01ff,
+    0x01000000ffff0101,
+    0x01000000ff00ffff,
+    0x01000000ff00ff00,
+    0x01000000ff0000ff,
+    0x01000000ff000000,
+    0x01000000ff000001,
+    0x01000000ff000100,
+    0x01000000ff01ff00,
+    0x01000000ff010000,
+    0x01000000ff010100,
+    0x01000000ff010101,
+    0x0100000000ffff00,
+    0x0100000000ff00ff,
+    0x0100000000ff0000,
+    0x0100000000ff0001,
+    0x0100000000ff0100,
+    0x010000000000ffff,
+    0x010000000000ff00,
+    0x010000000000ff01,
+    0x01000000000000ff,
+    0x0100000000000000,
+    0x0100000000000001,
+    0x01000000000001ff,
+    0x0100000000000100,
+    0x0100000000000101,
+    0x010000000001ff00,
+    0x01000000000100ff,
+    0x0100000000010000,
+    0x0100000000010001,
+    0x0100000000010100,
+    0x0100000001ffff00,
+    0x0100000001ff0000,
+    0x0100000001ff01ff,
+    0x010000000100ff00,
+    0x010000000100ff01,
+    0x01000000010000ff,
+    0x0100000001000000,
+    0x0100000001000001,
+    0x0100000001000100,
+    0x0100000001000101,
+    0x010000000101ffff,
+    0x010000000101ff01,
+    0x0100000001010000,
+    0x01000000010101ff,
+    0x0100000001010101,
+    0x01000001ffffff00,
+    0x01000001ffff00ff,
+    0x01000001ff00ffff,
+    0x01000001ff000000,
+    0x01000001ff000100,
+    0x01000001ff01ffff,
+    0x01000001ff010001,
+    0x01000001ff010100,
+    0x0100000100ff0000,
+    0x0100000100ff01ff,
+    0x0100000100ff0100,
+    0x010000010000ff00,
+    0x010000010000ff01,
+    0x0100000100000000,
+    0x0100000100000001,
+    0x0100000100000100,
+    0x0100000100010000,
+    0x01000001000101ff,
+    0x0100000101ffff01,
+    0x0100000101ff00ff,
+    0x0100000101ff0100,
+    0x0100000101ff0101,
+    0x010000010100ff01,
+    0x01000001010000ff,
+    0x0100000101000000,
+    0x01000001010100ff,
+    0x0100000101010001,
+    0x0100000101010100,
+    0x010001ffffff0000,
+    0x010001ffff000001,
+    0x010001ffff000100,
+    0x010001ffff010000,
+    0x010001ff00ffff00,
+    0x010001ff00ff0001,
+    0x010001ff0000ffff,
+    0x010001ff0000ff01,
+    0x010001ff00000000,
+    0x010001ff00000001,
+    0x010001ff00000101,
+    0x010001ff000100ff,
+    0x010001ff00010000,
+    0x010001ff01ff0000,
+    0x010001ff0100ff00,
+    0x010001ff01000001,
+    0x010001ff01000100,
+    0x010001ff01010000,
+    0x01000100ffff00ff,
+    0x01000100ffff0001,
+    0x01000100ffff0100,
+    0x01000100ff00ffff,
+    0x01000100ff00ff01,
+    0x01000100ff000000,
+    0x01000100ff0001ff,
+    0x01000100ff000101,
+    0x01000100ff01ffff,
+    0x01000100ff01ff00,
+    0x01000100ff0100ff,
+    0x01000100ff010001,
+    0x0100010000ffffff,
+    0x0100010000ffff01,
+    0x0100010000ff0000,
+    0x0100010000ff01ff,
+    0x0100010000ff0101,
+    0x010001000000ff00,
+    0x01000100000000ff,
+    0x0100010000000000,
+    0x0100010000000001,
+    0x0100010000000100,
+    0x010001000001ff01,
+    0x0100010000010000,
+    0x0100010000010001,
+    0x0100010000010101,
+    0x0100010001ffff00,
+    0x0100010001ff00ff,
+    0x010001000100ffff,
+    0x010001000100ff01,
+    0x0100010001000000,
+    0x0100010001000101,
+    0x010001000101ff00,
+    0x0100010001010001,
+    0x01000101ffff0000,
+    0x01000101ff000000,
+    0x01000101ff010000,
+    0x0100010100ff00ff,
+    0x0100010100ff0001,
+    0x0100010100ff0100,
+    0x010001010000ffff,
+    0x0100010100000000,
+    0x01000101000001ff,
+    0x010001010001ff00,
+    0x0100010101ff0000,
+    0x010001010100ff00,
+    0x01000101010000ff,
+    0x0100010101000000,
+    0x0100010101000001,
+    0x0101ffffffffffff,
+    0x0101ffffffffff01,
+    0x0101ffffffff01ff,
+    0x0101ffffffff0101,
+    0x0101ffffff000000,
+    0x0101ffffff01ffff,
+    0x0101ffffff01ff01,
+    0x0101ffffff0101ff,
+    0x0101ffffff010101,
+    0x0101ffff00ff0000,
+    0x0101ffff0000ff00,
+    0x0101ffff000000ff,
+    0x0101ffff00000001,
+    0x0101ffff00000100,
+    0x0101ffff01ffffff,
+    0x0101ffff01ffff01,
+    0x0101ffff01ff01ff,
+    0x0101ffff01ff0101,
+    0x0101ffff01000000,
+    0x0101ffff0101ffff,
+    0x0101ffff0101ff01,
+    0x0101ffff010101ff,
+    0x0101ffff01010101,
+    0x0101ff00ffff0000,
+    0x0101ff00ffff0100,
+    0x0101ff00ff00ff00,
+    0x0101ff00ff0000ff,
+    0x0101ff00ff000001,
+    0x0101ff00ff000100,
+    0x0101ff00ff000101,
+    0x0101ff0000ff0001,
+    0x0101ff0000ff0100,
+    0x0101ff000000ff00,
+    0x0101ff0000000000,
+    0x0101ff00000001ff,
+    0x0101ff0000000101,
+    0x0101ff000001ff00,
+    0x0101ff00000100ff,
+    0x0101ff0001ff0000,
+    0x0101ff000100ffff,
+    0x0101ff000100ff01,
+    0x0101ff0001000001,
+    0x0101ff0001000100,
+    0x0101ff01ffffff01,
+    0x0101ff01ffff01ff,
+    0x0101ff01ffff0101,
+    0x0101ff01ff00ffff,
+    0x0101ff01ff000100,
+    0x0101ff01ff01ff01,
+    0x0101ff01ff0101ff,
+    0x0101ff01ff010101,
+    0x0101ff0100ff0000,
+    0x0101ff010000ff00,
+    0x0101ff0100000001,
+    0x0101ff0100000100,
+    0x0101ff0100010000,
+    0x0101ff0101ffffff,
+    0x0101ff0101ffff01,
+    0x0101ff0101ff01ff,
+    0x0101ff0101ff0101,
+    0x0101ff0101000000,
+    0x0101ff010101ffff,
+    0x0101ff010101ff01,
+    0x0101ff01010101ff,
+    0x0101ff0101010101,
+    0x010100ffff000100,
+    0x010100ffff010000,
+    0x010100ff00ffff00,
+    0x010100ff00ff00ff,
+    0x010100ff0000ffff,
+    0x010100ff000000ff,
+    0x010100ff00000000,
+    0x010100ff000001ff,
+    0x010100ff00000101,
+    0x010100ff0001ff00,
+    0x010100ff00010000,
+    0x010100ff00010001,
+    0x010100ff000101ff,
+    0x010100ff00010100,
+    0x010100ff01ff0000,
+    0x01010000ffff0001,
+    0x01010000ffff0100,
+    0x01010000ff00ffff,
+    0x01010000ff00ff01,
+    0x01010000ff000000,
+    0x01010000ff0001ff,
+    0x01010000ff010001,
+    0x01010000ff010100,
+    0x0101000000ffff01,
+    0x0101000000ff0000,
+    0x010100000000ff00,
+    0x01010000000000ff,
+    0x0101000000000000,
+    0x0101000000000001,
+    0x0101000000000100,
+    0x0101000000010000,
+    0x0101000000010101,
+    0x0101000001ffff00,
+    0x0101000001ff00ff,
+    0x0101000001ff0000,
+    0x0101000001ff0001,
+    0x0101000001ff0100,
+    0x010100000100ff01,
+    0x0101000001000000,
+    0x01010000010001ff,
+    0x01010001ffff0000,
+    0x01010001ff00ff00,
+    0x01010001ff000001,
+    0x01010001ff000101,
+    0x01010001ff01ff00,
+    0x01010001ff010000,
+    0x0101000100ff00ff,
+    0x0101000100ff0001,
+    0x0101000100ff0101,
+    0x010100010000ff01,
+    0x0101000100000000,
+    0x0101000100000001,
+    0x01010001000001ff,
+    0x010100010001ffff,
+    0x010100010001ff01,
+    0x0101000101ff0001,
+    0x010100010100ffff,
+    0x0101000101000000,
+    0x0101000101000001,
+    0x0101000101000100,
+    0x010100010101ff00,
+    0x01010001010100ff,
+    0x0101000101010001,
+    0x010101ffffffffff,
+    0x010101ffffffff01,
+    0x010101ffffff01ff,
+    0x010101ffffff0101,
+    0x010101ffff01ffff,
+    0x010101ffff01ff01,
+    0x010101ffff0101ff,
+    0x010101ffff010101,
+    0x010101ff0000ff00,
+    0x010101ff000000ff,
+    0x010101ff00000001,
+    0x010101ff00000100,
+    0x010101ff01ffffff,
+    0x010101ff01ffff01,
+    0x010101ff01ff01ff,
+    0x010101ff01ff0101,
+    0x010101ff01000000,
+    0x010101ff0101ffff,
+    0x010101ff0101ff01,
+    0x010101ff010101ff,
+    0x010101ff01010101,
+    0x01010100ffff0000,
+    0x01010100ff0000ff,
+    0x01010100ff000100,
+    0x01010100ff01ff00,
+    0x01010100ff010000,
+    0x0101010000ffff00,
+    0x010101000000ffff,
+    0x0101010000000000,
+    0x0101010000000101,
+    0x010101000001ff00,
+    0x0101010000010001,
+    0x0101010000010100,
+    0x010101000100ffff,
+    0x0101010001000001,
+    0x01010101ffffffff,
+    0x01010101ffffff01,
+    0x01010101ffff01ff,
+    0x01010101ffff0101,
+    0x01010101ff01ffff,
+    0x01010101ff01ff01,
+    0x01010101ff0101ff,
+    0x01010101ff010101,
+    0x010101010000ff00,
+    0x01010101000000ff,
+    0x0101010100000001,
+    0x0101010101ffffff,
+    0x0101010101ffff01,
+    0x0101010101ff01ff,
+    0x0101010101ff0101,
+    0x0101010101000000,
+    0x010101010101ffff,
+    0x010101010101ff01,
+    0x01010101010101ff,
+    0x0101010101010101,
+};
+}  // namespace iq_grids
diff --git a/apps/ggml/halide/k_quant_generators.cpp b/apps/ggml/halide/k_quant_generators.cpp
new file mode 100644
index 000000000000..72a089919960
--- /dev/null
+++ b/apps/ggml/halide/k_quant_generators.cpp
@@ -0,0 +1,80 @@
+// Generic, GeneratorParam-driven quantize/dequantize pair for GGML's
+// K-quant super-block formats (see quant_components.h's make_k_quant_scheme/
+// CombineBits/PlanarBitPack/K4ScaleMinPack/Q3KScalePack for the reusable
+// Approximation pieces this assembles). "Q2_K"/"Q3_K"/"Q4_K"/"Q5_K"/"Q6_K" are not
+// distinct C++ classes here -- they're just different GENERATOR_ARGS
+// instantiations of the same generator template, registered in
+// CMakeLists.txt as q4_k_quantize/q4_k_dequantize etc. -- following the
+// exact same shape as lookup_table_quant_generators.cpp's
+// LookupTableCodecGenerator<Direction> (see that file's header comment for
+// the full rationale: one shared configure() builds the whole pipeline once
+// from a real ImageParam, each direction just adopts whichever half applies
+// via add_input(const ImageParam&)/add_output(const Func&); generate() is
+// an empty stub).
+//
+// generate() never calls Approximation::encode()/decode() directly -- only
+// through Func::approximate_by() and Pipeline::compute_offline(). This
+// configure()/generate() body is identical across every *_quant_generators.cpp
+// file in this directory, so it lives in codec_generator_base.h's
+// CodecGeneratorBase<Derived, dir> instead of being repeated here -- this
+// class only needs to supply its own GeneratorParams and a build_scheme().
+
+#include "Halide.h"
+
+#include "codec_generator_base.h"
+#include "quant_components.h"
+
+using namespace Halide;
+using namespace ggml_halide;
+
+namespace {
+
+// Which of quant_components.h's make_*_scheme() factories to use, and the
+// on-disk block size that goes with it -- not derivable from a plain
+// GeneratorParam combination, since each K-quant format's field layout,
+// scale scheme, and code bit-width all differ.
+enum class Family { Q2_K,
+                    Q3_K,
+                    Q4_K,
+                    Q5_K,
+                    Q6_K };
+
+template<Direction dir>
+class KQuantCodecGenerator : public CodecGeneratorBase<KQuantCodecGenerator<dir>, dir> {
+public:
+    GeneratorParam<Family> family{
+        "family",
+        Family::Q4_K,
+        {{"q2_k", Family::Q2_K},
+         {"q3_k", Family::Q3_K},
+         {"q4_k", Family::Q4_K},
+         {"q5_k", Family::Q5_K},
+         {"q6_k", Family::Q6_K}}};
+
+    SchemeAndBytes build_scheme() const {
+        // switch's controlling expression can't resolve GeneratorParam<T>'s
+        // implicit conversion operators unambiguously -- .value() sidesteps
+        // that by returning the plain Family directly. Each make_*_scheme()
+        // now returns its own block_bytes alongside the scheme (computed from
+        // the same field list it builds internally), so there's no byte
+        // arithmetic to duplicate here.
+        switch (family.value()) {
+        case Family::Q2_K:
+            return make_q2_k_scheme();  // {scales[16]; qs[64]; fp16 d; fp16 dmin;}
+        case Family::Q3_K:
+            return make_q3_k_scheme();  // {hmask[32]; qs[64]; scales[12]; fp16 d;}
+        case Family::Q4_K:
+            return make_q4_k_scheme();  // {fp16 d; fp16 dmin; scales[12]; qs[128];}
+        case Family::Q5_K:
+            return make_q5_k_scheme();  // {fp16 d; fp16 dmin; scales[12]; qh[32]; qs[128];}
+        case Family::Q6_K:
+            return make_q6_k_scheme();  // {ql[128]; qh[64]; scales[16]; fp16 d;}
+        }
+        _halide_internal_error << "unreachable Family\n";
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(KQuantCodecGenerator<Direction::Quantize>, k_quant_quantize)
+HALIDE_REGISTER_GENERATOR(KQuantCodecGenerator<Direction::Dequantize>, k_quant_dequantize)
diff --git a/apps/ggml/halide/k_quant_vec_dot_generator.cpp b/apps/ggml/halide/k_quant_vec_dot_generator.cpp
new file mode 100644
index 000000000000..852f94034c47
--- /dev/null
+++ b/apps/ggml/halide/k_quant_vec_dot_generator.cpp
@@ -0,0 +1,63 @@
+// Generic, family-driven vec_dot for the K-quant super-block formats, the
+// vec_dot counterpart of k_quant_generators.cpp's KQuantCodecGenerator (same
+// family set). "q4_k_vec_dot" is a PARAMS family=q4_k instantiation of this
+// one generator. Weight is a block-indexed K-quant codec, activation is the
+// block-indexed Q8_K codec; VecDotGeneratorBase splices both via
+// approximate_by/compute_offline. K-quant decode is a two-level (sub-block)
+// scale, so the per-block scale is not single-invariant -> Float schedule.
+
+#include "Halide.h"
+
+#include "quant_components.h"
+#include "vec_dot_generator_base.h"
+
+using namespace Halide;
+using namespace ggml_halide;
+
+namespace {
+
+enum class Family { Q2_K,
+                    Q3_K,
+                    Q4_K,
+                    Q5_K,
+                    Q6_K };
+
+class KQuantVecDotGenerator : public VecDotGeneratorBase<KQuantVecDotGenerator> {
+public:
+    GeneratorParam<Family> family{
+        "family",
+        Family::Q4_K,
+        {{"q2_k", Family::Q2_K},
+         {"q3_k", Family::Q3_K},
+         {"q4_k", Family::Q4_K},
+         {"q5_k", Family::Q5_K},
+         {"q6_k", Family::Q6_K}}};
+
+    // Q8_K activation codec (block_q8_K = {float d; qs[256]; bsums[16]} = 292 bytes).
+    static std::unique_ptr<Halide::Approximation> q8_k_codec() {
+        return make_q8_k_scheme(256, 127, Layout::BlockIndexed).scheme;
+    }
+
+    VecDotSpec build_vec_dot() const {
+        // All K-quants: 256-element super-block, Q8_K activation, two-level
+        // scale -> Float schedule.
+        switch (family.value()) {
+        case Family::Q2_K:
+            return {make_q2_k_scheme(Layout::BlockIndexed).scheme, 84, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::Q3_K:
+            return {make_q3_k_scheme(Layout::BlockIndexed).scheme, 110, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::Q4_K:
+            return {make_q4_k_scheme(Layout::BlockIndexed).scheme, 144, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::Q5_K:
+            return {make_q5_k_scheme(Layout::BlockIndexed).scheme, 176, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::Q6_K:
+            return {make_q6_k_scheme(Layout::BlockIndexed).scheme, 210, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        }
+        _halide_internal_error << "KQuantVecDotGenerator: family not yet converted\n";
+        return {};
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(KQuantVecDotGenerator, k_quant_vec_dot)
diff --git a/apps/ggml/halide/lookup_table_quant_generators.cpp b/apps/ggml/halide/lookup_table_quant_generators.cpp
new file mode 100644
index 000000000000..3c3ca741e1de
--- /dev/null
+++ b/apps/ggml/halide/lookup_table_quant_generators.cpp
@@ -0,0 +1,116 @@
+// Generic, GeneratorParam-driven quantize/dequantize pair for GGML's
+// codebook-quantized (lookup-table) formats (see quant_components.h's
+// LookupTableQuantize/E8M0Pack for the reusable Approximation pieces this
+// assembles). "IQ4_NL"/"MXFP4" are not distinct C++ classes here -- they're
+// just different GENERATOR_ARGS instantiations of the same generator
+// template, registered in CMakeLists.txt as iq4_nl_quantize/mxfp4_quantize
+// etc. -- following the same shape as symmetric_quant_generators.cpp's
+// SymmetricCodecGenerator<Direction> (see that file's header comment for the
+// full rationale: one shared configure() builds the whole pipeline once from
+// a real ImageParam, each direction just adopts whichever half applies via
+// add_input(const ImageParam&)/add_output(const Func&); generate() is an
+// empty stub).
+//
+// generate() never calls Approximation::encode()/decode() directly -- only
+// through Func::approximate_by() and Pipeline::compute_offline(). This
+// configure()/generate() body is identical across every *_quant_generators.cpp
+// file in this directory, so it lives in codec_generator_base.h's
+// CodecGeneratorBase<Derived, dir> instead of being repeated here -- this
+// class only needs to supply its own GeneratorParams and a build_scheme().
+
+#include "Halide.h"
+
+#include "codec_generator_base.h"
+#include "quant_components.h"
+
+using namespace Halide;
+using namespace ggml_halide;
+
+namespace {
+
+// Which of quant_components.h's make_*_scheme() factories to use, and the
+// on-disk block size that goes with it -- not derivable from a plain
+// GeneratorParam combination the way the affine/symmetric family's
+// SchemeKind's block_bytes is, since each codebook has its own fixed layout.
+enum class Family { IQ4_NL,
+                    MXFP4,
+                    TQ2_0,
+                    TQ1_0,
+                    NVFP4,
+                    IQ2_S,
+                    IQ3_XXS,
+                    IQ3_S,
+                    IQ4_XS,
+                    IQ2_XS,
+                    IQ2_XXS,
+                    IQ1_S,
+                    IQ1_M };
+
+template<Direction dir>
+class LookupTableCodecGenerator : public CodecGeneratorBase<LookupTableCodecGenerator<dir>, dir> {
+public:
+    GeneratorParam<Family> family{
+        "family",
+        Family::IQ4_NL,
+        {{"iq4_nl", Family::IQ4_NL},
+         {"mxfp4", Family::MXFP4},
+         {"tq2_0", Family::TQ2_0},
+         {"tq1_0", Family::TQ1_0},
+         {"nvfp4", Family::NVFP4},
+         {"iq2_s", Family::IQ2_S},
+         {"iq3_xxs", Family::IQ3_XXS},
+         {"iq3_s", Family::IQ3_S},
+         {"iq4_xs", Family::IQ4_XS},
+         {"iq2_xs", Family::IQ2_XS},
+         {"iq2_xxs", Family::IQ2_XXS},
+         {"iq1_s", Family::IQ1_S},
+         {"iq1_m", Family::IQ1_M}}};
+
+    SchemeAndBytes build_scheme() const {
+        // switch's controlling expression can't resolve GeneratorParam<T>'s
+        // implicit conversion operators unambiguously -- .value() sidesteps
+        // that by returning the plain Family directly. Every make_*_scheme()
+        // now returns its own block_bytes alongside the scheme (from its
+        // field table, or -- for the IQ grid leaves, which are deliberately
+        // NOT field-table-decomposed; see quant_components.h section 6's
+        // design note -- the hand-verified constant declared next to the
+        // leaf), so there's no literal byte count to keep in sync here.
+        switch (family.value()) {
+        case Family::IQ4_NL:
+            return make_iq4_nl_scheme();
+        case Family::MXFP4:
+            return make_mxfp4_scheme();
+        case Family::TQ2_0:
+            return make_tq2_0_scheme();
+        case Family::TQ1_0:
+            return make_tq1_0_scheme();
+        case Family::NVFP4:
+            return make_nvfp4_scheme();
+        case Family::IQ2_S:
+            return make_iq2_s_scheme();  // {fp16 d; qs[32]; signs[32]; qh[8]; scales[8];}
+        case Family::IQ3_XXS:
+            return make_iq3_xxs_scheme();  // {fp16 d; qs[64]; scales_and_signs[32];}
+        case Family::IQ3_S:
+            return make_iq3_s_scheme();  // {fp16 d; qs[64]; qh[8]; signs[32]; scales[4];}
+        case Family::IQ4_XS:
+            return make_iq4_xs_scheme();  // {fp16 d; scales_h[2]; scales_l[4]; qs[128];}
+        // Importance-matrix-only (dequantize direction only -- no quantize
+        // library is built for these; SeveredEncode stands in for the missing
+        // forward map).
+        case Family::IQ2_XS:
+            return make_iq2_xs_scheme();
+        case Family::IQ2_XXS:
+            return make_iq2_xxs_scheme();
+        case Family::IQ1_S:
+            return make_iq1_s_scheme();
+        case Family::IQ1_M:
+            return make_iq1_m_scheme();
+        }
+        _halide_internal_error << "unreachable Family\n";
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(LookupTableCodecGenerator<Direction::Quantize>, lookup_table_quantize)
+HALIDE_REGISTER_GENERATOR(LookupTableCodecGenerator<Direction::Dequantize>, lookup_table_dequantize)
diff --git a/apps/ggml/halide/lookup_table_vec_dot_generator.cpp b/apps/ggml/halide/lookup_table_vec_dot_generator.cpp
new file mode 100644
index 000000000000..b6b86e8bca03
--- /dev/null
+++ b/apps/ggml/halide/lookup_table_vec_dot_generator.cpp
@@ -0,0 +1,111 @@
+// Generic, family-driven vec_dot for the codebook/grid formats, the vec_dot
+// counterpart of lookup_table_quant_generators.cpp's LookupTableCodecGenerator
+// (same family set). "iq4_nl_vec_dot" is not a distinct C++ class -- it's a
+// PARAMS family=iq4_nl instantiation of this one generator, registered in
+// CMakeLists.txt. Weight and activation are both block-indexed codecs from
+// quant_components.h; VecDotGeneratorBase splices them via approximate_by/
+// compute_offline (see vec_dot_generator_base.h).
+
+#include "Halide.h"
+
+#include "quant_components.h"
+#include "vec_dot_generator_base.h"
+
+using namespace Halide;
+using namespace ggml_halide;
+
+namespace {
+
+enum class Family { IQ4_NL,
+                    MXFP4,
+                    TQ2_0,
+                    TQ1_0,
+                    NVFP4,
+                    IQ2_S,
+                    IQ3_XXS,
+                    IQ3_S,
+                    IQ4_XS,
+                    IQ2_XS,
+                    IQ2_XXS,
+                    IQ1_S,
+                    IQ1_M };
+
+class LookupTableVecDotGenerator : public VecDotGeneratorBase<LookupTableVecDotGenerator> {
+public:
+    GeneratorParam<Family> family{
+        "family",
+        Family::IQ4_NL,
+        {{"iq4_nl", Family::IQ4_NL},
+         {"mxfp4", Family::MXFP4},
+         {"tq2_0", Family::TQ2_0},
+         {"tq1_0", Family::TQ1_0},
+         {"nvfp4", Family::NVFP4},
+         {"iq2_s", Family::IQ2_S},
+         {"iq3_xxs", Family::IQ3_XXS},
+         {"iq3_s", Family::IQ3_S},
+         {"iq4_xs", Family::IQ4_XS},
+         {"iq2_xs", Family::IQ2_XS},
+         {"iq2_xxs", Family::IQ2_XXS},
+         {"iq1_s", Family::IQ1_S},
+         {"iq1_m", Family::IQ1_M}}};
+
+    // Q8_0 activation codec (block_q8_0 = {fp16 d; qs[32]} = 34 bytes).
+    static std::unique_ptr<Halide::Approximation> q8_0_codec() {
+        return make_symmetric_block_scheme(32, 127, RoundingMode::Nearest, ScaleAnchor::AbsMax, 8, Layout::BlockIndexed).scheme;
+    }
+    // Q8_K activation codec (block_q8_K = {float d; qs[256]; bsums[16]} = 292 bytes).
+    static std::unique_ptr<Halide::Approximation> q8_k_codec() {
+        return make_q8_k_scheme(256, 127, Layout::BlockIndexed).scheme;
+    }
+
+    VecDotSpec build_vec_dot() const {
+        switch (family.value()) {
+        case Family::IQ4_NL:
+            // 4-bit codebook, single fp16 scale x Q8_0: single per-block scale
+            // and int8 codebook values -> SDOT-eligible (the base header's
+            // deep-inline SDOT exposes the scale even through the codebook LUT).
+            return {make_iq4_nl_scheme(Layout::BlockIndexed).scheme, 18, q8_0_codec(), 34, 32, ScheduleKind::SDOT};
+        case Family::MXFP4:
+            // Same single-scale codebook shape as IQ4_NL (E8M0 scale) x Q8_0.
+            return {make_mxfp4_scheme(Layout::BlockIndexed).scheme, 17, q8_0_codec(), 34, 32, ScheduleKind::SDOT};
+        case Family::NVFP4:
+            // 64-element block (4 sub-scales) x Q8_0 (32-block): the activation
+            // is Reblocked 32 -> 64 so both share the weight's block. Sub-block
+            // scales -> Float.
+            return {make_nvfp4_scheme(Layout::BlockIndexed).scheme, 36, reblock_activation(q8_0_codec(), 32, 64), 34, 64, ScheduleKind::Float};
+        // TQ1_0/TQ2_0 x Q8_K, IQ4_XS x Q8_K: single fp16 scale (TQ) or two-level
+        // scale (IQ4_XS) -> Float schedule for now.
+        case Family::TQ2_0:
+            return {make_tq2_0_scheme(Layout::BlockIndexed).scheme, 66, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::TQ1_0:
+            return {make_tq1_0_scheme(Layout::BlockIndexed).scheme, 54, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::IQ4_XS:
+            return {make_iq4_xs_scheme(Layout::BlockIndexed).scheme, 136, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        // Grid formats x Q8_K: per-group scale + sign -> Float schedule. The
+        // block-indexed codec collapses the leaf's {8,4,8} output to (kk, blk).
+        case Family::IQ2_S:
+            return {make_iq2_s_scheme(Layout::BlockIndexed).scheme, 82, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::IQ3_XXS:
+            return {make_iq3_xxs_scheme(Layout::BlockIndexed).scheme, 98, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::IQ3_S:
+            return {make_iq3_s_scheme(Layout::BlockIndexed).scheme, 110, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        // Importance-matrix-only formats (SeveredEncode weight scheme) x Q8_K.
+        case Family::IQ2_XS:
+            return {make_iq2_xs_scheme(Layout::BlockIndexed).scheme, 74, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::IQ2_XXS:
+            return {make_iq2_xxs_scheme(Layout::BlockIndexed).scheme, 66, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::IQ1_S:
+            return {make_iq1_s_scheme(Layout::BlockIndexed).scheme, 50, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        case Family::IQ1_M:
+            return {make_iq1_m_scheme(Layout::BlockIndexed).scheme, 56, q8_k_codec(), 292, 256, ScheduleKind::Float};
+        default:
+            break;
+        }
+        _halide_internal_error << "LookupTableVecDotGenerator: family not yet converted\n";
+        return {};
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(LookupTableVecDotGenerator, lookup_table_vec_dot)
diff --git a/apps/ggml/halide/quant_components.h b/apps/ggml/halide/quant_components.h
new file mode 100644
index 000000000000..71c27cbca5cd
--- /dev/null
+++ b/apps/ggml/halide/quant_components.h
@@ -0,0 +1,3195 @@
+#pragma once
+
+// Reusable Approximation components for GGML-style per-block quantized
+// weight formats -- see doc/ApproximationDesign.md and the plan this file
+// implements for the rationale. Every weight format is built by composing
+// these kinds of pieces via Halide::Compose/Halide::Apply (and, for the
+// extern-delegated formats, Halide::TrustedInverse) into a scheme (see the
+// make_*_scheme() factory functions below), which the
+// Generators then splice in via Func::approximate_by()/
+// Pipeline::compute_offline() -- never by calling Approximation::encode()/
+// decode() directly.
+//
+//   1. BlockReshape -- lossless relayout: flat values <-> (kk, blk).
+//   2. SymmetricAffineQuantize/AffineQuantize -- the actual lossy step:
+//      block values <-> (integer codes, one or two float(s) per block).
+//   3a. Fp16Pack/PlanarBitPack/BytePack -- per-field packing:
+//      a typed field (codes, scale, min) <-> its own on-disk byte encoding.
+//   3b. AppendSums -- a derived extra field, computed from other
+//      already-encoded fields rather than from the original values.
+//   3c. StructPack -- concatenates N already-packed fields into one
+//      byte-addressed buffer, matching a specific on-disk block layout
+//      (e.g. block_q4_0).
+//   4. Extern-delegated formats (codebook/K-quant/IQ grid/IQ4_XS): their
+//      quantize is an opaque GGML extern (ExternQuantize), paired with a
+//      compositional dequantize via Halide::TrustedInverse. The extra
+//      decode-only math leaves those need -- Codebook (codes -> table[codes]),
+//      LinearDequant (the scale multiply), CombineBits
+//      (K-quant split codes) -- live in section 4 alongside ExternQuantize.
+//
+// None of these know about any specific GGML type name -- "Q4_0"/"Q4_1"/etc.
+// are just particular parameter choices, assembled where the Generators live
+// (symmetric_quant_generators.cpp), not encoded here.
+//
+// None of these components call Func::bound() on their own intermediate
+// Funcs, even where a range is intrinsically known (e.g. codes(kk, blk) for
+// kk in [0, block_size)): with everything left at its default (inline)
+// schedule, as it is here, Halide already infers the true required range by
+// propagating backward from wherever the final packed buffer is actually
+// realized or scheduled (a Generator's Output dim bounds, or an explicit
+// realize() shape) -- an explicit bound() on an inlined Func is simply
+// ignored ("meaningless... because the function is scheduled inline", per
+// Halide's own warning). bound() only becomes useful once a component's Func
+// is deliberately scheduled non-inline (compute_root/compute_at), which is a
+// scheduling-time decision made where that happens, not decided here.
+//
+// DIMENSION / WILDCARD CONVENTION: a component's Func indices are laid out as
+// (field dims..., blk, lane dims...) -- the field's own within-block dims
+// first (kk, or byte, or (plane, sub), ...), then the block index blk, then
+// any trailing "lane" dims (e.g. a repack matmul weight's column-in-group j
+// and col-group x), carried by the Halide::_ placeholder. Decode-only stages
+// should be lane-general by default: write decode() with a trailing
+// Halide::_ on both sides (f(kk, blk, _) = g(..., blk, _)) so the same
+// component runs unchanged whether there are zero lane dims (the plain
+// (kk, blk) codecs) or several (the repack weight schemes). Encode stages
+// stay at fixed arity -- every encoder here runs on a plain (kk, blk) or
+// flat row, so there is nothing for a wildcard to carry. Currently
+// lane-general (decode side): Fp16Pack, F32Pack, E8M0Pack, BytePack,
+// PlanarBitPack (both normal and plane-axis modes), Codebook, LinearDequant,
+// K4ScaleMinPack, Q3KScalePack, IQ4XSScalePack. The remaining decoders
+// (BlockReshape/Reblock, the repack interleave/de-interleave leaves, the IQ
+// grid leaves, TritPack, BitPack, Int16Pack, UE4M3Pack) are fixed-arity by
+// design or have no lane-general consumer yet. The full dimension-general
+// BlockLayout sketched in the DESIGN NOTE by Reblock (splitting/permuting
+// arbitrary dims) remains deliberately deferred.
+
+// Only the aggregated Halide.h is installed for apps to consume (individual
+// per-class headers like Approximation.h are not) -- it already pulls in
+// Approximation/Compose/Apply/Pipeline::compute_offline.
+#include "Halide.h"
+
+#include <algorithm>
+
+#include "iq_grids_data.h"
+
+namespace ggml_halide {
+
+// The result of a make_*_scheme() factory: the scheme itself plus its
+// on-disk block byte count, computed once from the same field list
+// make_block_layout() (see FieldSpec below) already sums, rather than
+// hand-summed again at each Generator call site. Shared with
+// codec_generator_base.h's CodecGeneratorBase (which is what actually
+// consumes it -- see there).
+struct SchemeAndBytes {
+    std::unique_ptr<Halide::Approximation> scheme;
+    int block_bytes;
+    // When set (is_struct()), the scheme's encoded form is a first-class
+    // Type::Struct block (one struct per block index) rather than a 2-D
+    // (byte, blk) UInt(8) buffer. The Generator uses this to declare a
+    // struct-typed, 1-D packed ImageParam/Output, and block_bytes is then just
+    // block_type.bytes() -- the single source of truth for the on-disk width.
+    // Default-constructed (invalid) for the byte-buffer schemes not yet ported.
+    Halide::Type block_type;
+    // Set when the scheme appends a per-block scaled sum of its codes (Q8_1's
+    // `s` field -- AppendSums{SumMode::ScaledFloat}). A vec_dot pairing an
+    // affine weight against such an activation can sever the offset term's
+    // sum(act) accumulator straight to this stored field instead of recomputing
+    // it -- see VecDotGeneratorBase::configure(). Purely a byte-path,
+    // 32-element-block property today (Q8_1); wider layouts leave it false.
+    bool has_block_sums = false;
+    // Optional scheduling identities exported by composed schemes. q5_0 uses
+    // these to materialize reconstructed codes and its packed qh word without
+    // relying on generated Func names. Legacy q5_1 intentionally leaves them
+    // undefined until its separate migration.
+    Halide::ApproximationStageKey reconstructed_codes_stage;
+    Halide::ApproximationStageKey packed_high_word_stage;
+};
+
+// Every make_*_scheme() factory below takes a Layout, selecting what its
+// outermost BlockReshape (or grid BlockReshape) does with the "flat" side:
+//   - FlatRow (the default): a fully-flat 1-D row -- the shape
+//     quantize_row/dequantize_row Generators want.
+//   - BlockIndexed: a passthrough (kk, blk) -- the shape a vec_dot/repack
+//     Generator wants (its own reduction already runs over (kk, blk); no
+//     flat<->block reshape is needed on top). This used to be a *separate*
+//     make_*_codec() function per scheme (build the codec, skip the
+//     reshape); now it's the same factory with BlockReshape's own
+//     block_indexed flag set true, which makes it a lossless identity
+//     passthrough (see BlockReshape's own comment) -- so there's no
+//     behavioral difference, just one fewer named entry point per scheme.
+enum class Layout { FlatRow,
+                    BlockIndexed };
+
+// ---------------------------------------------------------------------------
+// 1. Lossless relayout.
+// ---------------------------------------------------------------------------
+
+// A lossless flat <-> block reshape. In the common one-dimensional case
+// (BlockReshape(block_size)), packed(kk, blk) = flat(blk*block_size + kk) --
+// one within-block index kk in [0, block_size), one block index blk.
+//
+// The general case (BlockReshape({e0, e1, ...})) unflattens the within-block
+// index into *several* dimensions, innermost/fastest-varying first, so a
+// component whose values have nested block structure can index those
+// dimensions directly instead of re-deriving them from a flat kk via div/mod.
+// E.g. an IQ 256-element superblock structured as group(8) x l(4) x elem(8)
+// uses extents {8, 4, 8}: packed(elem, l, group, blk), where the flat
+// within-block index kk = elem + 8*l + 32*group (product of extents = block
+// size). The single-int constructor is exactly the one-extent case.
+//
+// `block_indexed` selects what the "flat" side looks like:
+//   - false (default): a fully-flat 1-D row f(k), k = blk*block_size + within
+//     -- the shape quantize_row/dequantize_row want.
+//   - true: a block-indexed 2-D f(kk, blk), within-block index kept separate
+//     from the block index -- the shape a per-block vec_dot reduction wants
+//     (so the block index stays a distinct RVar for the SDOT rfactor hoist).
+// In block-indexed mode a single-extent reshape is a (kk,blk) passthrough,
+// and a multi-extent one collapses the nested dims (elem,l,group,blk) into
+// (kk,blk) -- the only difference from flat mode is folding blk into k or not.
+// Compatibility alias: the implementation is now a public core component.
+using BlockReshape = Halide::BlockReshape;
+
+// ---------------------------------------------------------------------------
+// Lossless block-layout relayouts (Reblock, and the repack Interleave below).
+//
+// DESIGN NOTE (intended library form, deferred): the clean, general shape for
+// these is a *dimension-general* block-relayout -- a component that splits and
+// permutes arbitrary index dimensions, carrying any trailing dims through
+// untouched (the way the Python research sketch's BlockLayout(splits=...)/
+// SplitStorage do, via Halide's `_` placeholder), quantizing/packing whatever
+// falls out. That would let a single `BlockLayout` utility live in the core
+// Approximation library and compose in front of *any* lossy quant, with the
+// repack interleave being just one instantiation. We deliberately do NOT do
+// that here: `Func`/`Var` `_` wildcard semantics were a source of trouble, and
+// pinning them down is its own rabbit hole. Instead these relayouts are
+// written at fixed, concrete arities (folding any extra "lane" -- e.g. repack's
+// 4 interleaved rows -- into the block index blk), reusing the existing
+// (kk, blk) quant/pack components unchanged. When the wildcard story is sorted,
+// these should graduate to the dimension-general form.
+// ---------------------------------------------------------------------------
+
+// Losslessly re-view a block-indexed Func at a different block size (the flat
+// element order is unchanged; only the (kk, blk) factorization differs).
+// decode(): (kk_from, blk_from) at `from_block` -> (kk_to, blk_to) at
+// `to_block`, reading the same global element g = blk_to*to_block + kk_to from
+// its source position (g % from_block, g / from_block). encode() is the
+// mirror. This is what lets a vec_dot present an activation stored in its own
+// (smaller) block size -- e.g. Q8_0's 32-element blocks -- at a weight's
+// (larger) block size (Q1_0's 128, NVFP4's 64), so both operands share one
+// (kk, blk) and the Generator's reduction stays uniform. The block-structure
+// reconciliation lives here, in an Approximation, not open-coded in the
+// reduction.
+class Reblock : public Halide::Approximation {
+public:
+    Reblock(int from_block, int to_block)
+        : from_(from_block), to_(to_block) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func in = inputs[0];  // (kk, blk) at to_block
+        Var kk("kk"), blk("blk");
+        Expr g = blk * from_ + kk;
+        Func out("reblock_encoded");
+        out(kk, blk) = in(g % to_, g / to_);
+        return {{out}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func in = encoded[0];  // (kk, blk) at from_block
+        Var kk("kk"), blk("blk");
+        Expr g = blk * to_ + kk;
+        Func out("reblock_decoded");
+        out(kk, blk) = in(g % from_, g / from_);
+        return {{out}, {}};
+    }
+
+private:
+    int from_, to_;
+};
+
+// An activation codec that decodes to (kk, blk) at `to_block`: `act_codec`
+// (block-indexed at the activation's own `from_block`) composed with a Reblock
+// when the two differ, else `act_codec` unchanged. Lets a vec_dot pair a weight
+// of one block size against an activation of another (Q1_0/NVFP4 x Q8_0).
+inline std::unique_ptr<Halide::Approximation> reblock_activation(
+    std::unique_ptr<Halide::Approximation> act_codec, int from_block, int to_block) {
+    if (from_block == to_block) {
+        return act_codec;
+    }
+    return std::make_unique<Halide::Compose>(std::move(act_codec), Reblock{from_block, to_block});
+}
+
+// ---------------------------------------------------------------------------
+// 2. The lossy step.
+// ---------------------------------------------------------------------------
+
+// GGML's own reference quantizers round differently depending on the target
+// bit width, not out of taste but because they use different formulas:
+//   - Nearest: plain round-half-away-from-zero (Q8_0's quantize_row_q8_0_ref
+//     uses roundf()). Halide's round() matches this exactly.
+//   - TruncateHalfUpWithOffset: a truncate-based "+qmax+0.5f then cast"
+//     trick used by nibble-packed formats (Q4_0's quantize_row_q4_0_ref).
+//     Verified by hand this is round-half-*up*, not round-half-away-from-
+//     zero: floor(x+8.5) at x=-0.5 gives 8 (rounds toward +inf), whereas
+//     round-half-away-from-zero would give 7.
+//   - SignOnly: code = sign(x0) in {-1, +1}, ignoring magnitude entirely --
+//     Q1_0's actual quantizer (1-bit codes, no rounding to speak of; paired
+//     with ScaleAnchor::MeanAbs below, not qmax-based like every other
+//     anchor).
+//   - NearestEvenClampedHigh: round-half-to-even (not round-half-away-from-
+//     zero like Nearest), then clamp only the high end to qmax -- Q8_K's
+//     actual quantizer. GGML computes this via a magic-number float trick
+//     (nearest_int(), reproduced by nearest_int() below) that exploits the
+//     default IEEE-754 round-to-nearest-even rounding of the addition
+//     itself; Halide has no round-to-even primitive exposed, so the same
+//     bit trick is used here to match bit-for-bit. Always paired with
+//     ScaleAnchor::ExtremeSignedValueTwoStep below.
+enum class RoundingMode { Nearest,
+                          TruncateHalfUpWithOffset,
+                          SignOnly,
+                          NearestEvenClampedHigh };
+
+// Same magic-number trick as GGML's static inline nearest_int() in
+// src/ggml-quants.c: adding 1.5*2^23 forces the CPU's default round-to-
+// nearest-even addition to round fval's fractional part, then the rounded
+// integer is recovered from the float's mantissa bits.
+inline Halide::Expr nearest_int(Halide::Expr fval) {
+    using namespace Halide;
+    Expr val = fval + 12582912.0f;  // 1.5 * 2^23
+    Expr bits = reinterpret<int32_t>(val);
+    return (bits & 0x007fffff) - 0x00400000;
+}
+
+// How a block's scale is derived from its values -- this is a second,
+// independent axis GGML varies per format, not just rounding:
+//   - AbsMax: scale = max(|v|) / qmax -- ordinary symmetric quantization
+//     (Q8_0).
+//   - ExtremeSignedValue: scale = -extreme / qmax, where `extreme` is the
+//     *signed* value with the largest magnitude in the block (ties keep the
+//     first-seen value, matching GGML's single left-to-right loop with a
+//     strict '<' comparison). This deliberately anchors the block's most
+//     extreme value at code -qmax, using the full negative side of an
+//     asymmetric signed range like [-8, 7] (Q4_0).
+//   - MeanAbs: scale = mean(|v|) over the block (a sum reduction divided by
+//     block_size, not a max reduction divided by qmax) -- Q1_0's anchor,
+//     always paired with RoundingMode::SignOnly.
+//   - ExtremeSignedValueTwoStep: mathematically the same value as
+//     ExtremeSignedValue (scale = -extreme/qmax), but computed as GGML's
+//     own two *separate* divisions -- `iscale = -qmax/extreme` first, then
+//     `scale = 1/iscale` -- rather than the algebraically-equivalent single
+//     multiply above. Floating point isn't associative, so these round
+//     differently in the last bit; `iscale` itself (not a fresh 1/scale
+//     recomputed afterward) is also what SymmetricAffineQuantize::encode()
+//     uses to derive codes for this anchor, to stay bit-exact with GGML's
+//     quantize_row_q8_K_ref -- see encode()'s `id`/`scale` computation.
+//     Always paired with RoundingMode::NearestEvenClampedHigh.
+enum class ScaleAnchor { AbsMax,
+                         ExtremeSignedValue,
+                         MeanAbs,
+                         ExtremeSignedValueTwoStep };
+
+// encode(): block(kk, blk) -> {codes(kk, blk) in [-qmax, qmax-1], scale(blk)}.
+// decode(): {codes, scale} -> cast<float>(codes) * scale -- this half is
+// exactly the same regardless of rounding/anchor (both Q4_0's and Q8_0's
+// existing hand-written dequantize math already reduce to this one formula).
+inline Halide::BlockRoundingMode core_rounding_mode(RoundingMode mode) {
+    switch (mode) {
+    case RoundingMode::Nearest:
+        return Halide::BlockRoundingMode::Nearest;
+    case RoundingMode::TruncateHalfUpWithOffset:
+        return Halide::BlockRoundingMode::TruncateHalfUpWithOffset;
+    case RoundingMode::SignOnly:
+        return Halide::BlockRoundingMode::SignOnly;
+    case RoundingMode::NearestEvenClampedHigh:
+        return Halide::BlockRoundingMode::NearestEvenClampedHigh;
+    }
+    _halide_internal_error << "Unknown symmetric rounding mode\n";
+}
+
+inline Halide::BlockScaleAnchor core_scale_anchor(ScaleAnchor anchor) {
+    switch (anchor) {
+    case ScaleAnchor::AbsMax:
+        return Halide::BlockScaleAnchor::AbsMax;
+    case ScaleAnchor::ExtremeSignedValue:
+        return Halide::BlockScaleAnchor::ExtremeSignedValue;
+    case ScaleAnchor::MeanAbs:
+        return Halide::BlockScaleAnchor::MeanAbs;
+    case ScaleAnchor::ExtremeSignedValueTwoStep:
+        return Halide::BlockScaleAnchor::ExtremeSignedValueTwoStep;
+    }
+    _halide_internal_error << "Unknown symmetric scale anchor\n";
+}
+
+class SymmetricAffineQuantize : public Halide::SymmetricBlockQuantize {
+public:
+    SymmetricAffineQuantize(int block_size, int qmax, RoundingMode rounding, ScaleAnchor anchor)
+        : Halide::SymmetricBlockQuantize(block_size, qmax,
+                                         core_rounding_mode(rounding), core_scale_anchor(anchor)) {
+    }
+};
+
+// How AffineQuantize rounds+truncates code = round((x-min)*id) into its
+// final representable range -- a different formula than SymmetricAffineQuantize's
+// RoundingMode, and not a variation on it: there's no centering/offset here
+// (codes are naturally unsigned starting at 0), and GGML's two affine legacy
+// formats don't even agree on whether to clamp at all:
+//   - ClampedInt8: (int8_t)(v+0.5f), then an explicit min(.., levels) --
+//     Q4_1's exact formula.
+//   - UnclampedUint8: (uint8_t)(v+0.5f) directly, no further clamp -- Q5_1's
+//     exact formula. GGML's own reference genuinely doesn't clamp this one
+//     (verified against quantize_row_q5_1_ref); reproduced faithfully since
+//     quantize output is checked bit-exact.
+enum class AffineRounding { ClampedInt8,
+                            UnclampedUint8 };
+
+// encode(): block(kk, blk) -> {codes(kk, blk) in [0, levels], scale(blk),
+// min(blk)}. decode(): {codes, scale, min} -> cast<float>(codes)*scale + min.
+// The min-max (not max-abs) scale derivation is what makes this "affine"
+// rather than "symmetric" -- every value in a block is representable, not
+// just those centered on zero, at the cost of needing a second per-block
+// float (Q4_1/Q5_1's 'm').
+class AffineQuantize : public Halide::Approximation {
+public:
+    AffineQuantize(int block_size, int levels, AffineRounding rounding)
+        : block_size_(block_size), levels_(levels), rounding_(rounding) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func block = inputs[0];  // block(kk, blk)
+        Var kk("kk"), blk("blk");
+        RDom r(0, block_size_, "r");
+
+        // Plain min/max reduction -- unlike SymmetricAffineQuantize's
+        // ScaleAnchor::ExtremeSignedValue, min and max are independent here,
+        // forming an affine (not centered-on-zero) range.
+        Func stat("affine_quantize_minmax");
+        stat(blk) = Tuple(std::numeric_limits<float>::max(), std::numeric_limits<float>::lowest());
+        Expr v = block(r, blk);
+        stat(blk) = Tuple(min(stat(blk)[0], v), max(stat(blk)[1], v));
+
+        Func scale("affine_quantize_scale");
+        scale(blk) = (stat(blk)[1] - stat(blk)[0]) / (float)levels_;
+        Func minv("affine_quantize_min");
+        minv(blk) = stat(blk)[0];
+
+        Expr id = select(scale(blk) != 0.0f, 1.0f / scale(blk), 0.0f);
+        Expr x0 = (block(kk, blk) - minv(blk)) * id;
+
+        Func codes("affine_quantize_codes");
+        if (rounding_ == AffineRounding::ClampedInt8) {
+            Expr raw = cast<int32_t>(cast<int8_t>(x0 + 0.5f));
+            codes(kk, blk) = cast<int8_t>(min(raw, levels_));
+        } else {
+            codes(kk, blk) = cast<int8_t>(cast<uint8_t>(x0 + 0.5f));
+        }
+
+        return {{codes, scale, minv}, {stat}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func codes = encoded[0], scale = encoded[1], minv = encoded[2];
+        Var kk("kk"), blk("blk");
+        Func dequantized("affine_dequantized_am");
+        dequantized(kk, blk) = cast<float>(codes(kk, blk)) * scale(blk) + minv(blk);
+        return {{dequantized}, {}};
+    }
+
+private:
+    int block_size_, levels_;
+    AffineRounding rounding_;
+};
+
+// ---------------------------------------------------------------------------
+// 3a. Per-field bit packing.
+// ---------------------------------------------------------------------------
+
+// Assemble a little-endian integer word starting at bytes(base, blk). The
+// uint32 specialization below is the on-disk byte order shared by F32Pack's
+// scale and IQ3_XXS's/IQ2_XXS's aux32.
+inline Halide::Expr le_uint(Halide::Func bytes, Halide::Expr base, Halide::Var blk, int byte_count) {
+    using namespace Halide;
+    Expr result = cast<uint32_t>(0);
+    for (int i = 0; i < byte_count; i++) {
+        result = result | (cast<uint32_t>(bytes(base + i, blk)) << (8 * i));
+    }
+    return result;
+}
+
+inline Halide::Expr le_u32(Halide::Func bytes, Halide::Expr base, Halide::Var blk) {
+    using namespace Halide;
+    return le_uint(bytes, base, blk, 4);
+}
+
+// le_uint's encode-side mirror: byte `byte_idx` (little-endian) of `bits`.
+// Widening to uint32 first makes one expression work for every word size
+// used below (Fp16Pack's 16-bit word, F32Pack's 32-bit one, Int16Pack's
+// per-group 16-bit one) -- shifting a narrower type by up to 24 bits would
+// silently truncate instead.
+inline Halide::Expr word_to_le_byte(Halide::Expr bits, Halide::Expr byte_idx) {
+    using namespace Halide;
+    return cast<uint8_t>((cast<uint32_t>(bits) >> (cast<uint32_t>(byte_idx) * 8)) & 0xff);
+}
+
+// le_uint's encode-side-agnostic generalization: a little-endian 16-bit word
+// from two bytes, but via an arbitrary per-byte accessor (`byte_at(0)` = low
+// byte, `byte_at(1)` = high byte) instead of a fixed `Func bytes` indexed at
+// (base+i, blk) -- so it also covers a Halide::_-general decoder's
+// bytes(offset, blk, _) reads, or a byte pair that isn't a plain Func index
+// at all (e.g. IQ1_M's per-scale-word accumulation further below). Replaces
+// the hand-rolled "lo | (hi << 8)" pattern that used to be written out at
+// each call site.
+template<typename ByteAt>
+inline Halide::Expr le_u16(ByteAt byte_at) {
+    using namespace Halide;
+    Expr lo = cast<uint16_t>(byte_at(0));
+    Expr hi = cast<uint16_t>(byte_at(1));
+    return cast<uint16_t>(lo | (hi << 8));
+}
+
+// encode(float scale) -> 2 bytes; decode(2 bytes) -> float. Matches every
+// format's existing fp16 delta-byte code (byte0 = bits&0xff, byte1 =
+// (bits>>8)&0xff).
+class Fp16Pack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func scale = inputs[0];  // scale(blk)
+        Var byte("byte"), blk("blk");
+        Expr bits = reinterpret<uint16_t>(cast<float16_t>(scale(blk)));
+        Func bytes("fp16_pack_bytes");
+        bytes(byte, blk) = word_to_le_byte(bits, byte);
+        return {{bytes}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte, blk[, ...]), byte in [0, 2)
+        Var blk("blk");
+        Expr bits = le_u16([&](int i) { return bytes(i, blk, _); });
+        Func scale("fp16_pack_scale");
+        scale(blk, _) = cast<float>(reinterpret<float16_t>(bits));
+        return {{scale}, {}};
+    }
+};
+
+// encode(float scale) -> 4 bytes (plain IEEE-754 binary32, little-endian);
+// decode(4 bytes) -> float -- Q8_K's scale format, the one format here whose
+// delta is a full float, not fp16 (matching block_q8_K's `float d;`, not
+// GGML's usual `ggml_fp16_t d;`).
+class F32Pack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func scale = inputs[0];  // scale(blk)
+        Var byte("byte"), blk("blk");
+        Expr bits = reinterpret<uint32_t>(scale(blk));
+        Func bytes("f32_pack_bytes");
+        bytes(byte, blk) = word_to_le_byte(bits, byte);
+        return {{bytes}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte, blk[, ...]), byte in [0, 4)
+        Var blk("blk");
+        // Dimension-general via Halide::_ (matches Fp16Pack): any trailing
+        // "lane" dims -- e.g. a repack weight's columns -- ride through, so this
+        // pack can decode a repack scale field, not just a plain (byte, blk) one.
+        Expr b0 = cast<uint32_t>(bytes(0, blk, _)), b1 = cast<uint32_t>(bytes(1, blk, _));
+        Expr b2 = cast<uint32_t>(bytes(2, blk, _)), b3 = cast<uint32_t>(bytes(3, blk, _));
+        Func scale("f32_pack_scale");
+        scale(blk, _) = reinterpret<float>(b0 | (b1 << 8) | (b2 << 16) | (b3 << 24));
+        return {{scale}, {}};
+    }
+};
+
+// encode(int16 values(g, blk)) -> 2 little-endian bytes each, byte `2g`/
+// `2g+1` -- e.g. Q8_K's bsums[16] int16 array (one value per 16-element
+// group, not one per block).
+class Int16Pack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func values = inputs[0];  // values(g, blk)
+        Var byte_idx("byte_idx"), blk("blk");
+        Expr g = byte_idx / 2;
+        Expr bits = reinterpret<uint16_t>(values(g, blk));
+        Func bytes("int16_pack_bytes");
+        bytes(byte_idx, blk) = word_to_le_byte(bits, byte_idx % 2);
+        return {{bytes}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte_idx, blk), byte_idx in [0, 2*num_groups)
+        Var g("g"), blk("blk");
+        Expr bits = le_u16([&](int i) { return bytes(2 * g + i, blk); });
+        Func values("int16_pack_values");
+        values(g, blk) = reinterpret<int16_t>(bits);
+        return {{values}, {}};
+    }
+};
+
+// decode(1 byte) -> float, an E8M0 power-of-two exponent (GGML's MXFP4/NVFP4
+// scale format). Reproduces ggml_e8m0_to_fp32_half's exact bit construction:
+// d = 2^(e-128) for every e in [0, 255], computed via a subnormal-exploiting
+// shift trick for e<2 instead of a normal exponent-field write (both branches
+// compute the same uniform 2^(e-128); see the comment inline). Decode-only:
+// MXFP4 quantize is extern-delegated (see ExternQuantize).
+class E8M0Pack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "E8M0Pack is decode-only -- quantize is deferred to an ExternQuantize.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func byte = encoded[0];  // byte(byte_idx, blk[, ...]), byte_idx in [0, 1)
+        Var blk("blk");
+        // Dimension-general via Halide::_ (matches Fp16Pack/F32Pack) so this pack
+        // can decode a repack weight's E8M0 scale header, columns riding through.
+        Expr e = cast<uint32_t>(byte(0, blk, _));
+        Expr bits = select(e < 2, cast<uint32_t>(0x00200000) << e, (e - 1) << 23);
+        Func scale("e8m0_pack_scale");
+        scale(blk, _) = reinterpret<float>(bits);
+        return {{scale}, {}};
+    }
+};
+
+// decode(1 byte per sub-block) -> float(sub, blk), a UE4M3 unsigned
+// 4-exponent/3-mantissa float (GGML's NVFP4 per-sub-block scale format).
+// Reproduces ggml_ue4m3_to_fp32_half's exact construction: subnormal
+// (exp==0) is man/512, normal is (1+man/8)*2^(exp-7), both halved; byte 0x00
+// or 0x7f (GGML's NVFP4 zero/sentinel bytes) decode to 0. Unlike
+// Fp16Pack/E8M0Pack (exactly one scale value per block), this is meant to be
+// used with LinearDequant's per-sub-block (sub_size > 0) mode: the Funcs here are indexed by
+// `sub` directly (one byte each). Decode-only: NVFP4 quantize is
+// extern-delegated (see ExternQuantize).
+class UE4M3Pack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "UE4M3Pack is decode-only -- quantize is deferred to an ExternQuantize.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func byte = encoded[0];  // byte(sub, blk)
+        Var sub("sub"), blk("blk");
+        Expr ue = byte(sub, blk);                                  // codespell:ignore ue
+        Expr is_zero = (ue == 0) || (ue == 0x7f);                  // codespell:ignore ue
+        Expr exp_ = cast<int32_t>(cast<uint32_t>(ue) >> 3) & 0xf;  // codespell:ignore ue
+        Expr man_ = cast<int32_t>(ue) & 0x7;                       // codespell:ignore ue
+        Expr raw = select(exp_ == 0,
+                          cast<float>(man_) / 512.0f,
+                          (1.0f + cast<float>(man_) / 8.0f) * pow(2.0f, cast<float>(exp_ - 7)));
+        Func scale("ue4m3_pack_scale");
+        scale(sub, blk) = select(is_zero, 0.0f, raw * 0.5f);
+        return {{scale}, {}};
+    }
+};
+
+// The four scalar-scale on-disk formats above (Fp16Pack/F32Pack/E8M0Pack/
+// UE4M3Pack), named so call sites that need to pick one at runtime (the
+// repack weight schemes' RepackWeightScale used to be a bespoke copy of
+// exactly this same enum; make_codebook_scheme's scale_pack/scale_bytes
+// parameter pair was the other) can hand a single value across instead of a
+// (unique_ptr<Approximation>, int width) pair kept in sync by hand.
+enum class ScaleFormat { Fp16,
+                         F32,
+                         E8M0,
+                         UE4M3 };
+
+inline std::unique_ptr<Halide::Approximation> make_scale_pack(ScaleFormat fmt) {
+    switch (fmt) {
+    case ScaleFormat::Fp16:
+        return std::make_unique<Fp16Pack>();
+    case ScaleFormat::F32:
+        return std::make_unique<F32Pack>();
+    case ScaleFormat::E8M0:
+        return std::make_unique<E8M0Pack>();
+    case ScaleFormat::UE4M3:
+        return std::make_unique<UE4M3Pack>();
+    }
+    _halide_internal_error << "unreachable ScaleFormat\n";
+}
+
+// On-disk byte width of `num_scales` consecutive values in `fmt` (e.g.
+// NVFP4's 4 per-sub-block UE4M3 bytes, or a repack weight's n_cols-wide
+// per-column scale header).
+inline int scale_width(ScaleFormat fmt, int num_scales = 1) {
+    switch (fmt) {
+    case ScaleFormat::Fp16:
+        return 2 * num_scales;
+    case ScaleFormat::F32:
+        return 4 * num_scales;
+    case ScaleFormat::E8M0:
+        return num_scales;
+    case ScaleFormat::UE4M3:
+        return num_scales;
+    }
+    _halide_internal_error << "unreachable ScaleFormat\n";
+}
+
+// The shared byte<->field decomposition behind every uniform-width,
+// non-base-3 bit-packed field in this file (nibble packs, 2-bit packs, and
+// the K-quants' rotating out-of-band high-bit arrays): a flat per-block
+// index kk factors as
+//   kk = outer*(plane_count*pos_count) + plane*pos_count + pos
+// and each byte packs `plane_count` fixed-width `field_bits`-bit fields
+// ("planes") -- byte `outer*pos_count + pos` holds element (outer, plane, pos)
+// at bit-shift `plane*field_bits`. This one decomposition is exactly (verified
+// by hand, not just formally) GGML's addressing for all of the following, each
+// just a different (field_bits, plane_count, pos_count) instantiated directly
+// at its call site below rather than through a named subclass, since the
+// class itself is the entire "component" here -- a named wrapper would only
+// rename a handful of integers, not remove any duplication:
+//   - Nibble-packed formats (Q4_0, Q4_1, Q5_0's low bits, Q4_K, NVFP4, ...):
+//     4-bit fields, 2 planes (low/high nibble), pos_count = window_size/2,
+//     outer = which independent sub-block "window" (1 for a plain whole-block
+//     low-half/high-half split, e.g. Q4_0/IQ4_NL; >1 for NVFP4/Q4_K/Q5_K's
+//     independent windows) -- GGML's actual convention for every nibble-
+//     packed format, not just the more obvious "adjacent pair per byte"
+//     layout.
+//   - TQ2_0/Q2_K/Q3_K/Q6_K's 2-bit fields: 2-bit fields, 4 planes, pos_count
+//     = block_size/8 (= window_bytes), outer = which half of the block --
+//     verified by hand against each format's original dequantize math (e.g.
+//     TQ2_0's `half = gi/half_block; l = local/window_bytes; m =
+//     local%window_bytes; byte_idx = half*window_bytes + m; shift = l*2`);
+//     NOT a plain "4 consecutive elements per byte" packing.
+//   - Q3_K's hmask / Q5_K's qh out-of-band high-bit arrays: 1-bit fields,
+//     num_windows planes, pos_count = window_size, outer always 0 (the whole
+//     array is one window_size*num_windows group) -- the "rotating bit
+//     position" scheme GGML uses for an extra high bit per element (their own
+//     source expresses this via a `half`/`iter`-based case split instead, but
+//     it's the same addressing). Always paired with a lower-bit code via
+//     CombineBits, never used standalone.
+// `qmax` recenters already-decoded codes before splitting (0 leaves the raw
+// field, e.g. a lookup-table index or a single out-of-band bit -- always 0
+// for the two cases above). encode() OR-accumulates the planes into each
+// byte via an RDom, exactly like BitPack's qh_accum-style OR-reduction.
+//
+// `plane_axis` (default false) selects an alternate decode that keeps `plane`
+// as an explicit LEADING output axis instead of folding it into a flat kk:
+//   fields(plane, pos, blk, _) = (bytes(pos, blk, _) >> plane*field_bits) & mask
+// This is the shape a *combined* (scale, min) field wants -- plane 0 = scale,
+// plane 1 = min -- so LinearDequant can read both from one func by its
+// plane index. It's dimension-general (Halide::_) and needs neither `outer` nor
+// `qmax` (raw unsigned fields), so it is exactly Q2_K's per-sub-block nibble-
+// pair (scale, min) byte array, with no bespoke leaf. Decode-only in this mode
+// (only ever an extern-delegated TrustedInverse decoder stage).
+class PlanarBitPack : public Halide::Approximation {
+public:
+    // plane_count is always 8/field_bits in every instantiation here (a byte
+    // packs exactly 8 bits' worth of same-width fields, full stop), so it's
+    // derived rather than taken as its own parameter -- see nibble_pack/
+    // crumb_pack/rotating_bit_pack/le_bit_pack below for the named-shape
+    // constructors most call sites should use instead of this directly.
+    PlanarBitPack(int field_bits, int pos_count, int qmax = 0, bool plane_axis = false,
+                  int value_scale = 1)
+        : field_bits_(field_bits), plane_count_(8 / field_bits), pos_count_(pos_count), qmax_(qmax),
+          plane_axis_(plane_axis), value_scale_(value_scale) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        if (plane_axis_) {
+            _halide_user_error << "PlanarBitPack plane-axis mode is decode-only "
+                                  "(only an extern-delegated TrustedInverse decoder stage).\n";
+            return {};
+        }
+        Func codes = inputs[0];
+        Var byte_idx("byte_idx"), blk("blk");
+        int group = plane_count_ * pos_count_;
+        Expr outer = byte_idx / pos_count_;
+        Expr pos = byte_idx % pos_count_;
+
+        RDom rp(0, plane_count_, "rp");
+        Expr kk = outer * group + rp * pos_count_ + pos;
+        Expr field = cast<uint32_t>((cast<int32_t>(codes(kk, blk)) + qmax_) / value_scale_) &
+                     ((1u << field_bits_) - 1);
+        Func bytes("planar_bit_pack_bytes");
+        bytes(byte_idx, blk) = cast<uint8_t>(0);
+        bytes(byte_idx, blk) = bytes(byte_idx, blk) | cast<uint8_t>(field << (rp * field_bits_));
+        return {{bytes}, {bytes}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];
+        Var kk("kk"), blk("blk");
+        if (plane_axis_) {
+            Var plane("plane"), pos("pos");
+            Expr field = (cast<uint32_t>(bytes(pos, blk, _)) >> (cast<uint32_t>(plane) * field_bits_)) &
+                         ((1u << field_bits_) - 1);
+            Func fields("planar_bit_pack_fields");
+            fields(plane, pos, blk, _) = cast<uint8_t>(field);
+            return {{fields}, {}};
+        }
+        // Lane-general via Halide::_ (see the DIMENSION / WILDCARD CONVENTION
+        // in the file's top comment), like the plane-axis mode above.
+        int group = plane_count_ * pos_count_;
+        Expr outer = kk / group;
+        Expr rem = kk % group;
+        Expr plane = rem / pos_count_;
+        Expr pos = rem % pos_count_;
+        Expr byte_idx = outer * pos_count_ + pos;
+        Expr byte = cast<uint32_t>(bytes(byte_idx, blk, _));
+        Expr field;
+        if (field_bits_ == 1) {
+            // A single-bit field (Q5_0/Q5_1's per-element high bit). Read it from
+            // a compile-time byte->bits expansion table (ggml's table_b2b idea):
+            // b2b(bit, byte) = (byte >> bit) & 1. The alternative arithmetic form
+            // `(byte >> plane) & 1` lowers to a per-lane byte-broadcast transpose
+            // plus mask (~24 NEON ops per 16 lanes), which is the whole
+            // q5_x-vs-q4_x gap. A Buffer<> is embedded in the binary as constant
+            // data, so this is a pure lookup, no runtime input.
+            static const Halide::Buffer<int8_t> b2b = [] {
+                Halide::Buffer<int8_t> t(8, 256);
+                t.set_min(0, 0);
+                for (int by = 0; by < 256; by++) {
+                    for (int bit = 0; bit < 8; bit++) {
+                        t(bit, by) = (by >> bit) & 1;
+                    }
+                }
+                return t;
+            }();
+            static const Halide::Buffer<int8_t> b2b_shifted = [] {
+                Halide::Buffer<int8_t> t(8, 256);
+                t.set_min(0, 0);
+                for (int by = 0; by < 256; by++) {
+                    for (int bit = 0; bit < 8; bit++) {
+                        t(bit, by) = ((by >> bit) & 1) << 4;
+                    }
+                }
+                return t;
+            }();
+            static const Halide::Buffer<int8_t> b2b_shifted_signed = [] {
+                Halide::Buffer<int8_t> t(8, 256);
+                t.set_min(0, 0);
+                for (int by = 0; by < 256; by++) {
+                    for (int bit = 0; bit < 8; bit++) {
+                        t(bit, by) = (((by >> bit) & 1) << 4) - 16;
+                    }
+                }
+                return t;
+            }();
+            const Expr bit_index = cast<int32_t>(plane * field_bits_);
+            const Expr byte_value = cast<int32_t>(byte);
+            if (value_scale_ == 16 && qmax_ == 16) {
+                field = b2b_shifted_signed(bit_index, byte_value);
+            } else if (value_scale_ == 16 && qmax_ == 0) {
+                field = b2b_shifted(bit_index, byte_value);
+            } else {
+                field = cast<int32_t>(b2b(bit_index, byte_value)) * value_scale_ - qmax_;
+            }
+        } else {
+            field = cast<int32_t>((byte >> (plane * field_bits_)) & ((1u << field_bits_) - 1)) * value_scale_ - qmax_;
+        }
+        Func codes("planar_bit_pack_codes");
+        codes(kk, blk, _) = cast<int8_t>(field);
+        return {{codes}, {}};
+    }
+
+private:
+    int field_bits_, plane_count_, pos_count_, qmax_;
+    bool plane_axis_;
+    int value_scale_;
+};
+
+// Named PlanarBitPack shapes for the four bit-widths this file actually
+// instantiates, so a call site reads as "a nibble pack over this many
+// elements" instead of a bare (field_bits, pos_count) pair the reader has to
+// re-derive the meaning of. `window` is the element span PlanarBitPack's own
+// class comment calls "window_size": for nibble_pack/crumb_pack, the size of
+// one independently low/high- (or low/high-2-bit-) split group (pos_count is
+// window/2 or window/4, one plane's share of it); for rotating_bit_pack, the
+// single rotating-bit-position span itself (pos_count = window directly,
+// since GGML's hmask/qh arrays have exactly one such group covering the
+// *whole* field, not several independent windows).
+inline std::unique_ptr<Halide::Approximation> nibble_pack(int window, int qmax = 0) {
+    return std::make_unique<PlanarBitPack>(4, window / 2, qmax);
+}
+inline std::unique_ptr<Halide::Approximation> crumb_pack(int window, int qmax = 0) {
+    return std::make_unique<PlanarBitPack>(2, window / 4, qmax);
+}
+inline std::unique_ptr<Halide::Approximation> rotating_bit_pack(int window, int qmax = 0) {
+    return std::make_unique<PlanarBitPack>(1, window, qmax);
+}
+// The Stage-2 qh addressing (make_code_pack's code_bits==5 case): one flat
+// bit per element, byte kk/8 at shift kk%8 -- PlanarBitPack{1, 1} regardless
+// of block size (pos_count is always 1; there's no "window" to parameterize).
+inline std::unique_ptr<Halide::Approximation> le_bit_pack(int value_scale = 1, int qmax = 0) {
+    return std::make_unique<PlanarBitPack>(1, 1, qmax, false, value_scale);
+}
+
+// GGML's Q5_0/Q5_1 5-bit code split (a 4-bit low nibble plus a 5th high bit,
+// OR-accumulated one bit per element into a 32-bit little-endian word) used
+// to be a bespoke FiveBitPack class here. It's deleted: verified by hand that
+// it is exactly the K-quant "combined bit code" shape (CombineBits, section
+// 5 below) already used for Q3_K/Q5_K/Q6_K's own adjacent {high-bit array;
+// low-bits array} regions -- qh's bit `kk` is element kk's own high bit,
+// i.e. le_bit_pack()'s addressing (byte kk/8, shift kk%8; GGML's own code
+// computes this via a low/high-half split for scalar-loop efficiency -- e.g.
+// `(qh >> (byte_idx+12)) & 0x10` for the high half -- but that's an
+// equivalent, more roundabout way of writing the same fact used directly
+// here), and the nibble half is exactly nibble_pack(block_size) (byte b
+// holds elements b and b+block_size/2 at shifts 0/4). See make_code_pack's
+// code_bits==5 case below, which assembles this
+// via make_combined_bit_codec exactly as make_q5_k_scheme does for its own
+// adjacent {qh; qs} region.
+
+// encode(codes(kk, blk) signed in {-1, +1}) -> block_size/8 bytes; decode
+// reverses it. Packs one sign bit per element, 8 elements per byte, bit
+// `kk % 8` of byte `kk / 8` set when code is +1 -- Q1_0's layout (paired
+// with RoundingMode::SignOnly/ScaleAnchor::MeanAbs above). Accumulates via
+// an OR-reduction exactly like PlanarBitPack::encode's per-byte accumulation,
+// just one full byte's worth of bits at a time instead of a per-plane field.
+class BitPack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func codes = inputs[0];
+        Var byte_idx("byte_idx"), blk("blk");
+
+        Func bit("bit_pack_bit");
+        Var kk("kk");
+        bit(kk, blk) = cast<uint8_t>(select(codes(kk, blk) > 0, 1, 0));
+
+        RDom rb(0, 8, "rb");
+        Func bytes("bit_pack_bytes");
+        bytes(byte_idx, blk) = cast<uint8_t>(0);
+        bytes(byte_idx, blk) = bytes(byte_idx, blk) | cast<uint8_t>(bit(byte_idx * 8 + rb, blk) << rb);
+
+        return {{bytes}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte_idx, blk), byte_idx in [0, block_size/8)
+        Var kk("kk"), blk("blk");
+        Expr byte_idx = kk / 8;
+        Expr bit_off = kk % 8;
+        Expr bit = (cast<uint32_t>(bytes(byte_idx, blk)) >> bit_off) & 1u;
+        Func codes("bit_pack_codes");
+        codes(kk, blk) = cast<int8_t>(select(bit != 0, 1, -1));
+        return {{codes}, {}};
+    }
+};
+
+// encode(codes(kk, blk) signed int8) -> 1 code per byte, same shape (the
+// identity-shaped case PlanarBitPack's nibble/2-bit packing doesn't cover,
+// used by e.g. Q8_0). Unlike PlanarBitPack, this formula has no precondition
+// on kk's range (reinterpret is valid for any kk) -- it doesn't know or care
+// what block_size is; bounds propagate backward from whatever actually
+// consumes it.
+class BytePack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func codes = inputs[0];
+        Var kk("kk"), blk("blk");
+        Func bytes("byte_pack_bytes");
+        bytes(kk, blk) = reinterpret<uint8_t>(codes(kk, blk));
+        return {{bytes}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];
+        Var kk("kk"), blk("blk");
+        // Lane-general via Halide::_ (see the DIMENSION / WILDCARD CONVENTION
+        // in the file's top comment): trailing lane dims ride through.
+        Func codes("byte_pack_codes");
+        codes(kk, blk, _) = reinterpret<int8_t>(bytes(kk, blk, _));
+        return {{codes}, {}};
+    }
+};
+
+// decode(52 bytes {qs[48]; qh[4]}) -> codes(kk, blk), the raw base-3 digit in
+// [0, 3) -- GGML's TQ1_0 packing: 256 elements in 3 sections (a 160-element
+// and an 80-element run at 5 trits/byte, then a 16-element run at 4 real
+// trits/byte, its 5th digit always 0). decode() reverses the ceiling-division
+// packing (`byte = ceil(digit_number * 256 / 243)`) via the same
+// multiply-truncate-rescale trick tq1_0_generators.cpp hand-rolled: extracting
+// digit `n` needs multiplier `3^n`, `n` from the most-significant digit. Codes
+// feed a codebook index directly (TQ2_0's {-1, 0, 1, unused} convention).
+// Decode-only: TQ1_0 quantize is extern-delegated.
+class TritPack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "TritPack is decode-only -- quantize is deferred to an ExternQuantize.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte_idx, blk), byte_idx in [0, 52)
+        Var kk("kk"), blk("blk");
+
+        Expr n_a = kk / 32;
+        Expr byte_a = kk % 32;
+
+        Expr local_b = kk - 160;
+        Expr n_b = local_b / 16;
+        Expr byte_b = 32 + local_b % 16;
+
+        Expr local_c = kk - 240;
+        Expr n_c = local_c / 4;
+        Expr byte_c = 48 + local_c % 4;
+
+        Expr n = select(kk < 160, n_a, select(kk < 240, n_b, n_c));
+        Expr byte_abs = select(kk < 160, byte_a, select(kk < 240, byte_b, byte_c));
+
+        Expr byte_val = bytes(byte_abs, blk);
+        Expr p3 = mux(n, {1, 3, 9, 27, 81, 243});
+
+        Expr q_trunc = cast<uint8_t>(widening_mul(byte_val, p3));
+        Expr xi = cast<int32_t>((cast<uint16_t>(q_trunc) * 3) >> 8);
+
+        Func codes("trit_pack_codes");
+        codes(kk, blk) = cast<int8_t>(xi);
+        return {{codes}, {}};
+    }
+};
+
+// ---------------------------------------------------------------------------
+// 3b. Derived extra fields (computed from other already-encoded fields, not
+//     from the original values -- appended before struct-packing).
+// ---------------------------------------------------------------------------
+
+// The two ways a derived "sum of codes" extra field gets appended:
+//   - ScaledFloat: one sum for the *whole* block, already multiplied by
+//     scale into a float -- GGML's Q8_1 "s" field (group_size == block_size,
+//     a single group), letting a paired vec_dot recover sum(dequantized
+//     values) cheaply from the block's own header instead of re-reducing
+//     codes itself.
+//   - RawInt16: one sum *per group* (group_size < block_size, several
+//     groups), each a plain int32-then-int16 integer sum with no scale
+//     multiply -- GGML's Q8_K "bsums" field, letting a paired K-quant
+//     vec_dot recover each 16-element group's sum of raw int8 codes cheaply.
+enum class SumMode { ScaledFloat,
+                     RawInt16 };
+
+// encode({codes(kk, blk), scale(blk)}) -> {codes, scale, sum}: sum(blk) (no
+// group dim) for ScaledFloat, sum(g, blk) for RawInt16 -- see SumMode above.
+// decode() discards sum and passes codes/scale through unchanged in both
+// modes: it's a redundant, derivable quantity, not needed to reconstruct
+// dequantized values, so there's nothing to invert. Arity-changing like the
+// grouped FieldSpec fields make_block_layout composes (see FieldSpec/
+// FieldLayout above), but in the *encode* direction instead (2 inputs -> 3
+// outputs; decode then undoes it in the same direction rather than the
+// mirror one, since sum isn't invertible into anything -- it's simply
+// dropped).
+class AppendSums : public Halide::Approximation {
+public:
+    AppendSums(int group_size, SumMode mode)
+        : group_size_(group_size), mode_(mode) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func codes = inputs[0], scale = inputs[1];
+        Var blk("blk");
+
+        if (mode_ == SumMode::ScaledFloat) {
+            RDom r(0, group_size_, "r");
+            Func sum_i("append_sums_i");
+            sum_i(blk) = 0;
+            sum_i(blk) += cast<int32_t>(codes(r, blk));
+
+            Func sum_f("append_sums_scaled");
+            sum_f(blk) = cast<float>(sum_i(blk)) * scale(blk);
+
+            return {{codes, scale, sum_f}, {sum_i}};
+        }
+        Var g("g");
+        RDom rg(0, group_size_, "rg");
+
+        Func sum_i("append_sums_i");
+        sum_i(g, blk) = cast<int32_t>(0);
+        sum_i(g, blk) += cast<int32_t>(codes(g * group_size_ + rg, blk));
+
+        Func bsums("append_sums_raw");
+        bsums(g, blk) = cast<int16_t>(sum_i(g, blk));
+
+        return {{codes, scale, bsums}, {sum_i}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        // The sum (encoded[2]) is a redundant derived quantity -- pass
+        // codes/scale through unchanged.
+        return {{encoded[0], encoded[1]}, {}};
+    }
+
+private:
+    int group_size_;
+    SumMode mode_;
+};
+
+// ---------------------------------------------------------------------------
+// 3c. Concatenation into one byte buffer.
+// ---------------------------------------------------------------------------
+
+// Concatenates N already-packed, fixed-width byte fields into one
+// byte-addressed buffer per block, at fixed offsets -- generalizes the
+// per-format "select(byte==0, delta_byte0, byte==1, delta_byte1,
+// packed(...))" pattern duplicated in every *_generators.cpp quantize
+// function today.
+//
+// `field_widths[k]` is the k-th field's width in bytes, *in on-disk byte
+// order*: `inputs[k]`/`encoded[k]` (encode/decode respectively) must already
+// be in that same order. When the rest of a Compose/Apply chain produces
+// fields in some other order (e.g. {codes_bytes, scale_bytes} when the
+// on-disk layout is scale-then-codes, as in block_q4_0), reorder into byte
+// order with a Permute stage composed in front of/behind this one --
+// FieldLayout below is the one call site that needs this, via
+// Permute{slots_of(fields_)}.
+class StructPack : public Halide::Approximation {
+public:
+    explicit StructPack(std::vector<int> field_widths)
+        : field_widths_(std::move(field_widths)) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Var byte("byte"), blk("blk");
+        std::vector<int> offsets = offsets_in_output_order();
+
+        // select()'s branches are all evaluated unconditionally (not
+        // short-circuiting), so each field's local index is clamped to its
+        // own valid range before use -- same idiom as q4_0_generators.cpp.
+        Expr result = cast<uint8_t>(0);
+        for (int k = (int)field_widths_.size() - 1; k >= 0; k--) {
+            Expr local = clamp(byte - offsets[k], 0, field_widths_[k] - 1);
+            Expr in_range = byte >= offsets[k] && byte < offsets[k] + field_widths_[k];
+            result = select(in_range, inputs[k](local, blk), result);
+        }
+
+        Func packed("struct_pack_packed");
+        packed(byte, blk) = result;
+        return {{packed}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func packed = encoded[0];
+        std::vector<int> offsets = offsets_in_output_order();
+        Var local("local"), blk("blk");
+
+        std::vector<Func> fields;
+        fields.reserve(field_widths_.size());
+        for (int k = 0; k < (int)field_widths_.size(); k++) {
+            Func field("struct_pack_field_" + std::to_string(k));
+            field(local, blk) = packed(local + offsets[k], blk);
+            fields.push_back(field);
+        }
+        return {fields, {}};
+    }
+
+private:
+    std::vector<int> offsets_in_output_order() const {
+        std::vector<int> offsets(field_widths_.size());
+        int acc = 0;
+        for (size_t k = 0; k < field_widths_.size(); k++) {
+            offsets[k] = acc;
+            acc += field_widths_[k];
+        }
+        return offsets;
+    }
+
+    std::vector<int> field_widths_;
+    std::vector<int> input_index_;
+};
+
+// A struct-typed replacement for StructPack + Permute + the scale field's
+// Fp16Pack, backed by a first-class Halide::Type::Struct instead of hand-summed
+// byte offsets. `block_type` is the on-disk block's struct type (e.g.
+// block_q4_0's `{fp16 d; uint8 qs[16]}`); the compiler owns the field offsets
+// and the total byte size (`block_type.bytes()`), so nothing here computes them.
+//
+// This leaf sits at the OUTERMOST (on-disk-byte) end of a scheme's Compose,
+// exactly where the old make_block_layout() stack did. It produces the same two
+// logical Funcs the symmetric/affine quantize stage consumes, in slot order:
+//   slot 0: `codes_bytes(local, blk)` -- the raw UInt(8) bytes of the codes
+//           field, still to be interpreted by the code_pack (nibble/byte/bit
+//           extraction) composed just inside this leaf. Struct types subsume the
+//           *layout* of these bytes, not the packing trick that reads sub-byte
+//           codes out of them.
+//   slot 1: `scale(blk)` -- the block's scale, read straight out of the typed
+//           `d` field (Float(16) -> Float(32)); this is what subsumes Fp16Pack's
+//           manual `lo | (hi<<8)` reassembly + reinterpret.
+// v1 pilot: exactly one scalar scale field + one UInt(8) array codes field (the
+// block_q4_0/block_q8_0 shape). Affine (min) and split-code (q5_0) layouts stay
+// on make_block_layout for now.
+class StructBlockLayout : public Halide::Approximation {
+public:
+    StructBlockLayout(Halide::Type block_type, std::string scale_field, std::string codes_field)
+        : block_type_(block_type), scale_field_(std::move(scale_field)), codes_field_(std::move(codes_field)) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func codes_bytes = inputs[0];  // codes_bytes(local, blk), UInt(8)
+        Func scale = inputs[1];        // scale(blk), Float(32)
+        Var blk("blk");
+
+        // Assemble one value per field element, in field declaration order, for
+        // the flattened pack_struct() form. The compiler places each at its own
+        // offset -- no hand-rolled shifts/masks/reinterpret in the reverse
+        // direction the way the old encode path needed.
+        const StructTypeInfo *info = block_type_.struct_type();
+        std::vector<Expr> vals;
+        for (const StructField &f : info->fields) {
+            int extent = f.array_extent.value_or(1);
+            for (int i = 0; i < extent; i++) {
+                if (f.name == scale_field_) {
+                    vals.push_back(cast(f.type, scale(blk)));
+                } else if (f.name == codes_field_) {
+                    vals.push_back(cast(f.type, codes_bytes(i, blk)));
+                } else {
+                    _halide_internal_error << "StructBlockLayout: unexpected field \"" << f.name << "\"\n";
+                }
+            }
+        }
+
+        Func packed("struct_block_packed");
+        packed(blk) = pack_struct(block_type_, vals);
+        return {{packed}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func packed = encoded[0];  // packed(blk), struct-typed
+        Var local("local"), blk("blk");
+
+        Func scale("struct_block_scale");
+        scale(blk) = cast<float>(field(packed(blk), scale_field_));
+
+        Func codes_bytes("struct_block_codes_bytes");
+        codes_bytes(local, blk) = cast<uint8_t>(field(packed(blk), codes_field_)[local]);
+
+        return {{codes_bytes, scale}, {}};
+    }
+
+private:
+    Halide::Type block_type_;
+    std::string scale_field_, codes_field_;
+};
+
+// Struct-typed layout for the split-code Q5_0/Q5_1 blocks. Keeping qh as a
+// typed UInt(32) field lets LowerStructTypes expose one four-byte load to LLVM;
+// the byte-buffer form presents four unrelated UInt(8) loads, which cannot be
+// coalesced before the byte-indexed expansion-table lookups. `codes_bytes`
+// retains the existing logical {qh[4], qs[16]} layout, so the reusable
+// combined-bit codec inside this leaf is unchanged.
+class Q5StructBlockLayout : public Halide::Approximation {
+public:
+    Q5StructBlockLayout(Halide::Type block_type, bool affine)
+        : block_type_(block_type), affine_(affine) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func codes_bytes = inputs[0];
+        Func scale = inputs[1];
+        Func min = affine_ ? inputs[2] : Func();
+        Var blk("blk");
+
+        Expr qh = cast<uint32_t>(codes_bytes(0, blk)) |
+                  (cast<uint32_t>(codes_bytes(1, blk)) << 8) |
+                  (cast<uint32_t>(codes_bytes(2, blk)) << 16) |
+                  (cast<uint32_t>(codes_bytes(3, blk)) << 24);
+        std::vector<Expr> vals = {cast<float16_t>(scale(blk))};
+        if (affine_) {
+            vals.push_back(cast<float16_t>(min(blk)));
+        }
+        vals.push_back(qh);
+        for (int i = 0; i < 16; i++) {
+            vals.push_back(codes_bytes(i + 4, blk));
+        }
+
+        Func packed("q5_struct_block_packed");
+        packed(blk) = pack_struct(block_type_, vals);
+        return {{packed}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func packed = encoded[0];
+        Var local("local"), blk("blk");
+
+        Func scale("q5_struct_block_scale");
+        scale(blk) = cast<float>(field(packed(blk), "d"));
+        Func qh("q5_struct_block_qh");
+        qh(blk) = field(packed(blk), "qh");
+        Func qh_bytes("q5_struct_block_qh_bytes");
+        qh_bytes(local, blk) = cast<uint8_t>(qh(blk) >> (local * 8));
+        Func qs_bytes("q5_struct_block_qs_bytes");
+        qs_bytes(local, blk) = cast<uint8_t>(field(packed(blk), "qs")[local]);
+        Func codes_bytes("q5_struct_block_codes_bytes");
+        codes_bytes(local, blk) = select(local < 4,
+                                         qh_bytes(local, blk),
+                                         qs_bytes(local - 4, blk));
+
+        if (affine_) {
+            Func min("q5_struct_block_min");
+            min(blk) = cast<float>(field(packed(blk), "m"));
+            return {{codes_bytes, scale, min}, {qh}};
+        }
+        return {{codes_bytes, scale}, {qh}};
+    }
+
+private:
+    Halide::Type block_type_;
+    bool affine_;
+};
+
+// One field, in ON-DISK byte order, of a struct-packed block layout: an
+// on-disk byte width plus which logical "slot" it lands in -- the index it
+// occupies in the Func vector immediately after StructPack::decode() (and,
+// symmetrically, immediately before StructPack::encode()) -- and how to
+// pack/unpack it. This is the single declaration that used to be split three
+// ways at every make_*_scheme call site: a StructPack{widths, input_index},
+// a stack of Apply{i,1,1,pack} lines whose `i` had to be kept in sync with
+// StructPack's own indices by hand, and (at the Generator call sites) a
+// hand-summed block byte count. FieldSpec/make_block_layout below fold all
+// three into one list.
+//
+// Most fields are their own arity-1 group: `pack` set, `arity` left at its
+// default of 1, one FieldSpec per on-disk field. A few fields aren't packed
+// independently, though -- make_code_pack's code_bits==5 combined codec's
+// {nibble, qh} decode into one `codes` field together (Q5_0/Q5_1's split
+// 5-bit code), and IQ4XSScalePack's {scales_h, scales_l} decode into one
+// `scale` field together. For a group like that, list every physical on-disk
+// field it spans (so StructPack still gets each one's own width/slot), but
+// only the *leader* -- conventionally, the one whose pack actually does the
+// work -- carries `pack` and `arity` (= how many consecutive slots,
+// [[slot, slot+arity), the group spans); every other member of the group
+// leaves `pack` null (a "this slot is spoken for by an earlier FieldSpec's
+// group" marker) and `arity` at its default (unused for non-leaders).
+struct FieldSpec {
+    int slot;
+    int width_bytes;
+    std::unique_ptr<Halide::Approximation> pack;
+    int arity = 1;
+};
+
+// The result of make_block_layout(): the assembled layout Approximation,
+// ready to compose in front of a scheme's lossy quantize/dequantize stage,
+// plus the on-disk block's total byte width -- summed here, once, from the
+// same field list every make_*_scheme() used to hand-sum separately at its
+// own Generator call site.
+struct BlockLayout {
+    std::unique_ptr<Halide::Approximation> layout;
+    int bytes;
+};
+
+// The plain overload, for the rare call site (make_codebook_scheme) that
+// must build its field list conditionally rather than write it out literally.
+inline BlockLayout make_block_layout(std::vector<FieldSpec> fields) {
+    using namespace Halide;
+
+    int bytes = 0;
+    std::vector<int> widths, permutation;
+    std::vector<FieldSpec *> leaders;
+    widths.reserve(fields.size());
+    permutation.reserve(fields.size());
+
+    for (FieldSpec &f : fields) {
+        permutation.push_back(f.slot);
+        widths.push_back(f.width_bytes);
+        bytes += f.width_bytes;
+        if (f.pack) {
+            leaders.push_back(&f);
+        }
+    }
+
+    std::sort(leaders.begin(), leaders.end(),
+              [](const FieldSpec *a, const FieldSpec *b) { return a->slot > b->slot; });
+
+    // StructPack first (outermost -- closest to the on-disk bytes), then
+    // Permute (byte order <-> slot order), then each group's Apply -- see
+    // StructPack's doc comment for why the Permute is needed at all.
+    ComposeBuilder packs;
+    packs.add(StructPack{widths});
+    packs.add(Permute{permutation});
+    for (FieldSpec *f : leaders) {
+        // encode_arity is always 1 (a group's pack consumes one already-
+        // combined logical Func and expands it into `arity` on-disk fields);
+        // decode_arity is `arity` (the mirror: collapse those `arity`
+        // on-disk fields back into the one logical Func).
+        packs.add(Apply{f->slot, /*encode_arity=*/1, /*decode_arity=*/f->arity, std::move(f->pack)});
+    }
+
+    return {packs.build(), bytes};
+}
+
+// Takes each field as its own argument (FieldSpec{...}, FieldSpec{...}, ...),
+// not a single braced list: FieldSpec holds a std::unique_ptr, so it isn't
+// CopyConstructible, and std::initializer_list<T> -- unlike a plain function
+// parameter pack -- requires copying its elements, even into a move-only
+// std::vector<T>. A variadic template sidesteps that entirely (each argument
+// is forwarded, never copied), at the cost of every call site needing to
+// spell out `FieldSpec{...}` instead of a bare `{...}`.
+template<typename... Fields>
+inline BlockLayout make_block_layout(Fields &&...fields) {
+    std::vector<FieldSpec> v;
+    v.reserve(sizeof...(Fields));
+    (v.push_back(std::forward<Fields>(fields)), ...);
+    return make_block_layout(std::move(v));
+}
+
+// ---------------------------------------------------------------------------
+// Shared helpers for the extern-delegated formats.
+// ---------------------------------------------------------------------------
+
+// The extern quantize body, shared by every format whose real quantizer is a
+// named GGML extern (see ggml_extern_quantize.cpp): it computes nothing
+// itself, just names the *_quantize_via_ggml symbol and returns the whole
+// packed byte buffer as a single 2-D uint8 Func. Wrapped as ExternQuantize
+// below and used as the *encoder* half of a Halide::TrustedInverse, whose
+// decoder half is the Compose that unpacks and dequantizes those bytes.
+inline Halide::EncodeResult extern_quantize_blocks(std::vector<Halide::Func> inputs,
+                                                   const std::string &extern_name) {
+    using namespace Halide;
+    Func flat = inputs[0];
+    Func blocks(extern_name + "_blocks");
+    std::vector<ExternFuncArgument> args = {flat};
+    blocks.define_extern(extern_name, args, UInt(8), 2, NameMangling::C);
+    return {{blocks}, {}};
+}
+
+// Decode the 2-byte little-endian fp16 delta stored at bytes(offset)/
+// bytes(offset+1). Returns the delta as an Expr in `blk`. Same bit twiddling
+// as Fp16Pack::decode, written inline here because the grid leaves below read
+// their delta out of a raw byte buffer at a fixed offset rather than through a
+// composed Fp16Pack stage.
+inline Halide::Expr fp16_delta(Halide::Func bytes, int offset, Halide::Var blk) {
+    using namespace Halide;
+    return cast<float>(reinterpret<float16_t>(cast<uint16_t>(le_uint(bytes, offset, blk, 2))));
+}
+
+// (grid(idx) >> (j*8)) & 0xff -- the byte-within-grid-entry extraction every
+// grid leaf below does, whether the grid buffer's entries are 64-bit
+// (iq2s_grid/iq2xs_grid/iq1s_grid) or 32-bit (iq3xxs_grid/iq3s_grid/
+// iq2xxs_grid). Templated on the grid's element type so one function covers
+// both widths; `j` indexes the byte within the (8- or 4-byte) grid entry.
+template<typename T>
+inline Halide::Expr grid_byte(Halide::Buffer<T> grid, Halide::Expr idx, Halide::Expr j) {
+    using namespace Halide;
+    Expr grid_val = grid(idx);
+    return cast<uint8_t>((grid_val >> (cast<T>(j) * 8)) & 0xff);
+}
+
+// select(bit, -1.0f, 1.0f) -- the sign-bit-to-multiplier idiom every grid
+// leaf's final dequantize multiply uses.
+inline Halide::Expr sign_select(Halide::Expr bit) {
+    using namespace Halide;
+    return select(bit, -1.0f, 1.0f);
+}
+
+// The ksigns_iq2xs indirection + bit test shared by IQ3_XXS/IQ2_XS/IQ2_XXS:
+// look `sign_idx` up in the 128-entry ksigns table, then test bit `j` of the
+// looked-up byte.
+inline Halide::Expr ksigns_sign(Halide::Buffer<uint8_t> ksigns, Halide::Expr sign_idx, Halide::Expr j) {
+    using namespace Halide;
+    Expr signs = ksigns(sign_idx);
+    return (cast<uint32_t>(signs) & (cast<uint32_t>(1) << j)) != 0;
+}
+
+// select(is_high, byte >> 4, byte & 0x0f) -- the low/high-nibble-of-a-byte
+// idiom used throughout the K-quant scale unpackers and the grid/repack
+// leaves alike.
+inline Halide::Expr nibble_of(Halide::Expr byte_expr, Halide::Expr is_high) {
+    using namespace Halide;
+    return select(is_high, byte_expr >> 4, byte_expr & 0x0f);
+}
+
+// Copy one of iq_grids_data.h's static constant codebook tables into a named
+// Halide::Buffer the grid classes below index into.
+template<typename T>
+inline Halide::Buffer<T> make_grid_buffer(const T *data, int n, const char *name) {
+    Halide::Buffer<T> buf(n, name);
+    for (int i = 0; i < n; i++) {
+        buf(i) = data[i];
+    }
+    return buf;
+}
+
+template<size_t N>
+inline Halide::Buffer<int8_t> make_static_codebook(const int8_t (&values)[N], const char *name) {
+    return Halide::Buffer<int8_t>(const_cast<int8_t *>(values), (int)N, name);
+}
+
+// ---------------------------------------------------------------------------
+// 4. Extern-delegated quantize + decode-only dequantize-math leaves.
+// ---------------------------------------------------------------------------
+//
+// The formats below (codebook, K-quant, IQ grid, IQ4_XS) all share one shape:
+// their forward map (quantize) is an opaque offline black box -- a per-block
+// nearest-codeword search, an iterative error-minimizing scale fit, a
+// transcendental scale derivation -- that no composition of Halide Funcs
+// reproduces bit-for-bit, so it is delegated to a named GGML extern (see
+// ggml_extern_quantize.cpp). Their reverse map (dequantize) IS an ordinary,
+// bit-exact composition of invertible primitives. Halide::TrustedInverse
+// pairs the two: ExternQuantize (encode()) as the encoder half, a plain
+// Compose (decode()) as the decoder half. The leaves here are the pieces of
+// that decoder that aren't already covered by the packing/reshape components
+// in sections 1-3 -- the codebook lookup and the scale-multiply math. Each is
+// decode-only: its encode() is exactly the opaque forward map deferred to the
+// extern, so it is never called (it only ever lives inside a TrustedInverse's
+// decoder) and traps if it somehow is.
+
+// The encoder half of every extern-delegated format's TrustedInverse: encode()
+// delegates to the named GGML extern (extern_quantize_blocks), producing the
+// whole packed byte buffer as one 2-D uint8 Func. decode() is never called --
+// the TrustedInverse's decoder half owns dequantize.
+class ExternQuantize : public Halide::Approximation {
+public:
+    explicit ExternQuantize(std::string extern_name)
+        : extern_name_(std::move(extern_name)) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        return extern_quantize_blocks(std::move(inputs), extern_name_);
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func>) override {
+        _halide_user_error << "ExternQuantize::decode is never valid -- it is only "
+                              "the encoder half of a TrustedInverse.\n";
+        return {};
+    }
+
+private:
+    std::string extern_name_;
+};
+
+// The encoder half for formats that have NO forward map at all (the IQ1/IQ2
+// importance-matrix-only quantizers: GGML exposes no *_quantize_via_ggml for
+// them). encode() produces a correctly-shaped `blocks(byte, blk)` uint8 Func
+// so that Func::approximate_by() -- which always builds encode() before
+// decode() -- can splice the round trip; the value is a placeholder (0),
+// because Pipeline::compute_offline() always severs this encode and binds the
+// real already-quantized Input in its place, so it is never computed. This is
+// what lets a decode-only format still go through the standard
+// approximate_by/compute_offline path (exercising the framework) instead of a
+// bespoke direct-decode generator. decode() traps -- the paired decoder half
+// of the TrustedInverse owns dequantize.
+class SeveredEncode : public Halide::Approximation {
+public:
+    // `dims` is the dimensionality of the packed buffer this stands in for: 2
+    // for a plain (byte, blk) codec, 3 for a repack weight buffer
+    // (byte, k-block, col-group).
+    explicit SeveredEncode(int block_bytes, int dims = 2)
+        : block_bytes_(block_bytes), dims_(dims) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        using namespace Halide;
+        std::vector<Var> args;
+        for (int d = 0; d < dims_; d++) {
+            args.push_back(Var("se" + std::to_string(d)));
+        }
+        Func blocks("severed_encode_blocks");
+        blocks(args) = cast<uint8_t>(0);
+        return {{blocks}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func>) override {
+        _halide_user_error << "SeveredEncode::decode is never valid -- it is only "
+                              "the (always-severed) encoder half of a TrustedInverse.\n";
+        return {};
+    }
+
+private:
+    int block_bytes_, dims_;
+};
+
+// codes(kk, blk) -> table[codes], a fixed int8 codebook lookup -- the shared
+// codes->value step of every codebook-quantized format (IQ4_NL, MXFP4, TQ1_0,
+// TQ2_0, NVFP4, IQ4_XS). Apply'd on the codes field between unpacking and the
+// scale multiply, so LinearDequant sees the looked-up value
+// in place of a raw integer code. `table` is a Buffer over `static const`
+// backing data (matching every per-format lookup_*() helper's idiom), copied
+// around as a lightweight handle. encode() is the nearest-codeword search
+// deferred to the extern, so it never runs.
+class Codebook : public Halide::Approximation {
+public:
+    explicit Codebook(Halide::Buffer<int8_t> table)
+        : table_(table) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "Codebook::encode is never valid -- the forward "
+                              "codeword search is deferred to an ExternQuantize.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func codes = encoded[0];
+        Var kk("kk"), blk("blk");
+        Buffer<int8_t> table = table_;
+        // Clamp the index to the table's own extent -- a no-op on valid codes
+        // (unpackers produce in-range indices), but it gives Halide's bounds
+        // inference a provable index range across this Func boundary, instead
+        // of falling back to the full int8 range (accessing table at -128).
+        // The grid leaves clamp their grid index the same way.
+        // Dimension-general (pure) decode: the Halide::_ placeholder carries
+        // any extra trailing "lane" dims (e.g. a matmul weight's column dims)
+        // through untouched; with zero trailing dims it collapses to the
+        // familiar (kk, blk). See the DESIGN NOTE by Reblock.
+        Func values("codebook_values");
+        values(kk, blk, _) = table(clamp(cast<int32_t>(codes(kk, blk, _)), 0, table.dim(0).extent() - 1));
+        return {{values}, {}};
+    }
+
+private:
+    Halide::Buffer<int8_t> table_;
+};
+
+// The one decode-only linear dequantize behind every extern-delegated
+// format, unifying what used to be two separate leaves (ScaleDequant and
+// TwoLevelScaleDequant):
+//
+//   - One-level (has_super_d = false, always has_min = false): the flat
+//     codebook formats' cast<float>(codes) * scale. Inputs {codes, scale}.
+//     `sub_size` selects the scale's indexing: 0 means one scale for the
+//     whole block, a Func with NO sub dimension (scale(blk, _), the shape
+//     Fp16Pack/F32Pack/E8M0Pack decode to -- IQ4_NL/MXFP4/TQ*); > 0 means
+//     one scale per `sub_size`-element sub-block, indexed
+//     scale(kk / sub_size, blk, _) (NVFP4's per-sub-block UE4M3 bytes).
+//     This is SymmetricAffineQuantize::decode generalized with a sub-block
+//     scale index; the native symmetric formats keep their own invertible
+//     SymmetricAffineQuantize, so this is decode-only.
+//
+//   - Two-level (has_super_d = true): the K-quant / IQ4_XS dequantize -- a
+//     super-block-wide float `d` (and, for the affine K-quants, `dmin`)
+//     times a per-sub-block scale (and min). Inputs, in order:
+//       has_min:  {d, dmin, scale_min, codes}
+//       no min:   {d, scale, codes}
+//     When has_min, scale and min arrive *combined* in one func
+//     scale_min(plane, sub, ...) -- plane 0 = scale, plane 1 = min -- the
+//     shape PlanarBitPack's plane-axis mode and K4ScaleMinPack both produce,
+//     so a single field carries both halves of the affine per-sub-block
+//     parameters (no separate `min` slot to thread). `codes` may be raw
+//     integer codes (K-quants) or already-looked-up codebook values
+//     (IQ4_XS, via a Codebook stage).
+//
+// The two branches keep their exact original float expression shapes
+// (multiplication order matters for bit-exactness against GGML's reference
+// dequantizers) -- this class only merges the leaves, not the arithmetic.
+// Dimension-general via Halide::_: any trailing "lane" dims (a matmul
+// weight's columns) ride through untouched; zero of them collapses to the
+// familiar (kk, blk). See the DESIGN NOTE by Reblock.
+class LinearDequant : public Halide::Approximation {
+public:
+    LinearDequant(int sub_size, bool has_super_d, bool has_min)
+        : sub_size_(sub_size), has_super_d_(has_super_d), has_min_(has_min) {
+        _halide_user_assert(has_super_d || !has_min)
+            << "LinearDequant: has_min requires has_super_d (no one-level affine format exists).\n";
+        _halide_user_assert(!has_super_d || sub_size > 0)
+            << "LinearDequant: two-level mode always has a per-sub-block scale.\n";
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "LinearDequant::encode is never valid -- the forward "
+                              "quantize is deferred to an ExternQuantize.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Var kk("kk"), blk("blk");
+        Func dequantized("linear_dequantized");
+        if (!has_super_d_) {
+            Func codes = encoded[0], scale = encoded[1];
+            // cast<float> straight off the int8 codes (no int32 detour): the
+            // extra cast<int32_t> would break change_type()'s
+            // cast<f32>(int8)*cast<f32>(int8) SDOT pattern, forcing the vec_dot/
+            // repack inner product into a float16 multiply instead of an int8
+            // dot. See sdot_schedule.h.
+            if (sub_size_ == 0) {
+                dequantized(kk, blk, _) = cast<float>(codes(kk, blk, _)) * scale(blk, _);
+            } else {
+                dequantized(kk, blk, _) = cast<float>(codes(kk, blk, _)) * scale(kk / sub_size_, blk, _);
+            }
+        } else if (has_min_) {
+            Func d = encoded[0], dmin = encoded[1], scale_min = encoded[2], codes = encoded[3];
+            Expr sub = kk / sub_size_;
+            dequantized(kk, blk, _) = d(blk, _) * cast<float>(scale_min(0, sub, blk, _)) * cast<float>(codes(kk, blk, _)) -
+                                      dmin(blk, _) * cast<float>(scale_min(1, sub, blk, _));
+        } else {
+            Func d = encoded[0], scale = encoded[1], codes = encoded[2];
+            dequantized(kk, blk, _) = d(blk, _) * cast<float>(scale(kk / sub_size_, blk, _)) * cast<float>(codes(kk, blk, _));
+        }
+        return {{dequantized}, {}};
+    }
+
+private:
+    int sub_size_;
+    bool has_super_d_, has_min_;
+};
+
+// ---------------------------------------------------------------------------
+// 5. K-quants: combined-bit codes and per-sub-block (scale, min) packing.
+// ---------------------------------------------------------------------------
+
+// The invertible arithmetic behind GGML's K-quant "combined bit" codes: a
+// wider-than-one-field code split into a low part and a high part, where
+// `code = low + high*high_weight - offset` (verified by hand to collapse
+// Q3_K's/Q5_K's/Q6_K's actual bit-OR reconstruction into one formula -- OR
+// and + agree because the low/high bit ranges never overlap: high_weight is
+// always the low part's own value range). Unlike the extern-delegated leaves
+// in section 4, this is genuinely invertible in both directions, so it
+// composes as an ordinary symmetric stage: decode() combines {low, high} ->
+// code, encode() splits code -> {low, high}. The per-field packing (each part
+// through its own PlanarBitPack/BytePack, then StructPack concatenating them
+// in the format's on-disk order) is the composition around it -- e.g. for
+// Q5_K, Compose{StructPack{{qs, qh}, order}, Apply{low_pack}, Apply{high_pack},
+// CombineBits{...}} -- not this leaf, which is only the split/combine math.
+class CombineBits : public Halide::Approximation {
+public:
+    CombineBits(int high_weight, int offset, bool expanded_high = false)
+        : high_weight_(high_weight), offset_(offset), expanded_high_(expanded_high) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func codes = inputs[0];  // codes(kk, blk), the combined (pre-split) value
+        // `combined` is always >= 0 by construction (offset_ is exactly what
+        // decode() subtracts after reconstructing low + high*weight), so the
+        // %// below don't need to handle negative operands.
+        Expr combined = cast<int32_t>(codes(kk, blk)) + offset_;
+        Func low("combine_bits_low");
+        low(kk, blk) = cast<int8_t>(combined % high_weight_);
+        Func high("combine_bits_high");
+        high(kk, blk) = cast<int8_t>(expanded_high_ ?
+                                         (combined / high_weight_) * high_weight_ - offset_ :
+                                         combined / high_weight_);
+        return {{low, high}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func low = encoded[0], high = encoded[1];
+        Func code("combine_bits_code");
+        code(kk, blk) = cast<int8_t>(expanded_high_ ?
+                                         cast<int32_t>(low(kk, blk)) + cast<int32_t>(high(kk, blk)) :
+                                         (cast<int32_t>(low(kk, blk)) + high_weight_ * cast<int32_t>(high(kk, blk))) - offset_);
+        return {{code}, {}};
+    }
+
+private:
+    int high_weight_, offset_;
+    bool expanded_high_;
+    Halide::Var kk{"kk"}, blk{"blk"};
+};
+
+// (Q2_K's per-sub-block nibble-pair scale/min -- low nibble = scale, high
+// nibble = min -- is no longer a bespoke leaf: it is exactly PlanarBitPack's
+// plane-axis mode, PlanarBitPack{4, 16, 0, /*plane_axis=*/true}, producing
+// the same combined (plane, sub) field K4ScaleMinPack does. See make_q2_k_scheme.)
+
+// decode(bytes(byte_idx, blk), 12 bytes) -> scale_min(plane, sub, blk) for sub
+// in [0, 8), plane 0 = scale / plane 1 = min -- GGML's get_scale_min_k4 scheme,
+// shared by Q4_K and Q5_K: for sub<4, scale/min are simply the low 6 bits of
+// byte[sub]/byte[sub+4]; for sub>=4, each is a 4-bit low part from byte[sub+4]
+// combined with a 2-bit high part borrowed from the top 2 bits of an
+// earlier byte (byte[sub-4] for scale, byte[sub] for min) -- a
+// bit-interleaved packing that fits 8 six-bit values into 6 bytes' worth of
+// budget instead of 8 (see q4_k_generators.cpp's original header comment for
+// the full derivation). Emits the combined (plane, sub) field LinearDequant
+// consumes. Decode-only: Q4_K/Q5_K quantize is extern-delegated.
+class K4ScaleMinPack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "K4ScaleMinPack is decode-only -- quantize is deferred to an ExternQuantize.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte_idx, blk[, ...]), byte_idx in [0, 12)
+        Var plane("plane"), sub("sub"), blk("blk");
+
+        Expr jj = clamp(sub - 4, 0, 3);
+        Expr sc = select(sub < 4,
+                         bytes(sub, blk, _) & 0x3f,
+                         cast<uint8_t>((bytes(8 + jj, blk, _) & 0x0f) | ((bytes(jj, blk, _) >> 6) << 4)));
+        Expr m = select(sub < 4,
+                        bytes(sub + 4, blk, _) & 0x3f,
+                        cast<uint8_t>((bytes(8 + jj, blk, _) >> 4) | ((bytes(4 + jj, blk, _) >> 6) << 4)));
+
+        Func scale_min("k4_scale_min_pack");
+        scale_min(plane, sub, blk, _) = cast<uint8_t>(select(plane == 0, sc, m));
+        return {{scale_min}, {}};
+    }
+};
+
+// decode(bytes(byte_idx, blk), 12 bytes) -> scale(sub, blk) for sub in
+// [0, 16) -- Q3_K's 16 SIGNED 6-bit scale values (no min field), a
+// different bit-interleaving than get_scale_min_k4 above: the 2 high bits
+// always live in byte (sub%4)+8, at bit-shift 2*(sub/4); the 4 low bits
+// live in byte (sub%8), taken from the byte's low nibble if sub<8 or high
+// nibble if sub>=8. The final signed value is (low|(high<<4)) - 32 (see
+// q3_k_generators.cpp's original header comment for the full derivation).
+// Decode-only: Q3_K quantize is extern-delegated.
+class Q3KScalePack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "Q3KScalePack is decode-only -- quantize is deferred to an ExternQuantize.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte_idx, blk[, ...]), byte_idx in [0, 12)
+        Var sub("sub"), blk("blk");
+
+        // Dimension-general via Halide::_, matching the other scale unpackers.
+        Expr low_byte_idx = sub % 8;
+        Expr use_high_nibble = sub >= 8;
+        Expr low_byte = bytes(low_byte_idx, blk, _);
+        Expr low_val = cast<int32_t>(nibble_of(low_byte, use_high_nibble));
+        Expr high_byte_idx = (sub % 4) + 8;
+        Expr high_shift = (sub / 4) * 2;
+        Expr high = cast<int32_t>((bytes(high_byte_idx, blk, _) >> high_shift) & 0x3);
+
+        Func scale("q3k_scale_pack_scale");
+        scale(sub, blk, _) = cast<int8_t>((low_val | (high << 4)) - 32);
+        return {{scale}, {}};
+    }
+};
+
+// ---------------------------------------------------------------------------
+// 6. IQ2/IQ3 grid+sign codebook dequantize.
+// ---------------------------------------------------------------------------
+//
+// Unlike IQ4_NL/MXFP4/TQ1_0/TQ2_0/NVFP4 above, these codebooks map one index
+// to a whole *group* of 4 or 8 signed output bytes at once (GGML's published
+// iq2s_grid/iq3xxs_grid/iq3s_grid tables, embedded verbatim from
+// iq_grids_data.h), and each format combines its grid index, sign bits, and
+// per-group scale via its own distinct bit layout -- there's no shared
+// sub-formula across formats the way PlanarBitPack's instances turned out to
+// be for the K-quants. Rather than force an
+// artificial shared abstraction over 3 genuinely different bit layouts, each
+// format below is its own small, decode-only Approximation leaf, wrapped by
+// its make_*_scheme() factory in a TrustedInverse{ExternQuantize, Compose{...,
+// BlockReshape}} (GGML's own reference quantizer for these runs a per-block
+// codebook search -- see ggml_extern_quantize.cpp). decode() is a mechanical,
+// verified-unchanged transcription of iq2_s_generators.cpp's/
+// iq3_xxs_generators.cpp's/iq3_s_generators.cpp's own (already bit-exact)
+// dequantize math, just reading from a `bytes(byte, blk)` Func instead of
+// an `Input<Buffer<uint8_t, 2>>` directly, and producing block-indexed values
+// (the composed BlockReshape does the flat<->block reshape) instead of a flat
+// row itself. encode() traps: quantize is the ExternQuantize's job.
+
+// IQ2_S: 256-element superblock, 8 groups of 32 elements, grid index = an 8-
+// bit qs byte plus 2 extra high bits from a per-group qh byte (1024-entry,
+// 64-bit iq2s_grid, 8 output bytes/index); signs stored directly (no
+// ksigns_iq2xs indirection); scale a nibble byte array (2 groups/byte) via
+// `d*(0.5+nibble)*0.25` -- {fp16 d; qs[32]; signs[32]; qh[8]; scales[8];},
+// 82 bytes.
+class IQ2SGridDequantize : public Halide::Approximation {
+public:
+    IQ2SGridDequantize()
+        : grid_(make_grid_buffer(iq_grids::iq2s_grid, 1024, "iq2s_grid")) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "IQ2SGridDequantize is decode-only -- quantize is "
+                              "deferred to an ExternQuantize via TrustedInverse.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte, blk), byte in [0, 82)
+        // Superblock structure recovered by the composed BlockReshape({8, 4, 8}):
+        // j (element in [0,8)), l ([0,4)), ib32 (group in [0,8)).
+        Var j("j"), l("l"), ib32("ib32"), blk("blk");
+
+        constexpr int kQsOffset = 2;
+        constexpr int kSignsOffset = kQsOffset + 256 / 8;    // 34
+        constexpr int kQhOffset = kSignsOffset + 256 / 8;    // 66
+        constexpr int kScalesOffset = kQhOffset + 256 / 32;  // 74
+
+        Expr qs_l = bytes(kQsOffset + ib32 * 4 + l, blk);
+        Expr qh_byte = cast<uint32_t>(bytes(kQhOffset + ib32, blk));
+        Expr extra_bits = mux(l, {(qh_byte << 8) & 0x300,
+                                  (qh_byte << 6) & 0x300,
+                                  (qh_byte << 4) & 0x300,
+                                  (qh_byte << 2) & 0x300});
+        Expr grid_idx_raw = cast<uint32_t>(qs_l) + extra_bits;
+        Expr grid_idx = clamp(cast<int32_t>(cast<uint16_t>(grid_idx_raw)), 0, 1023);
+
+        Expr signs_byte = bytes(kSignsOffset + ib32 * 4 + l, blk);
+
+        Expr scales_byte = bytes(kScalesOffset + ib32, blk);
+        Expr nibble = nibble_of(scales_byte, l >= 2);
+
+        Expr d = fp16_delta(bytes, 0, blk);
+        Expr db = d * (0.5f + cast<float>(nibble)) * 0.25f;
+
+        Expr gbyte = grid_byte(grid_, grid_idx, j);
+        Expr sign_bit = (cast<uint32_t>(signs_byte) & (cast<uint32_t>(1) << j)) != 0;
+
+        Func dequantized("iq2s_grid_dequantized");
+        dequantized(j, l, ib32, blk) = db * cast<float>(gbyte) * sign_select(sign_bit);
+
+        return {{dequantized}, {}};
+    }
+
+private:
+    Halide::Buffer<uint64_t> grid_;
+};
+
+// IQ3_XXS: 256-element superblock, 8 groups of 32 elements, TWO grid indices
+// per l (plain 8-bit qs bytes, no extra bits; 256-entry, 32-bit
+// iq3xxs_grid, 4 output bytes/index); signs via the same ksigns_iq2xs
+// indirection as IQ2_XXS (a 7-bit sign_idx and a 4-bit scale exponent both
+// bit-packed into one little-endian uint32 "aux32" read from the scales-
+// and-signs field); scale via `d*(0.5+exp)*0.5` -- {fp16 d; qs[64];
+// scales_and_signs[32];}, 98 bytes.
+class IQ3XXSGridDequantize : public Halide::Approximation {
+public:
+    IQ3XXSGridDequantize()
+        : grid_(make_grid_buffer(iq_grids::iq3xxs_grid, 256, "iq3xxs_grid")),
+          ksigns_(make_grid_buffer(iq_grids::ksigns_iq2xs, 128, "ksigns_iq2xs")) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "IQ3XXSGridDequantize is decode-only -- quantize is "
+                              "deferred to an ExternQuantize via TrustedInverse.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte, blk), byte in [0, 98)
+        // Superblock structure recovered by the composed BlockReshape({8, 4, 8}):
+        // j8 (element in [0,8)), l ([0,4)), ib32 (group in [0,8)).
+        Var j8("j8"), l("l"), ib32("ib32"), blk("blk");
+        Expr j4 = j8 % 4;    // byte within the 4-byte grid entry
+        Expr half = j8 / 4;  // 0 (grid1) or 1 (grid2)
+
+        constexpr int kQsOffset = 2;
+        constexpr int kScalesSignsOffset = kQsOffset + 256 / 4;  // 66
+
+        Expr grid_qs_idx = ib32 * 8 + l * 2 + half;
+        Expr grid_idx = bytes(kQsOffset + grid_qs_idx, blk);
+
+        Expr aux32 = le_u32(bytes, kScalesSignsOffset + ib32 * 4, blk);
+
+        Expr d = fp16_delta(bytes, 0, blk);
+        Expr db = d * (0.5f + cast<float>(aux32 >> 28)) * 0.5f;
+
+        Expr sign_idx = cast<uint8_t>((aux32 >> (cast<uint32_t>(l) * 7)) & 127);
+        Expr sign_bit = ksigns_sign(ksigns_, sign_idx, j8);
+        Expr gbyte = grid_byte(grid_, grid_idx, j4);
+
+        Func dequantized("iq3xxs_grid_dequantized");
+        dequantized(j8, l, ib32, blk) = db * cast<float>(gbyte) * sign_select(sign_bit);
+
+        return {{dequantized}, {}};
+    }
+
+private:
+    Halide::Buffer<uint32_t> grid_;
+    Halide::Buffer<uint8_t> ksigns_;
+};
+
+// IQ3_S: 256-element superblock, 8 groups of 32 elements, TWO grid indices
+// per l (an 8-bit qs byte plus 1 extra high bit from a per-group qh byte,
+// combined into a 9-bit index; 512-entry, 32-bit iq3s_grid, 4 output
+// bytes/index); signs stored directly (no ksigns_iq2xs indirection, unlike
+// IQ2_XXS/IQ3_XXS); scale a nibble byte array (one byte per *pair* of
+// groups) via `d*(1+2*nibble)` (odd integers 1,3,...,31 -- not the
+// "0.5+x*k" formula every other type here uses) -- {fp16 d; qs[64]; qh[8];
+// signs[32]; scales[4];}, 110 bytes.
+class IQ3SGridDequantize : public Halide::Approximation {
+public:
+    IQ3SGridDequantize()
+        : grid_(make_grid_buffer(iq_grids::iq3s_grid, 512, "iq3s_grid")) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "IQ3SGridDequantize is decode-only -- quantize is "
+                              "deferred to an ExternQuantize via TrustedInverse.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];  // bytes(byte, blk), byte in [0, 110)
+        // Superblock structure recovered by the composed BlockReshape({8, 4, 8}):
+        // j8 (element in [0,8)), l ([0,4)), grp (group in [0,8)).
+        Var j8("j8"), l("l"), grp("grp"), blk("blk");
+        Expr j4 = j8 % 4;    // byte within the 4-byte grid entry
+        Expr half = j8 / 4;  // 0 (first grid index of this l) or 1 (second)
+
+        constexpr int kQsOffset = 2;
+        constexpr int kQhOffset = kQsOffset + 256 / 4;         // 66
+        constexpr int kSignsOffset = kQhOffset + 256 / 32;     // 74
+        constexpr int kScalesOffset = kSignsOffset + 256 / 8;  // 106
+
+        Expr qs_byte = bytes(kQsOffset + grp * 8 + l * 2 + half, blk);
+        Expr qh_byte = cast<uint32_t>(bytes(kQhOffset + grp, blk));
+        Expr bit_pos = l * 2 + half;
+        Expr high_bit = (qh_byte >> cast<uint32_t>(bit_pos)) & 1;
+        Expr grid_idx = clamp(cast<int32_t>(cast<uint16_t>(cast<uint32_t>(qs_byte) + (high_bit << 8))), 0, 511);
+
+        Expr signs_byte = bytes(kSignsOffset + grp * 4 + l, blk);
+        Expr sign_bit = (cast<uint32_t>(signs_byte) & (cast<uint32_t>(1) << j8)) != 0;
+
+        Expr scales_byte = bytes(kScalesOffset + grp / 2, blk);
+        Expr nibble = nibble_of(scales_byte, (grp % 2) != 0);
+
+        Expr d = fp16_delta(bytes, 0, blk);
+        Expr db = d * (1.0f + 2.0f * cast<float>(nibble));
+
+        Expr gbyte = grid_byte(grid_, grid_idx, j4);
+
+        Func dequantized("iq3s_grid_dequantized");
+        dequantized(j8, l, grp, blk) = db * cast<float>(gbyte) * sign_select(sign_bit);
+
+        return {{dequantized}, {}};
+    }
+
+private:
+    Halide::Buffer<uint32_t> grid_;
+};
+
+// The four IQ1/IQ2 importance-matrix-only formats have no from_float extern
+// (GGML exposes no *_quantize_via_ggml), so their make_*_scheme() below pairs
+// this decode leaf with a SeveredEncode via TrustedInverse (a dequantize-only /
+// vec_dot-only round trip; the placeholder encode is always severed). Each is a
+// verified-unchanged transcription of the matching *_generators.cpp dequantize,
+// reading bytes(byte, blk) and emitting the {8,4,8} superblock form
+// (j/j-elem, l, group, blk) so the composed BlockReshape does the reshape.
+
+// IQ2_XS: 74-byte block. qs[32] as 32 little-endian uint16 (4 per group): low
+// 9 bits index the 512-entry uint64 iq2xs_grid, top 7 bits index ksigns_iq2xs;
+// scale a nibble byte array (2 l's/byte) via d*(0.5+nibble)*0.25.
+class IQ2XSGridDequantize : public Halide::Approximation {
+public:
+    IQ2XSGridDequantize()
+        : grid_(make_grid_buffer(iq_grids::iq2xs_grid, 512, "iq2xs_grid")),
+          ksigns_(make_grid_buffer(iq_grids::ksigns_iq2xs, 128, "ksigns_iq2xs")) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "IQ2XSGridDequantize is decode-only.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];
+        Var j("j"), l("l"), ib32("ib32"), blk("blk");
+        constexpr int kQsOffset = 2;
+        constexpr int kScalesOffset = 66;
+
+        Expr qs_idx = ib32 * 4 + l;
+        Expr qs_val = le_u16([&](int i) { return bytes(kQsOffset + qs_idx * 2 + i, blk); });
+        Expr grid_idx = clamp(cast<int32_t>(qs_val & 511), 0, 511);
+        Expr sign_idx = clamp(cast<int32_t>(qs_val >> 9), 0, 127);
+
+        Expr scales_byte = bytes(kScalesOffset + ib32, blk);
+        Expr nibble = nibble_of(scales_byte, l >= 2);
+        Expr db = fp16_delta(bytes, 0, blk) * (0.5f + cast<float>(nibble)) * 0.25f;
+
+        Expr gbyte = grid_byte(grid_, grid_idx, j);
+        Expr sign_bit = ksigns_sign(ksigns_, sign_idx, j);
+
+        Func dequantized("iq2xs_grid_dequantized");
+        dequantized(j, l, ib32, blk) = db * cast<float>(gbyte) * sign_select(sign_bit);
+        return {{dequantized}, {}};
+    }
+
+private:
+    Halide::Buffer<uint64_t> grid_;
+    Halide::Buffer<uint8_t> ksigns_;
+};
+
+// IQ2_XXS: 66-byte block. Per group, an 8-byte window: bytes 0..3 are 4 grid
+// indices (one per l) into the 256-entry uint64 iq2xxs_grid; bytes 4..7 form a
+// uint32 aux32 whose top 4 bits are a scale exponent (d*(0.5+exp)*0.25) and
+// whose low 28 bits pack four 7-bit ksigns indices.
+class IQ2XXSGridDequantize : public Halide::Approximation {
+public:
+    IQ2XXSGridDequantize()
+        : grid_(make_grid_buffer(iq_grids::iq2xxs_grid, 256, "iq2xxs_grid")),
+          ksigns_(make_grid_buffer(iq_grids::ksigns_iq2xs, 128, "ksigns_iq2xs")) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "IQ2XXSGridDequantize is decode-only.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];
+        Var j("j"), l("l"), ib32("ib32"), blk("blk");
+        constexpr int kQsOffset = 2;
+
+        Expr grid_idx = clamp(cast<int32_t>(bytes(kQsOffset + ib32 * 8 + l, blk)), 0, 255);
+        Expr aux32 = le_u32(bytes, kQsOffset + ib32 * 8 + 4, blk);
+        Expr db = fp16_delta(bytes, 0, blk) * (0.5f + cast<float>(aux32 >> 28)) * 0.25f;
+        Expr sign_idx = clamp(cast<int32_t>((aux32 >> (cast<uint32_t>(l) * 7)) & 127), 0, 127);
+
+        Expr sign_bit = ksigns_sign(ksigns_, sign_idx, j);
+        Expr gbyte = grid_byte(grid_, grid_idx, j);
+
+        Func dequantized("iq2xxs_grid_dequantized");
+        dequantized(j, l, ib32, blk) = db * cast<float>(gbyte) * sign_select(sign_bit);
+        return {{dequantized}, {}};
+    }
+
+private:
+    Halide::Buffer<uint64_t> grid_;
+    Halide::Buffer<uint8_t> ksigns_;
+};
+
+// IQ1_S: 50-byte block. qs[32] low grid-index bytes (one per l); qh[8] as 8
+// uint16 (one per group): 3 bits/l give the grid index's high 3 bits, bits
+// 12..14 a per-group scale (dl = d*(2*s+1)), bit 15 selects a +/-IQ1S_DELTA
+// added to every value. iq1s_grid entries are SIGNED bytes used directly.
+class IQ1SGridDequantize : public Halide::Approximation {
+public:
+    IQ1SGridDequantize()
+        : grid_(make_grid_buffer(iq_grids::iq1s_grid, 2048, "iq1s_grid")) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "IQ1SGridDequantize is decode-only.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];
+        Var j("j"), l("l"), ib("ib"), blk("blk");
+        constexpr float kIQ1S_DELTA = 0.125f;
+        constexpr int kQsOffset = 2;
+        constexpr int kQhOffset = 34;
+
+        Expr qh_val = le_u16([&](int i) { return bytes(kQhOffset + ib * 2 + i, blk); });
+        Expr dl_scale = cast<int32_t>((qh_val >> 12) & 7);
+        Expr dl = fp16_delta(bytes, 0, blk) * cast<float>(2 * dl_scale + 1);
+        Expr delta = select((qh_val & 0x8000) != 0, -kIQ1S_DELTA, kIQ1S_DELTA);
+
+        Expr qs_byte = bytes(kQsOffset + ib * 4 + l, blk);
+        Expr high3 = cast<uint32_t>((qh_val >> (cast<uint16_t>(l) * 3)) & 7);
+        Expr grid_idx = clamp(cast<int32_t>(cast<uint16_t>(cast<uint32_t>(qs_byte) + (high3 << 8))), 0, 2047);
+
+        Expr grid_signed = reinterpret<int8_t>(grid_byte(grid_, grid_idx, j));
+
+        Func dequantized("iq1s_grid_dequantized");
+        dequantized(j, l, ib, blk) = dl * (cast<float>(grid_signed) + delta);
+        return {{dequantized}, {}};
+    }
+
+private:
+    Halide::Buffer<uint64_t> grid_;
+};
+
+// IQ1_M: 56-byte block, NO separate delta field. qs[32] low index bytes;
+// qh[16] (2/group) give a high-3-bit grid extension + a sign-delta bit per l2;
+// scales[8] as 4 uint16: the block's shared fp16 d is bit-gathered from the
+// top nibble of all 4 words, each word's low 12 bits holding two 3-bit
+// per-group scales. Signed codebook + /-IQ1S_DELTA, same as IQ1_S.
+class IQ1MGridDequantize : public Halide::Approximation {
+public:
+    IQ1MGridDequantize()
+        : grid_(make_grid_buffer(iq_grids::iq1s_grid, 2048, "iq1s_grid")) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "IQ1MGridDequantize is decode-only.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func bytes = encoded[0];
+        Var j("j"), l2("l2"), ib("ib"), blk("blk");
+        constexpr float kIQ1S_DELTA = 0.125f;
+        constexpr int kQsOffset = 0;
+        constexpr int kQhOffset = 32;
+        constexpr int kScalesOffset = 48;
+
+        auto sc_word = [&](int k) -> Expr {
+            return le_u16([&](int i) { return bytes(kScalesOffset + k * 2 + i, blk); });
+        };
+        Expr sc0 = sc_word(0), sc1 = sc_word(1), sc2 = sc_word(2), sc3 = sc_word(3);
+        Expr d_bits = (sc0 >> 12) | ((sc1 >> 8) & 0xf0) | ((sc2 >> 4) & 0xf00) | (sc3 & 0xf000);
+        Expr d = cast<float>(reinterpret<float16_t>(cast<uint16_t>(d_bits)));
+
+        Expr qh_half = l2 / 2;
+        Expr parity = l2 % 2;
+        Expr qh_byte = cast<uint32_t>(bytes(kQhOffset + ib * 2 + qh_half, blk));
+        Expr qs_byte = bytes(kQsOffset + ib * 4 + l2, blk);
+        // Constant-amount shifts per parity arm (a variable-amount shift into a
+        // buffer index defeats Halide bounds inference even when masked).
+        Expr extra_bits = select(parity == 0, (qh_byte << 8) & 0x700, (qh_byte << 4) & 0x700);
+        Expr grid_idx = clamp(cast<int32_t>(cast<uint16_t>(cast<uint32_t>(qs_byte) + extra_bits)), 0, 2047);
+
+        Expr delta_mask = select(parity == 0, cast<uint32_t>(0x08), cast<uint32_t>(0x80));
+        Expr delta = select((qh_byte & delta_mask) != 0, -kIQ1S_DELTA, kIQ1S_DELTA);
+
+        Expr sc_idx = ib / 2;
+        Expr sc_word_val = mux(sc_idx, {sc0, sc1, sc2, sc3});
+        Expr shift = (ib % 2) * 6 + qh_half * 3;
+        Expr scale3 = (sc_word_val >> cast<uint32_t>(shift)) & 7;
+        Expr dl = d * cast<float>(2 * scale3 + 1);
+
+        Expr grid_signed = reinterpret<int8_t>(grid_byte(grid_, grid_idx, j));
+
+        Func dequantized("iq1m_grid_dequantized");
+        dequantized(j, l2, ib, blk) = dl * (cast<float>(grid_signed) + delta);
+        return {{dequantized}, {}};
+    }
+
+private:
+    Halide::Buffer<uint64_t> grid_;
+};
+
+// decode({scales_h(2 bytes), scales_l(4 bytes)}) -> scale(sub, blk) for sub in
+// [0, 8) -- IQ4_XS's per-sub-block 6-bit scale `ls`, already minus its 32 bias
+// so it feeds LinearDequant's `d * scale(sub) * value` directly. `ls`
+// is 4 low bits from scales_l (2 sub-blocks/byte) plus 2 high bits from
+// scales_h (a little-endian uint16, 2 bits/sub-block) -- a two-field
+// bit-interleaving, the peer of Q3KScalePack/K4ScaleMinPack but reading two
+// separate byte fields. Decode-only: IQ4_XS quantize is extern-delegated.
+class IQ4XSScalePack : public Halide::Approximation {
+public:
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "IQ4XSScalePack::encode is never valid -- IQ4_XS "
+                              "quantize is deferred to an ExternQuantize.\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func scales_h = encoded[0], scales_l = encoded[1];
+        Var sub("sub"), blk("blk");
+
+        // Dimension-general via Halide::_, matching the other scale unpackers.
+        // ls = (scales_l[sub/2] >> 4*(sub%2)) & 0xf | ((scales_h >> 2*sub) & 3) << 4
+        Expr low4 = cast<int32_t>(nibble_of(scales_l(sub / 2, blk, _), (sub % 2) != 0));
+        Expr sh = le_u16([&](int i) { return scales_h(i, blk, _); });
+        Expr high2 = cast<int32_t>((sh >> cast<uint16_t>(sub * 2)) & 3);
+        Expr ls = low4 | (high2 << 4);
+
+        Func scale("iq4xs_scale");
+        scale(sub, blk, _) = cast<int8_t>(ls - 32);
+        return {{scale}, {}};
+    }
+};
+
+// ---------------------------------------------------------------------------
+// 7. Repack: interleaved multi-row activation layout (block_q8_0x4 / q8_Kx4).
+//
+// These are repack-specific instances of the deferred general block-relayout
+// (see the DESIGN NOTE by Reblock): "block-layout change prior to applying the
+// same lossy quantization Approximations". The 4 interleaved rows are folded
+// into the block index blk = ib*4 + row, so the existing (kk, blk)
+// SymmetricAffineQuantize + BytePack + Fp16Pack run unchanged; only the two
+// relayouts here (input row-blocking and output interleave/header assembly)
+// are new. n_rows is fixed at 4 (repack's "x4").
+// ---------------------------------------------------------------------------
+
+// Losslessly re-view a 2-D activation x(col, row in [0,n_rows)) as
+// block-indexed block(kk in [0, block_size), blk = ib*n_rows + row), where ib
+// is the k-block. The per-row scale then falls out of the quantizer as a
+// per-blk scale. `n_rows` is 4 for every current caller (repack's "x4"), but
+// isn't hardcoded here -- callers pass it explicitly.
+class RepackRowReshape : public Halide::Approximation {
+public:
+    RepackRowReshape(int block_size, int n_rows)
+        : block_size_(block_size), n_rows_(n_rows) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func x = inputs[0];  // x(col, row)
+        Var kk("kk"), blk("blk");
+        Func block("repack_row_block");
+        block(kk, blk) = x((blk / n_rows_) * block_size_ + kk, blk % n_rows_);
+        return {{block}, {}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func block = encoded[0];  // block(kk, blk)
+        Var col("col"), row("row");
+        Func x("repack_row_x");
+        x(col, row) = block(col % block_size_, (col / block_size_) * n_rows_ + row);
+        return {{x}, {}};
+    }
+
+private:
+    int block_size_, n_rows_;
+};
+
+// Assemble one interleaved output block (byte, ib) from the per-(row) packed
+// code bytes and scale bytes of the 4 rows blk = ib*4 + row. Header: 4 deltas
+// (`delta_bytes` each, row order). Payload: codes interleaved in groups of
+// `blocklen` -- payload position jj -> (row = (jj % (4*bl))/bl,
+// kk = (jj/(4*bl))*bl + jj%bl), matching GGML's src_id/src_offset.
+class RepackInterleavePack : public Halide::Approximation {
+public:
+    RepackInterleavePack(int block_size, int blocklen, int delta_bytes, bool with_bsums = false)
+        : block_size_(block_size), blocklen_(blocklen), delta_bytes_(delta_bytes), with_bsums_(with_bsums) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func> inputs) override {
+        using namespace Halide;
+        Func code = inputs[0], scale = inputs[1];
+        Var byte("byte"), ib("ib");
+        int header = 4 * delta_bytes_;
+        int payload_end = header + block_size_ * 4;
+
+        Expr row_d = clamp(byte / delta_bytes_, 0, 3);
+        Expr delta_byte = scale(clamp(byte % delta_bytes_, 0, delta_bytes_ - 1), ib * 4 + row_d);
+
+        Expr jj = clamp(byte - header, 0, block_size_ * 4 - 1);
+        Expr row_p = (jj % (4 * blocklen_)) / blocklen_;
+        Expr kk_p = (jj / (4 * blocklen_)) * blocklen_ + (jj % blocklen_);
+        Expr code_byte = code(kk_p, ib * 4 + row_p);
+
+        Func blocks("repack_interleave_blocks");
+        if (!with_bsums_) {
+            blocks(byte, ib) = select(byte < header, delta_byte, code_byte);
+            return {{blocks}, {}};
+        }
+
+        // Q8_K's block_q8_Kx4 appends `bsums`: int16 group-sums of the int8
+        // codes, scattered across rows/groups by GGML's index_q8_k mapping.
+        // Reduce over the interleaved payload order (rj), same as GGML.
+        Var g("g");
+        Func bsums("repack_bsums");
+        RDom rj(0, block_size_ * 4, "rj");
+        Expr rp = (rj % (4 * blocklen_)) / blocklen_;
+        Expr kp = (rj / (4 * blocklen_)) * blocklen_ + (rj % blocklen_);
+        Expr qval = reinterpret<int8_t>(code(kp, ib * 4 + rp));
+        int shift = blocklen_ == 8 ? 3 : 2;  // log2(blocklen)
+        Expr idx = (((rj & (4 * blocklen_ - 1)) >> shift) << 2) + ((rj >> 8) << 4) + ((rj >> 6) & 3);
+        bsums(g, ib) = cast<int32_t>(0);
+        bsums(idx, ib) += cast<int32_t>(qval);
+
+        int nbsum = (block_size_ / 16) * 4;  // 64 groups
+        Expr bsum_rel = clamp(byte - payload_end, 0, nbsum * 2 - 1);
+        Expr bsum_g = bsum_rel / 2;
+        Expr bsum_is_lo = (bsum_rel % 2) == 0;
+        Expr bsum_bits = reinterpret<uint16_t>(cast<int16_t>(bsums(bsum_g, ib)));
+        Expr bsum_byte = cast<uint8_t>(select(bsum_is_lo, bsum_bits & 0xff, (bsum_bits >> 8) & 0xff));
+
+        blocks(byte, ib) = select(byte < header, delta_byte, byte < payload_end, code_byte, bsum_byte);
+        return {{blocks}, {bsums}};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func blocks = encoded[0];
+        Var kk("kk"), blk("blk"), sbyte("sbyte");
+        int header = 4 * delta_bytes_;
+
+        // blk = ib*4 + row: ib = blk/4, row = blk%4.
+        Func code("repack_interleave_code");
+        Expr jj = (kk / blocklen_) * (4 * blocklen_) + (blk % 4) * blocklen_ + (kk % blocklen_);
+        code(kk, blk) = blocks(header + jj, blk / 4);
+        Func scaleb("repack_interleave_scale");
+        scaleb(sbyte, blk) = blocks((blk % 4) * delta_bytes_ + sbyte, blk / 4);
+        return {{code, scaleb}, {}};
+    }
+
+private:
+    int block_size_, blocklen_, delta_bytes_;
+    bool with_bsums_;
+};
+
+// block_q8_0x4 codec: block-layout relayout + the same symmetric Q8_0 quantize
+// (amax/127, round) as make_symmetric_block_scheme, interleaved by `blocklen`.
+inline std::unique_ptr<Halide::Approximation> make_q8_0x4_scheme(int blocklen) {
+    using namespace Halide;
+    return std::make_unique<Compose>(
+        RepackInterleavePack{32, blocklen, /*delta_bytes=*/2},
+        Apply{0, BytePack{}},  // codes -> bytes
+        Apply{1, Fp16Pack{}},  // scale -> fp16 bytes
+        SymmetricAffineQuantize{32, 127, RoundingMode::Nearest, ScaleAnchor::AbsMax},
+        RepackRowReshape{32, /*n_rows=*/4});
+}
+
+// block_q8_Kx4 codec: same shape as make_q8_0x4_scheme but a 256-element block,
+// a float32 delta per row (F32Pack), the same round-to-even -127/max Q8_K
+// quantize as make_q8_k_scheme, and the interleaved bsums field (with_bsums).
+inline std::unique_ptr<Halide::Approximation> make_q8_kx4_scheme(int blocklen) {
+    using namespace Halide;
+    return std::make_unique<Compose>(
+        RepackInterleavePack{256, blocklen, /*delta_bytes=*/4, /*with_bsums=*/true},
+        Apply{0, BytePack{}},  // codes -> bytes
+        Apply{1, F32Pack{}},   // scale -> float32 bytes
+        SymmetricAffineQuantize{256, 127, RoundingMode::NearestEvenClampedHigh,
+                                ScaleAnchor::ExtremeSignedValueTwoStep},
+        RepackRowReshape{256, /*n_rows=*/4});
+}
+
+// ---------------------------------------------------------------------------
+// 8. Repack weight un-interleave (for gemv/gemm): decode the 3-D interleaved
+// weight buffer (byte, k-block, col-group) into per-element codes + per-column
+// scale *bytes* (the composed Fp16/F32/E8M0 pack turns those into the float
+// scale -- no scale decode duplicated in the leaf), carrying the two column
+// dims (col-in-group j, col-group x) as explicit trailing dims so the
+// dimension-general LinearDequant/Codebook/scale packs (Halide::_) run unchanged
+// and the matmul reduces over (kk, blk). Decode-only (the weight is
+// pre-quantized; SeveredEncode is the severed encoder half). This is the
+// decode twin of RepackInterleavePack, for the four "simple" weight families.
+// ---------------------------------------------------------------------------
+enum class RepackWeightCode { SignedByte,    // Q8_0: whole signed int8
+                              SignedNibble,  // Q4_0: two's-complement 4-bit (repack's XOR-0x8)
+                              RawNibble };   // IQ4_NL/MXFP4: raw 4-bit codebook index
+
+class UnInterleaveWeight : public Halide::Approximation {
+public:
+    UnInterleaveWeight(int n_cols, int blocklen, int block_size,
+                       RepackWeightCode code_kind, ScaleFormat scale_kind)
+        : n_cols_(n_cols), blocklen_(blocklen), block_size_(block_size),
+          code_kind_(code_kind), scale_kind_(scale_kind) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "UnInterleaveWeight is decode-only (SeveredEncode is the "
+                              "severed encoder half of its TrustedInverse).\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func blocks = encoded[0];  // blocks(byte, k-block, col-group)
+        Var byte("byte"), kk("kk"), blk("blk"), j("j"), x("x");
+        // Per-column scale header width: fp16 = 2, f32 = 4, e8m0 = 1 byte/column.
+        int scale_stride = scale_width(scale_kind_);
+        int header = scale_stride * n_cols_;
+
+        Func codes("uninterleave_codes");
+        if (code_kind_ == RepackWeightCode::SignedByte) {
+            Expr qs_idx = (kk / blocklen_) * n_cols_ * blocklen_ + j * blocklen_ + (kk % blocklen_);
+            codes(kk, blk, j, x) = reinterpret<int8_t>(blocks(header + qs_idx, blk, x));
+        } else {
+            Expr half = kk / (block_size_ / 2);
+            Expr el = kk % (block_size_ / 2);
+            Expr qs_idx = (el / blocklen_) * n_cols_ * blocklen_ + j * blocklen_ + (el % blocklen_);
+            Expr byte_v = blocks(header + qs_idx, blk, x);
+            Expr nib = cast<int32_t>(nibble_of(byte_v, half != 0));
+            Expr code = code_kind_ == RepackWeightCode::SignedNibble ? select(nib < 8, nib, nib - 16) : nib;
+            codes(kk, blk, j, x) = cast<int8_t>(code);
+        }
+
+        // Pure addressing: gather column j's raw scale-header bytes; the composed
+        // Fp16Pack / F32Pack / E8M0Pack (dimension-general) turns them into the
+        // float scale, exactly as KQuantDeInterleave emits d_bytes for Fp16Pack.
+        Func scale_bytes("uninterleave_scale_bytes");
+        scale_bytes(byte, blk, j, x) = blocks(scale_stride * j + byte, blk, x);
+        return {{codes, scale_bytes}, {}};
+    }
+
+private:
+    int n_cols_, blocklen_, block_size_;
+    RepackWeightCode code_kind_;
+    ScaleFormat scale_kind_;
+};
+
+// Weight-decode scheme for a "simple" repack weight family (Q4_0/Q8_0/IQ4_NL/
+// MXFP4): un-interleave -> [codebook] -> one-level scale. The col dims ride the
+// dimension-general LinearDequant/Codebook via Halide::_. SeveredEncode (3-D)
+// stands in for the pre-quantized weight buffer, severed by the gemv/gemm
+// generator's compute_offline.
+inline std::unique_ptr<Halide::Approximation> make_repack_weight_scheme(
+    int n_cols, int blocklen, int block_bytes, RepackWeightCode code_kind, ScaleFormat scale_kind,
+    Halide::Buffer<int8_t> table = {}, int block_size = 32) {
+    using namespace Halide;
+    // Interpret the scale-header bytes UnInterleaveWeight gathers with the same
+    // packs the plain codecs use -- no per-kind scale decode duplicated here.
+    return std::make_unique<TrustedInverse>(
+        SeveredEncode{block_bytes, 3},
+        Compose{UnInterleaveWeight{n_cols, blocklen, block_size, code_kind, scale_kind},
+                Choose{code_kind == RepackWeightCode::RawNibble,
+                       Apply{0, Codebook{table}}, Identity{}},
+                Apply{1, make_scale_pack(scale_kind)},
+                LinearDequant{/*sub_size=*/0, /*has_super_d=*/false, /*has_min=*/false}});
+}
+
+// K-quant repack weight decode (block_q{4,5,6,2}_Kx8, n_cols=8): a bespoke,
+// verified-unchanged transcription of the repack_gemv_generators.cpp
+// weight_value helpers -- the interleaved 256-element super-block with a
+// two-level scale (and, for Q4_K/Q5_K, get_scale_min_k4 with the column index
+// standing in for the sub-block index). Its decode reads the severed 3-D
+// weight buffer and emits Wt(kk, blk, j, x) directly (kept as one leaf rather
+// than composed from LinearDequant + the scale-min packs, because the
+// interleave is intricate enough that a direct transcription is far less
+// error-prone; the SeveredEncode encoder half is severed by the matmul
+// generator's compute_offline). blocklen is the interleave width (4 or 8).
+enum class KQuantWeightFamily { Q4_K,
+                                Q5_K,
+                                Q6_K,
+                                Q2_K };
+
+// The genuinely repack-specific half of a K-quant weight decode: the *byte
+// addressing* that maps a logical (element kk, sub-block, column j, col-group
+// x) back to the interleaved block's scattered qs/qh/ql bytes and its scale
+// region. It is the K-quant analog of UnInterleaveWeight -- pure permutation,
+// no dequant arithmetic. It emits the same logical field slots the plain
+// K-quant decode's StructPack+packs produce, so the shared downstream
+// Compose (Fp16Pack on the fp16 headers, then LinearDequant, in
+// LOGICAL element order) finishes the job identically:
+//
+//   Q4_K/Q5_K: {d_bytes, dmin_bytes, scale_min, codes}   sub_size 32
+//   Q6_K:      {d_bytes,             scale,     codes}   sub_size 16 (no min)
+//   Q2_K:      {d_bytes, dmin_bytes, scale_min, codes}   sub_size 16
+//
+// (has-min families emit scale and min combined in one scale_min(plane, sub, ...)
+// field, plane 0 = scale / 1 = min -- the same shape K4ScaleMinPack and
+// PlanarBitPack's plane-axis mode produce, so LinearDequant consumes it
+// identically.) scale/min are produced as VALUES here (not bytes) because their
+// bit layouts can't be delegated to the plain packs: get_scale_min_k4 (Q4_K/Q5_K)
+// does its bit-math on what is the *column* index j in the repack while the
+// sub-block rides a separate axis -- an axis transpose K4ScaleMinPack's
+// (sub-first) interface can't express -- and the qs/qh code stream is
+// column-interleaved, so PlanarBitPack's contiguous-window assumption doesn't
+// hold. The scale index reduces to kk/sub_size in logical order for all four families
+// (verified: Q6_K's base_l/base_h and Q2_K's sm_idx both collapse to kk/16
+// since blocklen divides 16), which is exactly what LinearDequant
+// consumes, so no re-order is needed downstream.
+class KQuantDeInterleave : public Halide::Approximation {
+public:
+    KQuantDeInterleave(KQuantWeightFamily family, int blocklen)
+        : family_(family), blocklen_(blocklen) {
+    }
+
+    Halide::EncodeResult encode(std::vector<Halide::Func>) override {
+        _halide_user_error << "KQuantDeInterleave is decode-only (SeveredEncode is the severed encoder).\n";
+        return {};
+    }
+
+    Halide::DecodeResult decode(std::vector<Halide::Func> encoded) override {
+        using namespace Halide;
+        Func b = encoded[0];  // blocks(byte, k-superblock, col-group)
+        Var byte("byte"), kk("kk"), blk("blk"), sub("sub"), plane("plane"), j("j"), x("x");
+        const int bl = blocklen_;
+        const int nc = 8;
+
+        // has-min families (Q4_K/Q5_K/Q2_K) emit scale and min combined in one
+        // field scale_min(plane, sub, ...) (plane 0 = scale, 1 = min), the shape
+        // LinearDequant consumes; Q6_K (no min) emits a plain scale.
+        Func codes("kq_codes"), scale("kq_scale"), scale_min("kq_scale_min"), d_bytes("kq_d"), dmin_bytes("kq_dmin");
+
+        if (family_ == KQuantWeightFamily::Q4_K || family_ == KQuantWeightFamily::Q5_K) {
+            const bool is_q5 = family_ == KQuantWeightFamily::Q5_K;
+            const int kScalesOffset = 32;
+            const int kQsOffset = is_q5 ? 384 : 128;
+            const int kQhOffset = 128;  // Q5_K only
+
+            Expr iter = kk / 64, local64 = kk % 64, half64 = local64 / 32, lpos = local64 % 32;
+            Expr k_inner = lpos / bl, ii = lpos % bl, k = iter * (32 / bl) + k_inner;
+            Expr qs_byte = b(kQsOffset + k * nc * bl + j * bl + ii, blk, x);
+            Expr nibble = cast<int32_t>(nibble_of(qs_byte, half64 != 0));
+            Expr value = nibble;
+            if (is_q5) {
+                Expr qh_byte = b(kQhOffset + k_inner * (bl * nc) + j * bl + ii, blk, x);
+                Expr h_bit = cast<int32_t>((cast<uint32_t>(qh_byte) >> (iter * 2 + half64)) & 1);
+                value = nibble | (h_bit << 4);
+            }
+            codes(kk, blk, j, x) = value;
+
+            // get_scale_min_k4, addressed by window(sub) and bit-position j,
+            // scale and min combined into one (plane, sub) field.
+            Expr window = (sub / 4) * 48 + (sub % 4) * 12;
+            Expr jj = clamp(j - 4, 0, 3);
+            Expr sc = select(j < 4, b(kScalesOffset + window + j, blk, x) & 0x3f,
+                             cast<uint8_t>((b(kScalesOffset + window + 8 + jj, blk, x) & 0x0f) |
+                                           ((b(kScalesOffset + window + jj, blk, x) >> 6) << 4)));
+            Expr mn = select(j < 4, b(kScalesOffset + window + j + 4, blk, x) & 0x3f,
+                             cast<uint8_t>((b(kScalesOffset + window + 8 + jj, blk, x) >> 4) |
+                                           ((b(kScalesOffset + window + 4 + jj, blk, x) >> 6) << 4)));
+            scale_min(plane, sub, blk, j, x) = cast<uint8_t>(select(plane == 0, sc, mn));
+
+            d_bytes(byte, blk, j, x) = b(2 * j + byte, blk, x);
+            dmin_bytes(byte, blk, j, x) = b(16 + 2 * j + byte, blk, x);
+            return {{d_bytes, dmin_bytes, scale_min, codes}, {}};
+        } else if (family_ == KQuantWeightFamily::Q6_K) {
+            const int kScalesOffset = 16, kQlOffset = 144, kQhOffset = 1168;
+            const int blocks_per_half = 64 / bl;
+            constexpr int kQlSize = (256 * 8) / 2, kQhSize = (256 * 8) / 4;
+
+            Expr local128 = kk % 128, is_high = local128 >= 64, pos64 = local128 % 64, i = pos64 % bl;
+            Expr base_l = kk - i - select(is_high, 64, 0), base_h = base_l + 64;
+            Expr k = (base_l / 128) * blocks_per_half + (base_l % 128) / bl;
+            Expr ql_byte = b(kQlOffset + clamp(k * nc * bl + j * bl + i, 0, kQlSize - 1), blk, x);
+            Expr nibble = cast<int32_t>(nibble_of(ql_byte, is_high));
+            Expr qh_shift = select(is_high, ((base_h % 128) / 32) * 2, ((base_l % 128) / 32) * 2);
+            Expr qh_idx_l = (base_l / 128) * 32 + ((base_l + i) % 32);
+            Expr qh_idx_h = (base_h / 128) * 32 + ((base_h + i) % 32);
+            Expr qh_off_l = clamp((qh_idx_l / bl) * (bl * nc) + j * bl + (qh_idx_l % bl), 0, kQhSize - 1);
+            Expr qh_off_h = clamp((qh_idx_h / bl) * (bl * nc) + j * bl + (qh_idx_h % bl), 0, kQhSize - 1);
+            Expr qh_byte = b(kQhOffset + select(is_high, qh_off_h, qh_off_l), blk, x);
+            Expr hi2 = cast<int32_t>((qh_byte >> qh_shift) & 3);
+            codes(kk, blk, j, x) = (nibble | (hi2 << 4)) - 32;
+
+            // 16 plain signed int8 scales, sub = kk/16 (see class comment).
+            scale(sub, blk, j, x) = cast<int32_t>(reinterpret<int8_t>(b(kScalesOffset + sub * nc + j, blk, x)));
+            d_bytes(byte, blk, j, x) = b(2 * j + byte, blk, x);
+            return {{d_bytes, scale, codes}, {}};
+        } else {  // Q2_K (blocklen fixed 8 upstream)
+            const int kDminOffset = 16, kScalesOffset = 32, kQsOffset = 160;
+            Expr half = kk / 128, local = kk % 128, subg = local / 32, rem32 = local % 32;
+            Expr k = half * 4 + rem32 / bl, i = rem32 % bl;
+            Expr qs_byte = b(kQsOffset + k * nc * bl + j * bl + i, blk, x);
+            codes(kk, blk, j, x) = cast<int32_t>((qs_byte >> (subg * 2)) & 3);
+
+            // scale/min nibble pair, sub = kk/16 (see class comment), combined
+            // into one (plane, sub) field (plane 0 = scale, 1 = min).
+            Expr sm_idx = (sub / 8) * 64 + ((sub % 8) / 2) * 16 + j * 2 + (sub % 2);
+            Expr sm_byte = b(kScalesOffset + sm_idx, blk, x);
+            scale_min(plane, sub, blk, j, x) = cast<int32_t>(nibble_of(sm_byte, plane != 0));
+            d_bytes(byte, blk, j, x) = b(2 * j + byte, blk, x);
+            dmin_bytes(byte, blk, j, x) = b(kDminOffset + 2 * j + byte, blk, x);
+            return {{d_bytes, dmin_bytes, scale_min, codes}, {}};
+        }
+    }
+
+private:
+    KQuantWeightFamily family_;
+    int blocklen_;
+};
+
+// K-quant repack weight scheme: the de-interleave addressing leaf composed
+// with the SAME arithmetic pieces the plain K-quant decode uses (Fp16Pack for
+// the fp16 headers, then LinearDequant in logical element order),
+// paired with a severed 3-D encoder (the weight is pre-quantized; encode is
+// severed by compute_offline). No BlockReshape: the weight stays block-indexed
+// (kk, blk) with the column dims (j, x) riding through.
+inline std::unique_ptr<Halide::Approximation> make_kquant_repack_weight_scheme(
+    KQuantWeightFamily family, int blocklen, int block_bytes) {
+    using namespace Halide;
+    const bool has_min = family != KQuantWeightFamily::Q6_K;
+    const int sub_size = family == KQuantWeightFamily::Q4_K || family == KQuantWeightFamily::Q5_K ? 32 : 16;
+    return std::make_unique<TrustedInverse>(
+        SeveredEncode{block_bytes, 3},
+        Compose{KQuantDeInterleave{family, blocklen},
+                Apply{0, Fp16Pack{}},                               // d_bytes -> d
+                Choose{has_min, Apply{1, Fp16Pack{}}, Identity{}},  // dmin_bytes -> dmin
+                LinearDequant{sub_size, /*has_super_d=*/true, has_min}});
+}
+
+// The make_*() factories below each return one owned Halide::Approximation
+// (as a std::unique_ptr<Halide::Approximation>, the framework's polymorphic
+// scheme handle) -- a single leaf, a Compose, or a TrustedInverse, whichever
+// the format actually is, never a single-element Compose wrapper. A Compose's
+// stage 0 is outermost (its encoded output is the whole thing's result) and
+// its last stage is innermost (closest to the original values).
+//
+// Native (in-Halide-quantizable) formats are a plain Compose. Extern-delegated
+// formats -- whose forward quantize is an opaque GGML extern -- are a
+// TrustedInverse pairing ExternQuantize (encode) with a Compose (decode); see
+// Halide::TrustedInverse and section 4 above.
+
+// The invertible combined-bit split/combine (see CombineBits above), with
+// its own per-part packing folded in via make_block_layout: `fields` lists
+// the {low, high} on-disk sub-fields (in on-disk order, tagged with their
+// logical slots -- 0 for low, 1 for high, so CombineBits sees {low, high}
+// regardless of on-disk order). `fields` is a trailing variadic pack (see
+// make_block_layout), so `high_weight, offset` come first.
+template<typename... Fields>
+inline std::unique_ptr<Halide::Approximation> make_combined_bit_codec(
+    int high_weight, int offset, Fields &&...fields) {
+    using namespace Halide;
+    return std::make_unique<Compose>(
+        make_block_layout(std::forward<Fields>(fields)...).layout,
+        CombineBits{high_weight, offset});
+}
+
+struct CodePackField {
+    std::unique_ptr<Halide::Approximation> pack;
+    int bytes;
+};
+
+inline CodePackField make_code_pack(int block_size, int code_bits, int qmax) {
+    using namespace Halide;
+    if (code_bits == 4) {
+        return {nibble_pack(block_size, qmax), block_size / 2};
+    }
+    if (code_bits == 5) {
+        // Q5_0/Q5_1's split 5-bit code (a 4-bit low nibble plus a 5th high
+        // bit) -- exactly the K-quant combined-bit-code shape make_q5_k_scheme
+        // uses for its own adjacent {qh; qs} region (see the deleted
+        // FiveBitPack's comment, above BitPack, for the verified-by-hand
+        // equivalence): qh's bit `kk` is element kk's own high bit, i.e.
+        // le_bit_pack()'s pos_count=1 addressing (unlike Q5_K's per-window
+        // rotating_bit_pack -- Q5_0/Q5_1's qh is one flat bit array over the
+        // *whole* block, not windowed); the nibble half is the ordinary
+        // nibble_pack. `qmax` becomes CombineBits' final recentering offset
+        // (0 for Q5_1's already-unsigned affine codes) rather than a per-part
+        // qmax, since the parts here are raw, uncentered digits.
+        return {std::make_unique<Compose>(
+                    make_block_layout(
+                        FieldSpec{1, 4, le_bit_pack(16, qmax)},                 // qh -> folded high bit and offset
+                        FieldSpec{0, block_size / 2, nibble_pack(block_size)})  // qs -> low nibble
+                        .layout,
+                    CombineBits{16, qmax, /*expanded_high=*/true}),
+                block_size / 2 + 4};
+    }
+    if (code_bits == 1) {
+        return {std::make_unique<BitPack>(), block_size / 8};
+    }
+    return {std::make_unique<BytePack>(), block_size};
+}
+
+// quantize -> pack codes -> pack scale -> concatenate into one byte buffer
+// with the scale stored first (matching every GGML block_* struct's
+// `{fp16 d; ...qs;}` layout) -> reshape, plus the on-disk block byte count
+// (2 + the code field's own width) -- the source of truth every Generator
+// switch used to re-derive by hand. `layout` picks BlockReshape's flat-row
+// (quantize_row/dequantize_row) vs block-indexed (vec_dot/repack, whose
+// Inputs are already block-indexed so the reshape is a lossless passthrough)
+// shape -- see the Layout enum above; this one factory now covers what used
+// to be a separate make_symmetric_block_codec()/make_symmetric_block_scheme()
+// pair.
+inline SchemeAndBytes make_symmetric_block_scheme(
+    int block_size, int qmax, RoundingMode rounding, ScaleAnchor anchor, int code_bits,
+    Layout layout = Layout::FlatRow, bool struct_layout = false) {
+    using namespace Halide;
+
+    if (struct_layout && code_bits == 4) {
+        _halide_user_assert(block_size == 32 && qmax == 8)
+            << "The core q4_0 layout requires a 32-element, qmax=8 block\n";
+        Type block_type = Type::Struct({{"d", Float(16)}, {"qs", UInt(8), 16}});
+        return {std::make_unique<Compose>(
+                    Halide::StructLayout{block_type, {"qs", "d"}},
+                    Apply{1, Halide::StorageCast<float, float16_t>{}},
+                    Apply{0, Halide::PlanarFieldPack{4, 16}},
+                    Apply{0, Halide::AdditiveOffset<int8_t, uint8_t>{8}},
+                    Halide::SymmetricBlockQuantize{block_size, qmax,
+                                                   core_rounding_mode(rounding), core_scale_anchor(anchor)},
+                    Halide::BlockReshape{block_size, layout == Layout::BlockIndexed}),
+                block_type.bytes(),
+                block_type};
+    }
+
+    if (struct_layout && code_bits == 8) {
+        _halide_user_assert(block_size == 32 && qmax == 127)
+            << "The core q8_0 layout requires a 32-element, qmax=127 block\n";
+        Type block_type = Type::Struct({{"d", Float(16)}, {"qs", Int(8), 32}});
+        return {std::make_unique<Compose>(
+                    Halide::StructLayout{block_type, {"qs", "d"}},
+                    Apply{1, Halide::StorageCast<float, float16_t>{}},
+                    Halide::SymmetricBlockQuantize{block_size, qmax,
+                                                   core_rounding_mode(rounding), core_scale_anchor(anchor)},
+                    Halide::BlockReshape{block_size, layout == Layout::BlockIndexed}),
+                block_type.bytes(),
+                block_type};
+    }
+
+    auto [code_pack, code_bytes] = make_code_pack(block_size, code_bits, qmax);
+
+    if (struct_layout) {
+        // Compatibility path for Q1_0. Q4_0/Q8_0 use their faithful, fully
+        // core-composed layouts above; other symmetric formats migrate
+        // deliberately rather than changing behavior through this fallback.
+        Type block_type = Type::Struct({{"d", Float(16)}, {"qs", UInt(8), code_bytes}});
+        return {std::make_unique<Compose>(
+                    StructBlockLayout{block_type, "d", "qs"},
+                    Apply{0, std::move(code_pack)},
+                    SymmetricAffineQuantize{block_size, qmax, rounding, anchor},
+                    BlockReshape{block_size, layout == Layout::BlockIndexed}),
+                block_type.bytes(),
+                block_type};
+    }
+
+    BlockLayout bl = make_block_layout(
+        FieldSpec{1, 2, std::make_unique<Fp16Pack>()},    // scale
+        FieldSpec{0, code_bytes, std::move(code_pack)});  // codes
+    return {std::make_unique<Compose>(
+                std::move(bl.layout),
+                SymmetricAffineQuantize{block_size, qmax, rounding, anchor},
+                BlockReshape{block_size, layout == Layout::BlockIndexed}),
+            bl.bytes};
+}
+
+// quantize -> pack codes -> pack scale -> pack min -> concatenate (scale, min,
+// codes; matching block_q4_1/block_q5_1's `{fp16 d; fp16 m; ...qs;}` layout)
+// -> reshape -- the affine (min+scale) counterpart to
+// make_symmetric_block_scheme(), used by Q4_1 (code_bits=4, ClampedInt8). Q5_1
+// pairs with make_affine_5bit_block_scheme() below instead, since it also
+// needs the qh high-bit field.
+inline SchemeAndBytes make_affine_block_scheme(
+    int block_size, int levels, AffineRounding rounding, int code_bits, Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    CodePackField code = make_code_pack(block_size, code_bits, /*qmax=*/0);
+    BlockLayout bl = make_block_layout(
+        FieldSpec{1, 2, std::make_unique<Fp16Pack>()},    // scale
+        FieldSpec{2, 2, std::make_unique<Fp16Pack>()},    // min
+        FieldSpec{0, code.bytes, std::move(code.pack)});  // codes
+    return {std::make_unique<Compose>(
+                std::move(bl.layout),
+                AffineQuantize{block_size, levels, rounding},
+                BlockReshape{block_size, layout == Layout::BlockIndexed}),
+            bl.bytes};
+}
+
+// Symmetric quantize (like make_symmetric_block_scheme()) but 5-bit -- now
+// just make_symmetric_block_scheme with code_bits=5, since make_code_pack's
+// code_bits==5 case already assembles the {qh; qs} split (matching
+// block_q5_0's `{fp16 d; qh[4]; qs[16];}`) via the combined-bit codec.
+// `qmax` is always 16 (5-bit signed range [-16, 15]). Kept as its own named
+// entry point (rather than collapsed into SchemeKind::Symmetric in
+// symmetric_quant_generators.cpp) so Q5_0/Q5_1's CMakeLists GENERATOR_ARGS
+// (kind=symmetric_5bit/affine_5bit) don't need to change in lockstep.
+inline SchemeAndBytes make_symmetric_5bit_block_scheme(int block_size, int qmax,
+                                                       Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    _halide_user_assert(block_size == 32) << "The Q5 struct layout requires a 32-element block.\n";
+    _halide_user_assert(qmax == 16) << "The q5_0 additive representation requires qmax 16.\n";
+
+    // Faithful physical declaration: qh remains an array of four bytes in the
+    // public type. LittleEndianScalarPack's concat_bits decode lets
+    // LowerStructTypes recover a single unaligned word load from those bytes.
+    Type block_type = Type::Struct({{"d", Float(16)}, {"qh", UInt(8), 4}, {"qs", UInt(8), 16}});
+
+    Halide::LittleEndianScalarPack<uint32_t> qh_word;
+    Halide::AdditiveRadixSplit radix_split(16, 16);
+    ApproximationStageKey qh_word_key = qh_word.stage_key();
+    ApproximationStageKey reconstructed_codes_key = radix_split.stage_key();
+
+    auto scheme = std::make_unique<Compose>(
+        Halide::StructLayout{block_type, {"qs", "qh", "d"}},
+        Apply{2, Halide::StorageCast<float, float16_t>{}},
+        Apply{1, std::move(qh_word)},
+        Apply{1, Halide::BinaryAlphabetPack<int8_t>{32, UInt(32), -16, 0}},
+        Apply{0, Halide::PlanarFieldPack{4, 16}},
+        Apply{0, /*encode_arity=*/1, /*decode_arity=*/2, std::move(radix_split)},
+        Halide::SymmetricBlockQuantize{block_size, qmax,
+                                       Halide::BlockRoundingMode::TruncateHalfUpWithOffset,
+                                       Halide::BlockScaleAnchor::ExtremeSignedValue},
+        Halide::BlockReshape{block_size, layout == Layout::BlockIndexed});
+    return {std::move(scheme), block_type.bytes(), block_type, false,
+            reconstructed_codes_key, qh_word_key};
+}
+
+// Affine quantize (like make_affine_block_scheme()) but 5-bit -- likewise now
+// just make_affine_block_scheme with code_bits=5, matching block_q5_1's
+// `{fp16 d; fp16 m; qh[4]; qs[16];}`. `qmax=0` passed to make_code_pack here
+// (unlike Q5_0's 16): AffineQuantize's codes are already unsigned [0,
+// levels], not centered, so there's no offset to re-apply before splitting
+// into nibble+high-bit.
+inline SchemeAndBytes make_affine_5bit_block_scheme(int block_size, int levels,
+                                                    AffineRounding rounding,
+                                                    Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    _halide_user_assert(block_size == 32) << "The Q5 struct layout requires a 32-element block.\n";
+    Type block_type = Type::Struct({{"d", Float(16)}, {"m", Float(16)}, {"qh", UInt(32)}, {"qs", UInt(8), 16}});
+    CodePackField code = make_code_pack(block_size, /*code_bits=*/5, /*qmax=*/0);
+    return {std::make_unique<Compose>(
+                Q5StructBlockLayout{block_type, /*affine=*/true},
+                Apply{0, std::move(code.pack)},
+                AffineQuantize{block_size, levels, rounding},
+                BlockReshape{block_size, layout == Layout::BlockIndexed}),
+            block_type.bytes(),
+            block_type};
+}
+
+// Symmetric byte-packed quantize (like make_symmetric_block_scheme() with
+// code_bits=8) plus AppendSums's derived 's' field (SumMode::ScaledFloat),
+// matching block_q8_1's `{fp16 d; fp16 s; qs[32];}` -- Q8_1's scheme. Q8_1 is
+// activation-only (GGML has no public to_float for it), so there's normally
+// no dequantize_row Generator for this scheme's flat-array variant below --
+// but its decode() is still correct and used by any vec_dot pairing against
+// Q8_1 as the activation format. AppendSums needs no Apply wrapper: it
+// consumes and produces the *whole* current list (like quantize itself),
+// not just one element of it.
+inline SchemeAndBytes make_symmetric_byte_sum_block_scheme(int block_size, int qmax,
+                                                           Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    BlockLayout bl = make_block_layout(
+        FieldSpec{1, 2, std::make_unique<Fp16Pack>()},            // scale
+        FieldSpec{2, 2, std::make_unique<Fp16Pack>()},            // sum
+        FieldSpec{0, block_size, std::make_unique<BytePack>()});  // codes
+    return {std::make_unique<Compose>(
+                std::move(bl.layout),
+                AppendSums{block_size, SumMode::ScaledFloat},
+                SymmetricAffineQuantize{block_size, qmax, RoundingMode::Nearest, ScaleAnchor::AbsMax},
+                BlockReshape{block_size, layout == Layout::BlockIndexed}),
+            bl.bytes,
+            /*block_type=*/Halide::Type{},
+            /*has_block_sums=*/true};
+}
+
+// Q8_K: activation-only (quantize_row only, matching Q8_1's own situation
+// above -- see q8_k_generators.cpp), one 256-element superblock, plain int8
+// codes (BytePack), one float32 (not fp16) scale (F32Pack), and 16
+// per-group int32-then-int16 sums (AppendSums, SumMode::RawInt16) -- {float d;
+// qs[256]; bsums[16];}, 292 bytes. RoundingMode::NearestEvenClampedHigh/
+// ScaleAnchor::ExtremeSignedValueTwoStep reproduce GGML's exact
+// nearest-int-then-reciprocal-pair quantizer bit-for-bit -- see their own
+// comments in SymmetricAffineQuantize above for why the usual
+// Nearest/ExtremeSignedValue formulas aren't equivalent here.
+inline SchemeAndBytes make_q8_k_scheme(int block_size, int qmax, Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    BlockLayout bl = make_block_layout(
+        FieldSpec{1, 4, std::make_unique<F32Pack>()},                         // scale
+        FieldSpec{0, block_size, std::make_unique<BytePack>()},               // codes
+        FieldSpec{2, (block_size / 16) * 2, std::make_unique<Int16Pack>()});  // bsums
+    return {std::make_unique<Compose>(
+                std::move(bl.layout),
+                AppendSums{16, SumMode::RawInt16},
+                SymmetricAffineQuantize{block_size, qmax, RoundingMode::NearestEvenClampedHigh,
+                                        ScaleAnchor::ExtremeSignedValueTwoStep},
+                BlockReshape{block_size, layout == Layout::BlockIndexed}),
+            bl.bytes};
+}
+
+// ---------------------------------------------------------------------------
+// Factory helpers for the shared extern-delegated shapes. Each is a plain
+// function that assembles a TrustedInverse{ExternQuantize, Compose{...}} out
+// of the section-4 leaves -- transparent (it returns exactly the Compose you'd
+// write by hand), unlike the bespoke Approximation subclasses this file used
+// to have. The canonical composition for each family lives here once; the
+// per-format make_*_scheme() below are just its parameters.
+// ---------------------------------------------------------------------------
+
+// Codebook formats (IQ4_NL/MXFP4/TQ2_0/TQ1_0/NVFP4): unpack the code field,
+// look it up in `table`, unpack the scale field (`scale_fmt`/`num_scales`
+// picking the pack and its width via make_scale_pack/scale_width, rather
+// than a (pack, width) pair kept in sync by hand), one-level scale multiply,
+// reshape. `scale_first` is the on-disk field order ({scale; codes} vs
+// {codes; scale}); StructPack normalizes both to logical {codes, scale}.
+inline SchemeAndBytes make_codebook_scheme(
+    std::string extern_name, int block_size, Halide::Buffer<int8_t> table,
+    std::unique_ptr<Halide::Approximation> code_pack, int code_bytes,
+    ScaleFormat scale_fmt, int num_scales, bool scale_first, Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    int scale_bytes = scale_width(scale_fmt, num_scales);
+    std::vector<FieldSpec> fields;
+    if (scale_first) {
+        fields.push_back({1, scale_bytes, make_scale_pack(scale_fmt)});
+        fields.push_back({0, code_bytes, std::move(code_pack)});
+    } else {
+        fields.push_back({0, code_bytes, std::move(code_pack)});
+        fields.push_back({1, scale_bytes, make_scale_pack(scale_fmt)});
+    }
+    BlockLayout bl = make_block_layout(std::move(fields));
+    return {std::make_unique<TrustedInverse>(
+                ExternQuantize{std::move(extern_name)},
+                Compose{
+                    std::move(bl.layout),
+                    Apply{0, Codebook{std::move(table)}},  // codes -> codebook values
+                    LinearDequant{num_scales == 1 ? 0 : block_size / num_scales,
+                                  /*has_super_d=*/false, /*has_min=*/false},
+                    BlockReshape{block_size, layout == Layout::BlockIndexed},
+                }),
+            bl.bytes};
+}
+
+// K-quant formats (Q2_K/Q3_K/Q4_K/Q5_K/Q6_K): `fields` lists every on-disk
+// field (d, [dmin,] scale_min, code -- in on-disk order, tagged with their
+// logical slots; see FieldSpec) that make_block_layout unpacks, then the
+// two-level scale multiply and reshape. This is the old KQuantDequantize
+// parameter list -- now assembling a Compose, not a class. `fields` is a
+// trailing variadic pack (see make_block_layout for why), so `layout` --
+// unlike every other make_*_scheme() here -- comes before it rather than
+// trailing with a default.
+template<typename... Fields>
+inline SchemeAndBytes make_k_quant_scheme(
+    std::string extern_name, int block_size, int sub_size, bool has_min, Layout layout,
+    Fields &&...fields) {
+    using namespace Halide;
+    BlockLayout bl = make_block_layout(std::forward<Fields>(fields)...);
+    return {std::make_unique<TrustedInverse>(
+                ExternQuantize{std::move(extern_name)},
+                Compose{std::move(bl.layout),
+                        LinearDequant{sub_size, /*has_super_d=*/true, has_min},
+                        BlockReshape{block_size, layout == Layout::BlockIndexed}}),
+            bl.bytes};
+}
+
+// IQ grid formats (IQ2_S/IQ3_XXS/IQ3_S): a bespoke grid+sign+scale decode leaf
+// that emits values in the superblock's nested structure, with
+// BlockReshape(`block_extents`) doing the flat<->block reshape. `block_bytes`
+// is the leaf's own hand-verified on-disk block size -- unlike the
+// field-table schemes above it can't be derived from a FieldSpec list (the
+// grid leaves are deliberately NOT field-table-decomposed; see section 6's
+// design note), so it's declared here, once, next to the leaf that owns it,
+// and returned in SchemeAndBytes like every other make_*_scheme().
+inline SchemeAndBytes make_grid_scheme(
+    std::string extern_name, int block_bytes, std::unique_ptr<Halide::Approximation> grid_leaf,
+    std::vector<int> block_extents, Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    return {std::make_unique<TrustedInverse>(
+                ExternQuantize{std::move(extern_name)},
+                Compose{std::move(grid_leaf), BlockReshape{std::move(block_extents), layout == Layout::BlockIndexed}}),
+            block_bytes};
+}
+
+inline SchemeAndBytes make_severed_grid_scheme(
+    int block_bytes, std::unique_ptr<Halide::Approximation> grid_leaf,
+    std::vector<int> block_extents, Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    return {std::make_unique<TrustedInverse>(
+                SeveredEncode{block_bytes},
+                Compose{std::move(grid_leaf), BlockReshape{std::move(block_extents), layout == Layout::BlockIndexed}}),
+            block_bytes};
+}
+
+// IQ4_NL: 32-element blocks, 4-bit codes into a 16-value non-uniform
+// codebook, one fp16 scale per block -- {fp16 d; qs[16];}, 18 bytes. Extern
+// quantize; decode unpacks {code_bytes, scale_bytes} (ScaleFirst), looks the
+// nibbles up in the codebook, applies the one fp16 scale, and reshapes to a
+// flat row.
+inline SchemeAndBytes make_iq4_nl_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    static const int8_t kValues[16] = {-127, -104, -83, -65, -49, -35, -22, -10,
+                                       1, 13, 25, 38, 53, 69, 89, 113};
+    static const Buffer<int8_t> table = make_static_codebook(kValues, "kvalues_iq4nl");
+    return make_codebook_scheme("iq4_nl_quantize_via_ggml", 32, table,
+                                nibble_pack(32), 16,
+                                ScaleFormat::Fp16, /*num_scales=*/1, /*scale_first=*/true, layout);
+}
+
+// MXFP4: 32-element blocks, 4-bit codes into the same-shaped 16-value
+// codebook as IQ4_NL (different values), one E8M0 (1-byte, power-of-two)
+// scale per block -- {e8m0 e; qs[16];}, 17 bytes.
+inline SchemeAndBytes make_mxfp4_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    static const int8_t kValues[16] = {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12};
+    static const Buffer<int8_t> table = make_static_codebook(kValues, "kvalues_mxfp4");
+    return make_codebook_scheme("mxfp4_quantize_via_ggml", 32, table,
+                                nibble_pack(32), 16,
+                                ScaleFormat::E8M0, /*num_scales=*/1, /*scale_first=*/true, layout);
+}
+
+// TQ2_0: 256-element superblock, 2-bit codes (each in {0,1,2}, meaning
+// {-1,0,1}) via crumb_pack(128)'s window-interleaved layout, one
+// fp16 scale -- {qs[64]; fp16 d;}, 66 bytes -- qs *before* d, unlike most
+// formats here (StructPack's codes-first field order below).
+inline SchemeAndBytes make_tq2_0_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    static const int8_t kValues[4] = {-1, 0, 1, 0};  // index 3 is never produced
+    static const Buffer<int8_t> table = make_static_codebook(kValues, "kvalues_tq2_0");
+    return make_codebook_scheme("tq2_0_quantize_via_ggml", 256, table,
+                                crumb_pack(128), 64,
+                                ScaleFormat::Fp16, /*num_scales=*/1, /*scale_first=*/false, layout);
+}
+
+// TQ1_0: 256-element superblock, base-3 codes (each in {0,1,2}, meaning
+// {-1,0,1}) via TritPack's 5-trits/byte (+4-trits/byte tail) packing, one
+// fp16 scale -- {qs[48]; qh[4]; fp16 d;}, 54 bytes -- qs+qh (combined, 52
+// bytes) *before* d, like TQ2_0. Reuses TQ2_0's exact {-1, 0, 1, unused}
+// codebook (TritPack's codes are the same raw 0/1/2 digit either way).
+inline SchemeAndBytes make_tq1_0_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    static const int8_t kValues[4] = {-1, 0, 1, 0};  // index 3 is never produced
+    static const Buffer<int8_t> table = make_static_codebook(kValues, "kvalues_tq1_0");
+    return make_codebook_scheme("tq1_0_quantize_via_ggml", 256, table,
+                                std::make_unique<TritPack>(), 52,
+                                ScaleFormat::Fp16, /*num_scales=*/1, /*scale_first=*/false, layout);
+}
+
+// NVFP4: 64-element block, 4 sub-blocks of 16 elements each, 4-bit codes
+// into the same 16-value codebook MXFP4 uses (NVFP4 is MXFP4 with
+// finer-grained scales), one UE4M3 scale *per sub-block* via UE4M3Pack --
+// {d[4]; qs[32];}, 36 bytes -- LinearDequant's num_scales=4 (not 1) is what
+// makes each sub-block's dequantize use its own scale byte instead of one
+// shared scale for the whole 64-element block.
+inline SchemeAndBytes make_nvfp4_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    static const int8_t kValues[16] = {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12};
+    static const Buffer<int8_t> table = make_static_codebook(kValues, "kvalues_nvfp4");
+    return make_codebook_scheme("nvfp4_quantize_via_ggml", 64, table,
+                                nibble_pack(16), 32,
+                                ScaleFormat::UE4M3, /*num_scales=*/4, /*scale_first=*/true, layout);
+}
+
+// Q4_K: 256-element superblock, 8 sub-blocks of 32 elements each, plain
+// 4-bit codes (nibble_pack(64)) and get_scale_min_k4-packed
+// (scale, min) pairs (K4ScaleMinPack) -- {fp16 d; fp16 dmin; scales[12];
+// qs[128];}, 144 bytes, fields already in {d, dmin, scale_min, code} logical
+// order. Two-level scale (d*scale(sub)*code - dmin*min(sub)).
+inline SchemeAndBytes make_q4_k_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    return make_k_quant_scheme(
+        "q4_k_quantize_via_ggml", 256, 32, /*has_min=*/true, layout,
+        FieldSpec{0, 2, std::make_unique<Fp16Pack>()},         // d
+        FieldSpec{1, 2, std::make_unique<Fp16Pack>()},         // dmin
+        FieldSpec{2, 12, std::make_unique<K4ScaleMinPack>()},  // scale_min
+        FieldSpec{3, 128, nibble_pack(64)});                   // codes
+}
+
+// Q5_K: same super-block/sub-block/scale-min shape as Q4_K, but each code
+// is 5 bits: a plain 4-bit low nibble (nibble_pack(64)) plus a
+// 5th high bit from a separate 32-byte, 8-window rotating-bit array
+// (rotating_bit_pack(32)) -- {fp16 d; fp16 dmin; scales[12]; qh[32];
+// qs[128];}, 176 bytes. qh+qs are adjacent in memory, treated as one
+// 160-byte combined "code" field, split by an inner make_combined_bit_codec
+// (qh before qs on-disk), offset=0 since Q5_K's code is a plain 0..31
+// unsigned magnitude, not recentered.
+inline SchemeAndBytes make_q5_k_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    return make_k_quant_scheme(
+        "q5_k_quantize_via_ggml", 256, 32, /*has_min=*/true, layout,
+        FieldSpec{0, 2, std::make_unique<Fp16Pack>()},         // d
+        FieldSpec{1, 2, std::make_unique<Fp16Pack>()},         // dmin
+        FieldSpec{2, 12, std::make_unique<K4ScaleMinPack>()},  // scale_min
+        // Combined 5-bit code: qh (high bit) + qs (low nibble), on-disk
+        // qh before qs; offset 0 (plain 0..31).
+        FieldSpec{3, 160,
+                  make_combined_bit_codec(
+                      16, 0,
+                      FieldSpec{1, 32, rotating_bit_pack(32)},  // qh -> high bit
+                      FieldSpec{0, 128, nibble_pack(64)})});    // qs -> low nibble
+}
+
+// Q2_K: 256-element superblock, 16 sub-blocks of 16 elements each, plain
+// 2-bit codes (crumb_pack(128)) and independent per-sub-block
+// nibble-pair (scale, min) via PlanarBitPack's plane-axis mode (low nibble =
+// scale = plane 0, high nibble = min = plane 1; no bit-interleaving across
+// sub-blocks) -- {scales[16]; qs[64]; fp16 d; fp16 dmin;}, 84 bytes, fields
+// on-disk in {scale_min, code, d, dmin} order (scale_min/code *before* d/dmin,
+// unlike Q4_K/Q5_K), normalized by their own slots back to {d, dmin,
+// scale_min, code}.
+inline SchemeAndBytes make_q2_k_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    return make_k_quant_scheme(
+        "q2_k_quantize_via_ggml", 256, 16, /*has_min=*/true, layout,
+        FieldSpec{2, 16, std::make_unique<PlanarBitPack>(4, 16, 0, /*plane_axis=*/true)},  // scales
+        FieldSpec{3, 64, crumb_pack(128)},                                                 // qs
+        FieldSpec{0, 2, std::make_unique<Fp16Pack>()},                                     // d
+        FieldSpec{1, 2, std::make_unique<Fp16Pack>()});                                    // dmin
+}
+
+// Q3_K: 256-element superblock, 16 sub-blocks of 16 elements each, no min
+// (symmetric, not affine) -- each code is 3 bits: 2 low bits
+// (crumb_pack(128)) plus a high bit from a 32-byte, 8-window
+// rotating-bit "hmask" array (rotating_bit_pack(32)), recentered by -4
+// (CombineBits offset=4, matching a signed [-4, 3] range); scale is 16
+// SIGNED 6-bit values, its own bit-interleaving distinct from get_scale_min_k4
+// (Q3KScalePack) -- {hmask[32]; qs[64]; scales[12]; fp16 d;}, 110 bytes.
+// hmask+qs are adjacent in memory, treated as one 96-byte combined "code"
+// field (hmask before qs on-disk); on-disk {code, scale, d} normalizes to
+// logical {d, scale, code}.
+inline SchemeAndBytes make_q3_k_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    return make_k_quant_scheme(
+        "q3_k_quantize_via_ggml", 256, 16, /*has_min=*/false, layout,
+        // Combined 3-bit code: hmask (high bit) + qs (low 2 bits).
+        // On-disk hmask before qs; offset 4 (recenters to signed [-4, 3]).
+        FieldSpec{2, 96,
+                  make_combined_bit_codec(
+                      4, 4,
+                      FieldSpec{1, 32, rotating_bit_pack(32)},  // hmask -> high bit
+                      FieldSpec{0, 64, crumb_pack(128)})},      // qs -> low 2 bits
+        FieldSpec{1, 12, std::make_unique<Q3KScalePack>()},     // scales
+        FieldSpec{0, 2, std::make_unique<Fp16Pack>()});         // d
+}
+
+// Q6_K: 256-element superblock, 16 sub-blocks of 16 elements each, no min --
+// each code is 6 bits: a plain 4-bit low nibble over *two* 128-element
+// halves (nibble_pack(128)) plus 2 high bits
+// (crumb_pack(128)), recentered by -32; scale is 16 plain SIGNED
+// int8 values, no bit-interleaving at all (BytePack -- its plain
+// reinterpret<int8_t> is exactly what this needs) -- {ql[128]; qh[64];
+// scales[16]; fp16 d;}, 210 bytes. ql+qh are adjacent in memory, treated as
+// one 192-byte combined "code" field (ql before qh on-disk).
+inline SchemeAndBytes make_q6_k_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    return make_k_quant_scheme(
+        "q6_k_quantize_via_ggml", 256, 16, /*has_min=*/false, layout,
+        // Combined 6-bit code: ql (low nibble) + qh (high 2 bits).
+        // On-disk ql before qh; offset 32 (recenters to signed [-32, 31]).
+        FieldSpec{2, 192,
+                  make_combined_bit_codec(
+                      16, 32,
+                      FieldSpec{0, 128, nibble_pack(128)},  // ql -> low nibble
+                      FieldSpec{1, 64, crumb_pack(128)})},  // qh -> high 2 bits
+        FieldSpec{1, 16, std::make_unique<BytePack>()},     // scales, 16 signed int8 values
+        FieldSpec{0, 2, std::make_unique<Fp16Pack>()});     // d
+}
+
+// IQ2_S/IQ3_XXS/IQ3_S: see IQ2SGridDequantize/IQ3XXSGridDequantize/
+// IQ3SGridDequantize above for the bit-layout rationale -- each is a bespoke,
+// self-contained grid+sign+scale decode leaf (its three bit layouts share no
+// sub-formula worth abstracting). Extern quantize; the grid leaf's decode
+// produces block-indexed values, and BlockReshape composes the flat<->block
+// reshape on top.
+inline SchemeAndBytes make_iq2_s_scheme(Layout layout = Layout::FlatRow) {
+    return make_grid_scheme("iq2_s_quantize_via_ggml", 82, std::make_unique<IQ2SGridDequantize>(), {8, 4, 8}, layout);
+}
+
+inline SchemeAndBytes make_iq3_xxs_scheme(Layout layout = Layout::FlatRow) {
+    return make_grid_scheme("iq3_xxs_quantize_via_ggml", 98, std::make_unique<IQ3XXSGridDequantize>(), {8, 4, 8}, layout);
+}
+
+inline SchemeAndBytes make_iq3_s_scheme(Layout layout = Layout::FlatRow) {
+    return make_grid_scheme("iq3_s_quantize_via_ggml", 110, std::make_unique<IQ3SGridDequantize>(), {8, 4, 8}, layout);
+}
+
+// IQ2_XS/IQ2_XXS/IQ1_S/IQ1_M: importance-matrix-only formats with no forward
+// map -- SeveredEncode stands in for the (always-severed) encode half so the
+// dequantize/vec_dot still go through approximate_by/compute_offline. Block
+// bytes: 74 / 66 / 50 / 56.
+inline SchemeAndBytes make_iq2_xs_scheme(Layout layout = Layout::FlatRow) {
+    return make_severed_grid_scheme(74, std::make_unique<IQ2XSGridDequantize>(), {8, 4, 8}, layout);
+}
+
+inline SchemeAndBytes make_iq2_xxs_scheme(Layout layout = Layout::FlatRow) {
+    return make_severed_grid_scheme(66, std::make_unique<IQ2XXSGridDequantize>(), {8, 4, 8}, layout);
+}
+
+inline SchemeAndBytes make_iq1_s_scheme(Layout layout = Layout::FlatRow) {
+    return make_severed_grid_scheme(50, std::make_unique<IQ1SGridDequantize>(), {8, 4, 8}, layout);
+}
+
+inline SchemeAndBytes make_iq1_m_scheme(Layout layout = Layout::FlatRow) {
+    return make_severed_grid_scheme(56, std::make_unique<IQ1MGridDequantize>(), {8, 4, 8}, layout);
+}
+
+// IQ4_XS: 256-element superblock, 8 sub-blocks of 32 elements, the superblock
+// generalization of IQ4_NL's fixed 16-value codebook -- plain 4-bit codes
+// (nibble_pack(32)) into the same kvalues_iq4nl table, scaled by
+// `d * (ls - 32)`, a two-level scale (no min) whose per-sub-block `ls` is
+// bit-interleaved across two byte fields (IQ4XSScalePack) -- {fp16 d;
+// scales_h[2]; scales_l[4]; qs[128];}, 136 bytes.
+inline SchemeAndBytes make_iq4_xs_scheme(Layout layout = Layout::FlatRow) {
+    using namespace Halide;
+    static const int8_t kValues[16] = {-127, -104, -83, -65, -49, -35, -22, -10,
+                                       1, 13, 25, 38, 53, 69, 89, 113};
+    static const Buffer<int8_t> table = make_static_codebook(kValues, "kvalues_iq4nl_xs");
+    // scales_h leads a 2-slot group together with scales_l: IQ4XSScalePack's
+    // decode consumes both (scales_h bytes, then scales_l bytes) to recover
+    // one `scale(sub)` field, the same grouped-field shape make_code_pack's
+    // code_bits==5 combined codec uses for Q5_0/Q5_1's {nibble, qh} pair.
+    BlockLayout bl = make_block_layout(
+        FieldSpec{0, 2, std::make_unique<Fp16Pack>()},                     // d
+        FieldSpec{1, 2, std::make_unique<IQ4XSScalePack>(), /*arity=*/2},  // scales_h (leads the group)
+        FieldSpec{2, 4, nullptr},                                          // scales_l (part of the group above)
+        FieldSpec{3, 128, nibble_pack(32)});                               // qs -> nibbles
+    return {std::make_unique<TrustedInverse>(
+                ExternQuantize{"iq4_xs_quantize_via_ggml"},
+                Compose{
+                    std::move(bl.layout),
+                    Apply{2, Codebook{table}},  // nibbles -> codebook values (slot2: {d, scale, qs})
+                    LinearDequant{32, /*has_super_d=*/true, /*has_min=*/false},
+                    BlockReshape{256, layout == Layout::BlockIndexed},
+                }),
+            bl.bytes};
+}
+
+}  // namespace ggml_halide
diff --git a/apps/ggml/halide/repack_matmul_generator.cpp b/apps/ggml/halide/repack_matmul_generator.cpp
new file mode 100644
index 000000000000..118eea1aeb24
--- /dev/null
+++ b/apps/ggml/halide/repack_matmul_generator.cpp
@@ -0,0 +1,285 @@
+// Generic, family-driven repack gemv/gemm, the matmul counterpart of the
+// repack quantize_mat codecs. Like the vec_dot generators, the weight and
+// activation operands are decoded through the Approximation framework
+// (approximate_by + compute_offline), and the interleaved weight layout is a
+// lossless relayout (UnInterleaveWeight) composed in front of the same lossy
+// quant -- the col dims ride the dimension-general LinearDequant/Codebook via
+// Halide::_. One generator backs every (family, n_cols, blocklen) gemv library
+// (32 hand-rolled kernels -> 2 generic generators). This file covers the four
+// "simple" weight families (Q4_0/Q8_0/IQ4_NL/MXFP4); the interleaved K-quant
+// weights layer on top later.
+
+#include "Halide.h"
+
+#include "quant_components.h"
+#include "sdot_schedule.h"
+
+using namespace Halide;
+using namespace ggml_halide;
+
+namespace {
+
+enum class WFamily { Q4_0,
+                     Q8_0,
+                     IQ4_NL,
+                     MXFP4,
+                     Q4_K,
+                     Q5_K,
+                     Q6_K,
+                     Q2_K };
+
+inline bool is_kquant(WFamily f) {
+    return f == WFamily::Q4_K || f == WFamily::Q5_K || f == WFamily::Q6_K || f == WFamily::Q2_K;
+}
+
+struct WeightSpec {
+    std::unique_ptr<Halide::Approximation> scheme;
+    int block_bytes;
+};
+
+// Weight decode scheme + on-disk block byte width for a simple family. Byte
+// widths: fp16-delta families = 2*n_cols header + payload; mxfp4 = n_cols E8M0
+// header. Nibble payload = 16*n_cols, byte payload = 32*n_cols.
+WeightSpec weight_spec(WFamily fam, int n_cols, int blocklen) {
+    static const int8_t kIq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10,
+                                      1, 13, 25, 38, 53, 69, 89, 113};
+    static const Buffer<int8_t> iq4nl_lut(const_cast<int8_t *>(kIq4nl), 16, "kvalues_iq4nl_gemv");
+    static const int8_t kMxfp4[16] = {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12};
+    static const Buffer<int8_t> mxfp4_lut(const_cast<int8_t *>(kMxfp4), 16, "kvalues_mxfp4_gemv");
+
+    switch (fam) {
+    case WFamily::Q4_0:
+        return {make_repack_weight_scheme(n_cols, blocklen, 18 * n_cols,
+                                          RepackWeightCode::SignedNibble, ScaleFormat::Fp16),
+                18 * n_cols};
+    case WFamily::Q8_0:
+        return {make_repack_weight_scheme(n_cols, blocklen, 34 * n_cols,
+                                          RepackWeightCode::SignedByte, ScaleFormat::Fp16),
+                34 * n_cols};
+    case WFamily::IQ4_NL:
+        return {make_repack_weight_scheme(n_cols, blocklen, 18 * n_cols,
+                                          RepackWeightCode::RawNibble, ScaleFormat::Fp16, iq4nl_lut),
+                18 * n_cols};
+    case WFamily::MXFP4:
+        return {make_repack_weight_scheme(n_cols, blocklen, 17 * n_cols,
+                                          RepackWeightCode::RawNibble, ScaleFormat::E8M0, mxfp4_lut),
+                17 * n_cols};
+    // K-quant weights (n_cols=8 always): a bespoke interleaved decode leaf.
+    case WFamily::Q4_K:
+        return {make_kquant_repack_weight_scheme(KQuantWeightFamily::Q4_K, blocklen, 1152), 1152};
+    case WFamily::Q5_K:
+        return {make_kquant_repack_weight_scheme(KQuantWeightFamily::Q5_K, blocklen, 1408), 1408};
+    case WFamily::Q6_K:
+        return {make_kquant_repack_weight_scheme(KQuantWeightFamily::Q6_K, blocklen, 1680), 1680};
+    case WFamily::Q2_K:
+        return {make_kquant_repack_weight_scheme(KQuantWeightFamily::Q2_K, blocklen, 672), 672};
+    }
+    _halide_internal_error << "RepackGemvGenerator: bad family\n";
+    return {};
+}
+
+// gemv: one plain Q8_0 activation row x every column of a repack-interleaved
+// weight matrix -> s(col-in-group, col-group).
+class RepackGemvGenerator : public Generator<RepackGemvGenerator> {
+public:
+    GeneratorParam<WFamily> family{
+        "family",
+        WFamily::Q4_0,
+        {{"q4_0", WFamily::Q4_0},
+         {"q8_0", WFamily::Q8_0},
+         {"iq4_nl", WFamily::IQ4_NL},
+         {"mxfp4", WFamily::MXFP4},
+         {"q4_k", WFamily::Q4_K},
+         {"q5_k", WFamily::Q5_K},
+         {"q6_k", WFamily::Q6_K},
+         {"q2_k", WFamily::Q2_K}}};
+    GeneratorParam<int> n_cols{"n_cols", 4};
+    GeneratorParam<int> blocklen{"blocklen", 4};
+
+    void configure() {
+        bool kq = is_kquant(family);
+        int block_size = kq ? 256 : 32;
+        WeightSpec w = weight_spec(family, n_cols, blocklen);
+        // Activation: plain Q8_K for K-quant weights, plain Q8_0 otherwise.
+        auto act = kq ? make_q8_k_scheme(256, 127, Layout::BlockIndexed).scheme : make_symmetric_block_scheme(32, 127, RoundingMode::Nearest, ScaleAnchor::AbsMax, 8, Layout::BlockIndexed).scheme;
+        int act_bytes = kq ? (4 + 256 + 2 * (256 / 16)) : (2 + 32);
+
+        ImageParam weight_blocks(UInt(8), 3, "weight_blocks");  // (byte, k-block, col-group)
+        ImageParam act_blocks(UInt(8), 2, "act_blocks");        // plain Q8_0/Q8_K (byte, k-block)
+
+        Var kk("kk"), blk("blk"), j("j"), x("x");
+        Func Wt("wt_naive"), Vec("act_naive");
+        Wt(kk, blk, j, x) = 0.0f;
+        Vec(kk, blk) = 0.0f;
+
+        RDom r(0, block_size, 0, weight_blocks.dim(1).extent(), "r");
+        Func s("s");
+        s(j, x) = 0.0f;
+        s(j, x) += Wt(r.x, r.y, j, x) * Vec(r.x, r.y);
+
+        ApproximationResult wr = Wt.approximate_by(*w.scheme, {s});
+        ApproximationResult ar = Vec.approximate_by(*act, {s});
+        s.update().eager_inline({wr.replacement, ar.replacement});
+
+        std::vector<Func> sever = wr.encoded;
+        sever.insert(sever.end(), ar.encoded.begin(), ar.encoded.end());
+        std::vector<ImageParam> bind = {weight_blocks, act_blocks};
+        Pipeline({s}).compute_offline(sever, bind);
+
+        for (Func h : wr.handles) {
+            if (h.has_update_definition()) {
+                h.compute_root();
+            }
+        }
+        for (Func h : ar.handles) {
+            if (h.has_update_definition()) {
+                h.compute_root();
+            }
+        }
+
+        // Simple single-scale weight families (Q4_0/Q8_0/IQ4_NL/MXFP4) reduce
+        // as a scale-free Int(32) dot (SDOT), same as the vec_dot path: the
+        // per-block weight scale depends only on (block, column) and the
+        // activation scale only on the block, so both hoist out of the r.x sum.
+        // K-quant weights carry two-level (super/sub-block) scales that aren't a
+        // single per-block-invariant factor, so they keep the default schedule.
+        if (!kq) {
+            Var u("u");
+            Func s_i32 = sdot_partial(s, {{r.y, u}}, {wr, ar})[0];
+            s_i32.compute_root().update().atomic().vectorize(r.x, block_size);
+        }
+
+        weight_blocks.dim(0).set_bounds(0, w.block_bytes);
+        weight_blocks.dim(1).set_min(0);
+        weight_blocks.dim(2).set_min(0);
+        act_blocks.dim(0).set_bounds(0, act_bytes);
+        act_blocks.dim(1).set_min(0);
+        s.output_buffer().dim(0).set_bounds(0, n_cols);
+        s.output_buffer().dim(1).set_min(0);
+
+        add_input(weight_blocks);
+        add_input(act_blocks);
+        add_output(s);
+    }
+
+    void generate() {
+    }
+};
+
+// gemm activation: `nr` rows packed 4-at-a-time into the SAME interleaved
+// block layout the weight uses -- so it decodes through make_repack_weight_scheme
+// with n_cols=4 (row group of 4), the 4 "columns" being the 4 packed rows.
+// Q8_0x4 (fp16 scale, 32-block, 136 B) for simple weights; Q8_Kx4 (f32 scale,
+// 256-block, 1168 B incl. dropped bsums) for K-quant weights.
+struct ActSpec {
+    std::unique_ptr<Halide::Approximation> scheme;
+    int block_bytes;
+    int block_size;
+};
+ActSpec act_spec(bool kquant, int blocklen) {
+    if (kquant) {
+        // block_q8_Kx4 is 1168 B, but the gemm reads only the f32 d[4] header
+        // (16 B) + interleaved qs (1024 B) = 1040 B; the 128 B of bsums are
+        // never touched, so the input is bound to 1040, not the full 1168.
+        return {make_repack_weight_scheme(4, blocklen, 1040, RepackWeightCode::SignedByte,
+                                          ScaleFormat::F32, {}, 256),
+                1040, 256};
+    }
+    return {make_repack_weight_scheme(4, blocklen, 136, RepackWeightCode::SignedByte,
+                                      ScaleFormat::Fp16),
+            136, 32};
+}
+
+// gemm: 4 packed activation rows x every column of a repack-interleaved weight
+// matrix -> s(col-in-group j, col-group x, row-in-group m, row-group y). Both
+// operands are interleaved codecs decoded through the framework; the only
+// difference from gemv is that the activation is interleaved too (4 rows) and
+// the output gains the two activation-lane dims.
+class RepackGemmGenerator : public Generator<RepackGemmGenerator> {
+public:
+    GeneratorParam<WFamily> family{
+        "family",
+        WFamily::Q4_0,
+        {{"q4_0", WFamily::Q4_0},
+         {"q8_0", WFamily::Q8_0},
+         {"iq4_nl", WFamily::IQ4_NL},
+         {"mxfp4", WFamily::MXFP4},
+         {"q4_k", WFamily::Q4_K},
+         {"q5_k", WFamily::Q5_K},
+         {"q6_k", WFamily::Q6_K},
+         {"q2_k", WFamily::Q2_K}}};
+    GeneratorParam<int> n_cols{"n_cols", 4};
+    GeneratorParam<int> blocklen{"blocklen", 4};
+
+    void configure() {
+        bool kq = is_kquant(family);
+        int block_size = kq ? 256 : 32;
+        WeightSpec w = weight_spec(family, n_cols, blocklen);
+        ActSpec a = act_spec(kq, blocklen);
+
+        ImageParam weight_blocks(UInt(8), 3, "weight_blocks");  // (byte, k-block, col-group)
+        ImageParam act_blocks(UInt(8), 3, "act_blocks");        // (byte, k-block, row-group)
+
+        Var kk("kk"), blk("blk"), j("j"), x("x"), m("m"), y("y");
+        Func Wt("wt_naive"), Act("act_naive");
+        Wt(kk, blk, j, x) = 0.0f;
+        Act(kk, blk, m, y) = 0.0f;
+
+        RDom r(0, block_size, 0, weight_blocks.dim(1).extent(), "r");
+        Func s("s");
+        s(j, x, m, y) = 0.0f;
+        s(j, x, m, y) += Wt(r.x, r.y, j, x) * Act(r.x, r.y, m, y);
+
+        ApproximationResult wr = Wt.approximate_by(*w.scheme, {s});
+        ApproximationResult ar = Act.approximate_by(*a.scheme, {s});
+        s.update().eager_inline({wr.replacement, ar.replacement});
+
+        std::vector<Func> sever = wr.encoded;
+        sever.insert(sever.end(), ar.encoded.begin(), ar.encoded.end());
+        std::vector<ImageParam> bind = {weight_blocks, act_blocks};
+        Pipeline({s}).compute_offline(sever, bind);
+
+        for (Func h : wr.handles) {
+            if (h.has_update_definition()) {
+                h.compute_root();
+            }
+        }
+        for (Func h : ar.handles) {
+            if (h.has_update_definition()) {
+                h.compute_root();
+            }
+        }
+
+        // Simple single-scale weight families reduce as a scale-free Int(32) dot
+        // (SDOT), same as gemv/vec_dot; K-quant's two-level scales keep the
+        // default schedule. See sdot_schedule.h.
+        if (!is_kquant(family)) {
+            Var u("u");
+            Func s_i32 = sdot_partial(s, {{r.y, u}}, {wr, ar})[0];
+            s_i32.compute_root().update().atomic().vectorize(r.x, block_size);
+        }
+
+        weight_blocks.dim(0).set_bounds(0, w.block_bytes);
+        weight_blocks.dim(1).set_min(0);
+        weight_blocks.dim(2).set_min(0);
+        act_blocks.dim(0).set_bounds(0, a.block_bytes);
+        act_blocks.dim(1).set_min(0);
+        act_blocks.dim(2).set_min(0);
+        s.output_buffer().dim(0).set_bounds(0, n_cols);
+        s.output_buffer().dim(1).set_min(0);
+        s.output_buffer().dim(2).set_bounds(0, 4);
+        s.output_buffer().dim(3).set_min(0);
+
+        add_input(weight_blocks);
+        add_input(act_blocks);
+        add_output(s);
+    }
+
+    void generate() {
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(RepackGemvGenerator, repack_gemv)
+HALIDE_REGISTER_GENERATOR(RepackGemmGenerator, repack_gemm)
diff --git a/apps/ggml/halide/repack_quantize_mat_generators.cpp b/apps/ggml/halide/repack_quantize_mat_generators.cpp
new file mode 100644
index 000000000000..27425c819719
--- /dev/null
+++ b/apps/ggml/halide/repack_quantize_mat_generators.cpp
@@ -0,0 +1,121 @@
+// From-scratch Halide reimplementation of GGML's "repack" quantize_mat
+// kernels (see src/ggml-cpu/repack.cpp: ggml_quantize_mat_q8_0_4x4_generic /
+// ggml_quantize_mat_q8_0_4x8_generic / ggml_quantize_mat_q8_K_4x4_generic /
+// ggml_quantize_mat_q8_K_4x8_generic upstream, as of GGML v0.15.3). These
+// take 4 contiguous rows of `k` floats (row r at x[r*k .. r*k+k)) and
+// interleave them into ONE activation-format block per `k`-sized chunk,
+// where 4 per-row values are laid out consecutively (in groups of
+// `blck_size_interleave`) instead of one row's worth at a time -- this is
+// the packed activation format the corresponding repack_gemv/repack_gemm
+// kernels consume. There are only 4 distinct interleavings (2 activation
+// formats x 2 interleave widths), reused across every repack weight type
+// that shares that (activation, interleave) pair -- see k_repack_entries in
+// providers/ggml_provider.cpp and this file's registration in
+// halide_provider.cpp.
+//
+// block_q8_0x4 layout (136 bytes, one per 32-element chunk x 4 rows):
+//   byte 0-7:    4 fp16 deltas, one per row, in row order
+//   byte 8-135:  128 signed int8 quants, interleaved in groups of
+//                `blck_size_interleave` (4 or 8) per row
+//
+// block_q8_Kx4 layout (1168 bytes, one per 256-element chunk x 4 rows):
+//   byte 0-15:    4 float32 deltas, one per row, in row order
+//   byte 16-1039: 1024 signed int8 quants, interleaved the same way
+//   byte 1040-1167: 64 signed int16 "bsums" (sum of quants in groups of 16,
+//                  scattered across rows/groups by the same index mapping
+//                  GGML uses -- see index_q8_k below)
+//
+// Q8_0's per-row scale is the same amax/127 symmetric scale as plain Q8_0
+// (see quant_components.h's make_symmetric_block_scheme()), using round()
+// (roundf, not round-to-even). Q8_K's per-row scale is the same -127/max
+// signed scale as plain Q8_K (see quant_components.h's make_q8_k_scheme()),
+// using the same round-to-nearest-even magic-number trick -- but unlike
+// plain Q8_K's quantize_row, this repack version has no final MIN(127, ...)
+// clamp (safe here since the scale is derived from this exact block's own
+// amax, so values can never exceed +-127 already).
+//
+// This is intentionally unscheduled beyond the minimum Halide requires for
+// legality (an update-defined Func can't stay inline) -- scheduling for
+// performance is a later step.
+
+#include "Halide.h"
+
+#include "quant_components.h"
+
+using namespace Halide;
+using namespace ggml_halide;
+
+namespace {
+
+constexpr int kQK8_0 = 32;
+constexpr int kBlockBytesQ8_0x4 = 4 * 2 + kQK8_0 * 4;  // 136
+
+constexpr int kQK_K = 256;
+constexpr int kNumGroups = kQK_K / 16;                                     // 16
+constexpr int kBlockBytesQ8_Kx4 = 4 * 4 + kQK_K * 4 + kNumGroups * 4 * 2;  // 1168
+
+// Shared "quantize_mat" pipeline: a 2-D activation x(col, row in [0,4)) flows
+// through the codec `scheme` (block-relayout + Q8 quantize + interleave) via
+// the same approximate_by/compute_offline idiom as codec_generator_base.h's
+// Quantize direction -- the encode half is adopted as the output block buffer.
+struct QuantizeMatPipe {
+    ImageParam x;
+    Func blocks_out;
+};
+inline QuantizeMatPipe build_quantize_mat(std::unique_ptr<Approximation> scheme, int block_bytes) {
+    ImageParam x(Float(32), 2, "x");  // dim 0: col-within-row (mult. of block), dim 1: row (4)
+    Var col("col"), row("row"), byte("byte"), ib("ib");
+    Func identity("qm_identity");
+    identity(col, row) = x(col, row);
+
+    ApproximationResult r = Func(x).approximate_by(*scheme, {identity});
+    for (Func h : r.handles) {
+        if (h.has_update_definition()) {
+            h.compute_root();
+        }
+    }
+
+    ImageParam blocks_in(UInt(8), 2, "blocks_in");
+    ComputeOfflineResult q = Pipeline({identity}).compute_offline(r.encoded, {blocks_in});
+
+    Func blocks_out("blocks");
+    blocks_out(byte, ib) = q.offline.outputs()[0](byte, ib);
+    blocks_out.output_buffer().dim(0).set_bounds(0, block_bytes);
+    blocks_out.output_buffer().dim(1).set_min(0);
+    x.dim(0).set_min(0);
+    x.dim(1).set_bounds(0, 4);
+    return {x, blocks_out};
+}
+
+// q8_0_4x4 / q8_0_4x8 differ only by interleave width (blocklen).
+template<int Blocklen>
+class Q8_0QuantizeMatGenerator : public Generator<Q8_0QuantizeMatGenerator<Blocklen>> {
+public:
+    void configure() {
+        QuantizeMatPipe p = build_quantize_mat(make_q8_0x4_scheme(Blocklen), kBlockBytesQ8_0x4);
+        this->add_input(p.x);
+        this->add_output(p.blocks_out);
+    }
+    void generate() {
+    }
+};
+
+// q8_k_4x4 / q8_k_4x8: same shared pipeline, the Q8_K interleaved codec.
+template<int Blocklen>
+class Q8_KQuantizeMatGenerator : public Generator<Q8_KQuantizeMatGenerator<Blocklen>> {
+public:
+    void configure() {
+        QuantizeMatPipe p = build_quantize_mat(make_q8_kx4_scheme(Blocklen), kBlockBytesQ8_Kx4);
+        this->add_input(p.x);
+        this->add_output(p.blocks_out);
+    }
+    void generate() {
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(Q8_0QuantizeMatGenerator<4>, q8_0_4x4_quantize_mat)
+HALIDE_REGISTER_GENERATOR(Q8_0QuantizeMatGenerator<8>, q8_0_4x8_quantize_mat)
+HALIDE_REGISTER_GENERATOR(Q8_KQuantizeMatGenerator<4>, q8_k_4x4_quantize_mat)
+HALIDE_REGISTER_GENERATOR(Q8_KQuantizeMatGenerator<8>, q8_k_4x8_quantize_mat)
diff --git a/apps/ggml/halide/sdot_schedule.h b/apps/ggml/halide/sdot_schedule.h
new file mode 100644
index 000000000000..e2ae4c2ed8ce
--- /dev/null
+++ b/apps/ggml/halide/sdot_schedule.h
@@ -0,0 +1,93 @@
+#pragma once
+
+// Shared "make an Approximation-decoded block reduction accumulate as an
+// integer dot product" schedule, used by both the vec_dot and repack matmul
+// Generators. Given a reduction `acc` of the shape
+//   acc(...) += decode(Wt)(r.x, r.y, ...) * decode(Vec)(r.x, r.y)
+// whose operands' per-block scales are invariant across the within-block
+// reduction r.x (but vary with the block index r.y and any output dims), this
+// derives the scale-free Int(32) inner dot the caller then schedules.
+//
+// Why the deep inline: hoist_invariants() needs each operand's per-block scale
+// to appear as a top-level factor of the (rfactored) update product. A single
+// eager_inline() of an ApproximationResult's .replacement only peels the
+// outermost relayout wrapper; the scale*codes product lives deeper, inside the
+// dequantizer Func the Approximation combinators build. eager_inline() no-ops
+// on any Func not currently directly called and flattens exposed calls left to
+// right, so inlining the whole set of inlinable decode handles -- one pass per
+// possible chain level -- flattens the decode chains of every operand
+// regardless of build order, leaving (codes*scale)*... with the scales as
+// loop-invariant leaves. See doc: the SDOT investigation on ggml-on-qk.
+
+#include "Halide.h"
+
+#include <algorithm>
+#include <string>
+#include <vector>
+
+namespace ggml_halide {
+
+// rfactor `acc` preserving `preserved` (typically {{r.y, u}} -- the block
+// index), flatten every operand's decode chain into the resulting per-block
+// partial, hoist the now-invariant scales out of the remaining reduction, and
+// retype the scale-free inner dot to Int(32). Returns that Int(32) Func -- it
+// holds the real reduction, so the caller schedules *it* (compute_root,
+// vectorize the within-block RVar, etc.).
+// `distribute` multiplies the per-block product out before hoisting, for
+// formats whose decode carries an offset: an affine weight makes the product
+// (d*code + m) * (d_act*act), which has no single scale to hoist. Multiplied out
+// it is d*d_act * sum(code*act) + m*d_act * sum(act), and hoist_invariants()
+// gives each term its own accumulator -- both with integer bodies, so both reach
+// SDOT. That is ggml's own decomposition of the affine formats.
+// `keep_out` identifies decode-chain Funcs that must NOT be flattened -- a caller
+// that has scheduled one as a materialization boundary (e.g. Q5_x's
+// reconstructed `combine_bits_code`, computed once per block so its qh
+// byte->bits table read is a contiguous load instead of a per-lane gather).
+// can_be_inlined() only checks purity, so a compute_root schedule alone does not
+// stop eager_inline from flattening it; excluding it here does. The scale still
+// hoists past it, since it stays an opaque r.x-dependent factor of the product.
+inline std::vector<Halide::Func> sdot_partial(Halide::Func &acc,
+                                              const std::vector<std::pair<Halide::RVar, Halide::Var>> &preserved,
+                                              const std::vector<Halide::ApproximationResult> &operands,
+                                              bool distribute = false,
+                                              const std::vector<Halide::Func> &keep_out = {}) {
+    using namespace Halide;
+
+    Func acc_dot = acc.update().rfactor(preserved);
+
+    auto excluded = [&](const Func &f) {
+        return std::any_of(keep_out.begin(), keep_out.end(), [&](const Func &kept) {
+            return kept.defined() && kept.function().same_as(f.function());
+        });
+    };
+
+    std::vector<Func> decode_funcs;
+    for (const ApproximationResult &op : operands) {
+        decode_funcs.push_back(op.replacement);
+    }
+    for (const ApproximationResult &op : operands) {
+        for (const Func &h : op.handles) {
+            if (h.function().can_be_inlined() && !excluded(h)) {
+                decode_funcs.push_back(h);
+            }
+        }
+    }
+    for (size_t pass = 0; pass < decode_funcs.size(); pass++) {
+        acc_dot.update().eager_inline(decode_funcs);
+    }
+
+    if (distribute) {
+        acc_dot.update().distribute();
+    }
+
+    // One accumulator per term -- one for a symmetric weight, two once an affine
+    // weight's product has been multiplied out. Each is its own Func, so each
+    // retypes on its own.
+    std::vector<Func> parts;
+    for (Func &part : acc_dot.update().hoist_invariants()) {
+        parts.push_back(part.change_type(Int(32)));
+    }
+    return parts;
+}
+
+}  // namespace ggml_halide
diff --git a/apps/ggml/halide/symmetric_quant_generators.cpp b/apps/ggml/halide/symmetric_quant_generators.cpp
new file mode 100644
index 000000000000..5b2d6863d06a
--- /dev/null
+++ b/apps/ggml/halide/symmetric_quant_generators.cpp
@@ -0,0 +1,123 @@
+// Generic, GeneratorParam-driven quantize/dequantize pair for GGML's legacy
+// per-block quantized formats (see quant_components.h for the reusable
+// Approximation pieces this assembles). "Q4_0"/"Q4_1"/"Q5_0"/"Q5_1"/"Q8_0"/
+// "Q8_1" are not distinct C++ classes here -- they're just different
+// GENERATOR_ARGS instantiations of the same generator template, registered
+// in CMakeLists.txt as e.g. q4_0_quantize/q4_0_dequantize.
+//
+// Quantize and dequantize share every GeneratorParam (the scheme they
+// build is identical, just run in opposite directions), so rather than two
+// classes each redeclaring the same params, this is one class template
+// parameterized on Direction, following
+// apps/linear_algebra/src/blas_l1_generators.cpp's AXPYGenerator<T>
+// precedent (one generator template, registered multiple times under
+// different names/template args).
+//
+// The whole pipeline -- for *either* direction -- is built once in
+// configure(), not generate(): a single Func::approximate_by() +
+// Pipeline::compute_offline() call on a genuinely real ImageParam (not a
+// placeholder) produces both an "offline" half (the encode/quantize side,
+// still depending on that real ImageParam) and an "online" half (the
+// decode/dequantize side, reading from whatever ImageParam
+// compute_offline() severed it to instead). Each direction just adopts
+// whichever half applies to it as its own Input/Output, via
+// GeneratorBase::add_input(const ImageParam&)/add_output(const Func&) --
+// new overloads added to Generator.h/.cpp for exactly this use (see there),
+// since the stock add_input<Buffer<>>()/add_output<Buffer<>>() only ever
+// mint fresh, undefined ports for generate() to fill in later. generate()
+// is therefore an empty stub: by the time it would run, there's nothing
+// left to do.
+//
+// generate() never calls Approximation::encode()/decode() directly -- only
+// through Func::approximate_by() and Pipeline::compute_offline(). This
+// configure()/generate() body is identical across every *_quant_generators.cpp
+// file in this directory, so it lives in codec_generator_base.h's
+// CodecGeneratorBase<Derived, dir> instead of being repeated here -- this
+// class only needs to supply its own GeneratorParams and a build_scheme().
+
+#include "Halide.h"
+
+#include "codec_generator_base.h"
+#include "quant_components.h"
+
+using namespace Halide;
+using namespace ggml_halide;
+
+namespace {
+
+// Which of quant_components.h's make_*_scheme() factories to use -- the one
+// axis that can't be reduced to a GeneratorParam value alone, since each
+// scheme needs a different subset/arity of the other params below.
+enum class SchemeKind { Symmetric,
+                        Affine,
+                        Symmetric5Bit,
+                        Affine5Bit,
+                        SymmetricByteSum,
+                        Q8K };
+
+template<Direction dir>
+class SymmetricCodecGenerator : public CodecGeneratorBase<SymmetricCodecGenerator<dir>, dir> {
+public:
+    GeneratorParam<int> block_size{"block_size", 32};
+    GeneratorParam<int> qmax{"qmax", 127};
+    GeneratorParam<int> code_bits{"code_bits", 8};
+    GeneratorParam<int> levels{"levels", 15};
+    GeneratorParam<RoundingMode> rounding{
+        "rounding",
+        RoundingMode::Nearest,
+        {{"nearest", RoundingMode::Nearest},
+         {"truncate_half_up_with_offset", RoundingMode::TruncateHalfUpWithOffset},
+         {"sign_only", RoundingMode::SignOnly}}};
+    GeneratorParam<ScaleAnchor> anchor{
+        "anchor",
+        ScaleAnchor::AbsMax,
+        {{"abs_max", ScaleAnchor::AbsMax},
+         {"extreme_signed", ScaleAnchor::ExtremeSignedValue},
+         {"mean_abs", ScaleAnchor::MeanAbs}}};
+    GeneratorParam<AffineRounding> affine_rounding{
+        "affine_rounding",
+        AffineRounding::ClampedInt8,
+        {{"clamped_int8", AffineRounding::ClampedInt8},
+         {"unclamped_uint8", AffineRounding::UnclampedUint8}}};
+    GeneratorParam<SchemeKind> kind{
+        "kind",
+        SchemeKind::Symmetric,
+        {{"symmetric", SchemeKind::Symmetric},
+         {"affine", SchemeKind::Affine},
+         {"symmetric_5bit", SchemeKind::Symmetric5Bit},
+         {"affine_5bit", SchemeKind::Affine5Bit},
+         {"symmetric_byte_sum", SchemeKind::SymmetricByteSum},
+         {"q8k", SchemeKind::Q8K}}};
+
+    SchemeAndBytes build_scheme() const {
+        // switch's controlling expression can't resolve GeneratorParam<T>'s
+        // implicit conversion operators unambiguously -- .value() sidesteps
+        // that by returning the plain SchemeKind directly. Each make_*_scheme()
+        // now returns its own block_bytes alongside the scheme (computed from
+        // the same field list it builds internally), so there's no byte
+        // arithmetic to duplicate here.
+        switch (kind.value()) {
+        case SchemeKind::Symmetric:
+            // Phase 3 pilot: the symmetric codecs (Q4_0/Q8_0/Q1_0) build their
+            // on-disk block as a first-class Type::Struct.
+            return make_symmetric_block_scheme(block_size, qmax, rounding, anchor, code_bits,
+                                               Layout::FlatRow, /*struct_layout=*/true);
+        case SchemeKind::Affine:
+            return make_affine_block_scheme(block_size, levels, affine_rounding, code_bits);
+        case SchemeKind::Symmetric5Bit:
+            return make_symmetric_5bit_block_scheme(block_size, qmax);
+        case SchemeKind::Affine5Bit:
+            return make_affine_5bit_block_scheme(block_size, levels, affine_rounding);
+        case SchemeKind::SymmetricByteSum:
+            return make_symmetric_byte_sum_block_scheme(block_size, qmax);
+        case SchemeKind::Q8K:
+            return make_q8_k_scheme(block_size, qmax);
+        }
+        _halide_internal_error << "unreachable SchemeKind\n";
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(SymmetricCodecGenerator<Direction::Quantize>, symmetric_quantize)
+HALIDE_REGISTER_GENERATOR(SymmetricCodecGenerator<Direction::Dequantize>, symmetric_dequantize)
diff --git a/apps/ggml/halide/symmetric_vec_dot_generator.cpp b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
new file mode 100644
index 000000000000..e5ce474d7203
--- /dev/null
+++ b/apps/ggml/halide/symmetric_vec_dot_generator.cpp
@@ -0,0 +1,191 @@
+// Generic, family-driven vec_dot for the symmetric/affine per-block formats,
+// the vec_dot counterpart of symmetric_quant_generators.cpp's
+// SymmetricCodecGenerator. "q4_0_vec_dot"/"q4_1_vec_dot"/... are PARAMS
+// instantiations of this one generator (registered in CMakeLists.txt), not
+// per-format C++ classes. Weight and activation are both block-indexed codecs
+// from quant_components.h; VecDotGeneratorBase splices them via approximate_by/
+// compute_offline (see vec_dot_generator_base.h) -- generate() never calls
+// Approximation::encode()/decode() directly.
+//
+// The weight is one of the symmetric-family kinds (symmetric / affine /
+// symmetric_5bit / affine_5bit); the activation is Q8_0 or Q8_1. Single-scale
+// symmetric weights x Q8_0 reach an SDOT Int(32) inner dot; affine (+min) and
+// mismatched-block pairings (Q1_0 block 128 x Q8_0 block 32) fall back to a
+// Float reduction.
+
+#include "Halide.h"
+
+#include "quant_components.h"
+#include "vec_dot_generator_base.h"
+
+using namespace Halide;
+using namespace ggml_halide;
+
+namespace {
+
+enum class WKind { Symmetric,
+                   Affine,
+                   Symmetric5Bit,
+                   Affine5Bit };
+enum class AKind { Q8_0,
+                   Q8_1 };
+
+class SymmetricVecDotGenerator : public VecDotGeneratorBase<SymmetricVecDotGenerator> {
+public:
+    GeneratorParam<int> block_size{"block_size", 32};
+
+    GeneratorParam<WKind> w_kind{
+        "w_kind",
+        WKind::Symmetric,
+        {{"symmetric", WKind::Symmetric},
+         {"affine", WKind::Affine},
+         {"symmetric_5bit", WKind::Symmetric5Bit},
+         {"affine_5bit", WKind::Affine5Bit}}};
+    GeneratorParam<AKind> a_kind{
+        "a_kind",
+        AKind::Q8_0,
+        {{"q8_0", AKind::Q8_0},
+         {"q8_1", AKind::Q8_1}}};
+
+    // symmetric / symmetric_5bit weight params
+    GeneratorParam<int> w_qmax{"w_qmax", 8};
+    GeneratorParam<int> w_code_bits{"w_code_bits", 4};
+    GeneratorParam<RoundingMode> w_rounding{
+        "w_rounding",
+        RoundingMode::TruncateHalfUpWithOffset,
+        {{"nearest", RoundingMode::Nearest},
+         {"truncate_half_up_with_offset", RoundingMode::TruncateHalfUpWithOffset},
+         {"sign_only", RoundingMode::SignOnly}}};
+    GeneratorParam<ScaleAnchor> w_anchor{
+        "w_anchor",
+        ScaleAnchor::ExtremeSignedValue,
+        {{"abs_max", ScaleAnchor::AbsMax},
+         {"extreme_signed", ScaleAnchor::ExtremeSignedValue},
+         {"mean_abs", ScaleAnchor::MeanAbs}}};
+
+    // affine / affine_5bit weight params
+    GeneratorParam<int> w_levels{"w_levels", 15};
+    GeneratorParam<AffineRounding> w_affine_rounding{
+        "w_affine_rounding",
+        AffineRounding::ClampedInt8,
+        {{"clamped_int8", AffineRounding::ClampedInt8},
+         {"unclamped_uint8", AffineRounding::UnclampedUint8}}};
+
+    // activation param (Q8_0/Q8_1 are always 8-bit int8 codes)
+    GeneratorParam<int> a_qmax{"a_qmax", 127};
+
+    VecDotSpec build_vec_dot() const {
+        int wbs = block_size;
+
+        std::unique_ptr<Halide::Approximation> wc;
+        int wb;
+        ScheduleKind sched;
+        bool distribute = false;   // set for the affine (offset-carrying) weights
+        Halide::Type weight_type;  // set -> weight blocks are a 1-D Type::Struct buffer
+        Halide::ApproximationStageKey reconstructed_codes_stage, packed_high_word_stage;
+        switch (w_kind.value()) {
+        case WKind::Symmetric: {
+            // Struct-typed weight blocks (`{fp16 d; uint8 qs[...]}`); SDOT still
+            // works because the base header's deep inline flattens the struct
+            // decode's dequantizer just like the byte path's.
+            SchemeAndBytes sb = make_symmetric_block_scheme(wbs, w_qmax, w_rounding, w_anchor, w_code_bits,
+                                                            Layout::BlockIndexed, /*struct_layout=*/true);
+            wc = std::move(sb.scheme);
+            weight_type = sb.block_type;
+            wb = 2 + (w_code_bits == 4 ? wbs / 2 : (w_code_bits == 1 ? wbs / 8 : wbs));
+            // 1-bit (Q1_0) stays Float: change_type(Int(32)) can't prove its
+            // deep-inlined per-term range fits Int(32) (its 128-wide block trips
+            // the overflow check), and it's a niche format. Q4_0/Q8_0 -> SDOT.
+            sched = w_code_bits == 1 ? ScheduleKind::Float : ScheduleKind::SDOT;
+            break;
+        }
+        case WKind::Affine:
+            wc = make_affine_block_scheme(wbs, w_levels, w_affine_rounding, w_code_bits, Layout::BlockIndexed).scheme;
+            wb = 2 + 2 + (w_code_bits == 4 ? wbs / 2 : wbs);
+            // The affine decode is (d*code + m), so the per-block product is not
+            // a single scaled term. hoist_invariants() multiplies it out into
+            //   d*d_act * sum(code*act)  +  m*d_act * sum(act)
+            // and gives each term its own accumulator, which is ggml's own
+            // decomposition -- both inner sums are integer, so both reach SDOT.
+            sched = ScheduleKind::SDOT;
+            distribute = true;
+            break;
+        case WKind::Symmetric5Bit: {
+            SchemeAndBytes sb = make_symmetric_5bit_block_scheme(wbs, w_qmax, Layout::BlockIndexed);
+            wc = std::move(sb.scheme);
+            weight_type = sb.block_type;
+            reconstructed_codes_stage = sb.reconstructed_codes_stage;
+            packed_high_word_stage = sb.packed_high_word_stage;
+            wb = 2 + 4 + wbs / 2;
+            // Q5_0's 5-bit code is assembled via CombineBits (nibble |
+            // (high_bit << 4)); that reconstruction is all inside the (r.x-
+            // dependent) codes leaf, so the per-block scale is still a top-level
+            // invariant factor once the decode chain is fully inlined -- SDOT
+            // works the same as Q4_0 (see the base header's deep-inline SDOT).
+            sched = ScheduleKind::SDOT;
+            break;
+        }
+        case WKind::Affine5Bit: {
+            SchemeAndBytes sb = make_affine_5bit_block_scheme(wbs, w_levels, w_affine_rounding, Layout::BlockIndexed);
+            wc = std::move(sb.scheme);
+            weight_type = sb.block_type;
+            wb = 2 + 2 + 4 + wbs / 2;
+            // Same offset-carrying decode as the 4-bit affine case above; the
+            // 5-bit code reconstruction stays inside the codes leaf, so the
+            // multiplied-out terms are integer just the same.
+            sched = ScheduleKind::SDOT;
+            distribute = true;
+            break;
+        }
+        }
+
+        // Q8_0/Q8_1 activations are 32-element blocks. Build the codec at that
+        // natural block size, then Reblock to the weight's block size (a no-op
+        // when they already match, e.g. Q4_0/Q8_0); the byte width stays the
+        // natural-block width since y_blocks is stored at 32-element blocks.
+        const int a_nat = 32;
+        std::unique_ptr<Halide::Approximation> ac;
+        int ab;
+        bool act_has_block_sums = false;
+        Halide::Type act_type;
+        switch (a_kind.value()) {
+        case AKind::Q8_0: {
+            // Keep the shared activation ABI byte-addressed. A struct-typed
+            // activation was tested for same-size blocks, but disturbed the
+            // tuned q4_0/q5_0 load shape; the faithful struct scheme remains in
+            // use for q8_0's codecs and weight operand.
+            const bool structured_q8 = false;
+            SchemeAndBytes sb = make_symmetric_block_scheme(
+                a_nat, a_qmax, RoundingMode::Nearest, ScaleAnchor::AbsMax, 8,
+                Layout::BlockIndexed, structured_q8);
+            ac = std::move(sb.scheme);
+            act_type = sb.block_type;
+            ab = 2 + a_nat;
+            break;
+        }
+        case AKind::Q8_1: {
+            SchemeAndBytes sb = make_symmetric_byte_sum_block_scheme(a_nat, a_qmax, Layout::BlockIndexed);
+            ac = std::move(sb.scheme);
+            ab = 2 + 2 + a_nat;
+            // Q8_1 stores its per-block scaled code sum (`s`); the affine vec_dot
+            // severs the offset term straight to it. Only meaningful when the
+            // activation block matches the weight block (a_nat == wbs, i.e. no
+            // Reblock) -- true for the q4_1/q5_1 pairings.
+            act_has_block_sums = sb.has_block_sums && a_nat == wbs;
+            break;
+        }
+        }
+        ac = reblock_activation(std::move(ac), a_nat, wbs);
+
+        const bool five_bit = w_kind.value() == WKind::Symmetric5Bit || w_kind.value() == WKind::Affine5Bit;
+        const bool clean_symmetric = w_kind.value() == WKind::Symmetric &&
+                                     (w_code_bits == 4 || w_code_bits == 8);
+        return {std::move(wc), wb, std::move(ac), ab, wbs, sched, distribute, weight_type, act_type, act_has_block_sums,
+                five_bit ? 2 : 4, reconstructed_codes_stage, packed_high_word_stage,
+                clean_symmetric, clean_symmetric};
+    }
+};
+
+}  // namespace
+
+HALIDE_REGISTER_GENERATOR(SymmetricVecDotGenerator, symmetric_vec_dot)
diff --git a/apps/ggml/halide/test_bf16.cpp b/apps/ggml/halide/test_bf16.cpp
new file mode 100644
index 000000000000..87aa15a2ce74
--- /dev/null
+++ b/apps/ggml/halide/test_bf16.cpp
@@ -0,0 +1,45 @@
+// Standalone round-trip check: the from-scratch Halide BF16 kernels (both
+// directions fully native) vs GGML's own reference implementation, reached
+// via the public ggml_get_type_traits() API.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_BF16);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_BF16, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_bf16(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_bf16(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_f16.cpp b/apps/ggml/halide/test_f16.cpp
new file mode 100644
index 000000000000..ec1f8f619f7b
--- /dev/null
+++ b/apps/ggml/halide/test_f16.cpp
@@ -0,0 +1,45 @@
+// Standalone round-trip check: the from-scratch Halide F16 kernels (both
+// directions fully native) vs GGML's own reference implementation, reached
+// via the public ggml_get_type_traits() API.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_F16);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_F16, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_f16(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_f16(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq1_m.cpp b/apps/ggml/halide/test_iq1_m.cpp
new file mode 100644
index 000000000000..1f9324bee8d1
--- /dev/null
+++ b/apps/ggml/halide/test_iq1_m.cpp
@@ -0,0 +1,48 @@
+// Standalone check: the from-scratch Halide IQ1_M dequantize kernel vs
+// GGML's own reference implementation. GGML has no public from_float_ref
+// for this importance-matrix-only codebook type (see
+// ../providers/ggml_internal_abi.h's quantize_iq1_m doc comment) -- its
+// only quantizer is reached the same way ggml_provider.cpp reaches it: the
+// private whole-matrix quantize_iq1_m symbol, with a uniform (all-1.0)
+// weighting for consistency with IQ2_XXS/IQ2_XS/IQ1_S (whose equivalent
+// weights are a hard requirement, not just optional here).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "../providers/ggml_internal_abi.h"
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    ggml_quantize_init(GGML_TYPE_IQ1_M);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ1_M);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ1_M, k);
+
+    std::vector<float> weights(k, 1.0f);
+    std::vector<uint8_t> ref_blocks(out_bytes);
+    quantize_iq1_m(x.data(), ref_blocks.data(), /*nrows=*/1, /*n_per_row=*/k, weights.data());
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq1_m(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq1_s.cpp b/apps/ggml/halide/test_iq1_s.cpp
new file mode 100644
index 000000000000..67703e23492a
--- /dev/null
+++ b/apps/ggml/halide/test_iq1_s.cpp
@@ -0,0 +1,47 @@
+// Standalone check: the from-scratch Halide IQ1_S dequantize kernel vs
+// GGML's own reference implementation. GGML has no public from_float_ref
+// for this importance-matrix-only codebook type (see
+// ../providers/ggml_internal_abi.h's quantize_iq1_s doc comment) -- its
+// only quantizer is reached the same way ggml_provider.cpp reaches it: the
+// private whole-matrix quantize_iq1_s symbol, with a uniform (all-1.0)
+// weighting since it hard-requires a non-null quant_weights.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "../providers/ggml_internal_abi.h"
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    ggml_quantize_init(GGML_TYPE_IQ1_S);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ1_S);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ1_S, k);
+
+    std::vector<float> weights(k, 1.0f);
+    std::vector<uint8_t> ref_blocks(out_bytes);
+    quantize_iq1_s(x.data(), ref_blocks.data(), /*nrows=*/1, /*n_per_row=*/k, weights.data());
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq1_s(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq2_s.cpp b/apps/ggml/halide/test_iq2_s.cpp
new file mode 100644
index 000000000000..ec5bce940667
--- /dev/null
+++ b/apps/ggml/halide/test_iq2_s.cpp
@@ -0,0 +1,48 @@
+// Standalone round-trip check: the from-scratch Halide IQ2_S dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    ggml_quantize_init(GGML_TYPE_IQ2_S);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ2_S);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ2_S, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_iq2_s(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq2_s(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq2_xs.cpp b/apps/ggml/halide/test_iq2_xs.cpp
new file mode 100644
index 000000000000..05d738095761
--- /dev/null
+++ b/apps/ggml/halide/test_iq2_xs.cpp
@@ -0,0 +1,47 @@
+// Standalone check: the from-scratch Halide IQ2_XS dequantize kernel vs
+// GGML's own reference implementation. GGML has no public from_float_ref
+// for this importance-matrix-only codebook type (see
+// ../providers/ggml_internal_abi.h's quantize_iq2_xs doc comment) -- its
+// only quantizer is reached the same way ggml_provider.cpp reaches it: the
+// private whole-matrix quantize_iq2_xs symbol, with a uniform (all-1.0)
+// weighting since it hard-requires a non-null quant_weights.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "../providers/ggml_internal_abi.h"
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    ggml_quantize_init(GGML_TYPE_IQ2_XS);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ2_XS);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ2_XS, k);
+
+    std::vector<float> weights(k, 1.0f);
+    std::vector<uint8_t> ref_blocks(out_bytes);
+    quantize_iq2_xs(x.data(), ref_blocks.data(), /*nrows=*/1, /*n_per_row=*/k, weights.data());
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq2_xs(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq2_xxs.cpp b/apps/ggml/halide/test_iq2_xxs.cpp
new file mode 100644
index 000000000000..abc1bfdc9aa7
--- /dev/null
+++ b/apps/ggml/halide/test_iq2_xxs.cpp
@@ -0,0 +1,52 @@
+// Standalone check: the from-scratch Halide IQ2_XXS dequantize kernel vs
+// GGML's own reference implementation. GGML has no public from_float_ref
+// for this importance-matrix-only codebook type (see
+// ../providers/ggml_internal_abi.h's quantize_iq2_xxs doc comment) -- its
+// only quantizer is reached the same way ggml_provider.cpp reaches it: the
+// private whole-matrix quantize_iq2_xxs symbol, called with nrows=1 and no
+// importance matrix to get a plain per-row reference.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "../providers/ggml_internal_abi.h"
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    // The nearest-neighbor grid search this quantizer uses needs its lookup
+    // table built first (see ggml_provider.cpp's comment on ggml_quantize_init).
+    ggml_quantize_init(GGML_TYPE_IQ2_XXS);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ2_XXS);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ2_XXS, k);
+
+    // quantize_row_iq2_xxs_impl hard-requires a non-null quant_weights
+    // (GGML_ASSERT) -- a uniform (all-1.0) weighting treats every element
+    // as equally important, the closest equivalent to "no weighting".
+    std::vector<float> weights(k, 1.0f);
+    std::vector<uint8_t> ref_blocks(out_bytes);
+    quantize_iq2_xxs(x.data(), ref_blocks.data(), /*nrows=*/1, /*n_per_row=*/k, weights.data());
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq2_xxs(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq3_s.cpp b/apps/ggml/halide/test_iq3_s.cpp
new file mode 100644
index 000000000000..2e5132b99486
--- /dev/null
+++ b/apps/ggml/halide/test_iq3_s.cpp
@@ -0,0 +1,48 @@
+// Standalone round-trip check: the from-scratch Halide IQ3_S dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    ggml_quantize_init(GGML_TYPE_IQ3_S);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ3_S);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ3_S, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_iq3_s(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq3_s(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq3_xxs.cpp b/apps/ggml/halide/test_iq3_xxs.cpp
new file mode 100644
index 000000000000..5dd80636469b
--- /dev/null
+++ b/apps/ggml/halide/test_iq3_xxs.cpp
@@ -0,0 +1,48 @@
+// Standalone round-trip check: the from-scratch Halide IQ3_XXS dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    ggml_quantize_init(GGML_TYPE_IQ3_XXS);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ3_XXS);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ3_XXS, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_iq3_xxs(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq3_xxs(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq4_nl.cpp b/apps/ggml/halide/test_iq4_nl.cpp
new file mode 100644
index 000000000000..456ae55cc725
--- /dev/null
+++ b/apps/ggml/halide/test_iq4_nl.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide IQ4_NL dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK4_NL (32)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ4_NL);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ4_NL, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_iq4_nl(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq4_nl(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_iq4_xs.cpp b/apps/ggml/halide/test_iq4_xs.cpp
new file mode 100644
index 000000000000..628a3a39613a
--- /dev/null
+++ b/apps/ggml/halide/test_iq4_xs.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide IQ4_XS dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_IQ4_XS);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_IQ4_XS, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_iq4_xs(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_iq4_xs(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_mxfp4.cpp b/apps/ggml/halide/test_mxfp4.cpp
new file mode 100644
index 000000000000..9d46c4bfa2df
--- /dev/null
+++ b/apps/ggml/halide/test_mxfp4.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide MXFP4 dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_MXFP4 (32)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_MXFP4);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_MXFP4, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_mxfp4(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_mxfp4(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_nvfp4.cpp b/apps/ggml/halide/test_nvfp4.cpp
new file mode 100644
index 000000000000..3e1e9208dd48
--- /dev/null
+++ b/apps/ggml/halide/test_nvfp4.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide NVFP4 dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_NVFP4 (64)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_NVFP4);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_NVFP4, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_nvfp4(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_nvfp4(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q1_0.cpp b/apps/ggml/halide/test_q1_0.cpp
new file mode 100644
index 000000000000..a267bbfa5c6e
--- /dev/null
+++ b/apps/ggml/halide/test_q1_0.cpp
@@ -0,0 +1,45 @@
+// Standalone round-trip check: the from-scratch Halide Q1_0 kernels (both
+// directions fully native) vs GGML's own reference implementation, reached
+// via the public ggml_get_type_traits() API.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK1_0 (128)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q1_0);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q1_0, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q1_0(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q1_0(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q2_k.cpp b/apps/ggml/halide/test_q2_k.cpp
new file mode 100644
index 000000000000..0fd2c5deee59
--- /dev/null
+++ b/apps/ggml/halide/test_q2_k.cpp
@@ -0,0 +1,49 @@
+// Standalone round-trip check: the from-scratch Halide Q2_K dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp), so its
+// output is trivially identical to GGML's -- this test's real purpose is
+// exercising the from-scratch dequantize implementation against blocks
+// GGML itself produced.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q2_K);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q2_K, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q2_k(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q2_k(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q3_k.cpp b/apps/ggml/halide/test_q3_k.cpp
new file mode 100644
index 000000000000..767eb193727b
--- /dev/null
+++ b/apps/ggml/halide/test_q3_k.cpp
@@ -0,0 +1,49 @@
+// Standalone round-trip check: the from-scratch Halide Q3_K dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp), so its
+// output is trivially identical to GGML's -- this test's real purpose is
+// exercising the from-scratch dequantize implementation against blocks
+// GGML itself produced.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q3_K);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q3_K, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q3_k(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q3_k(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q4_0.cpp b/apps/ggml/halide/test_q4_0.cpp
new file mode 100644
index 000000000000..f568b0f17401
--- /dev/null
+++ b/apps/ggml/halide/test_q4_0.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide Q4_0 kernels vs
+// GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API (same accessor providers/ggml_provider.cpp
+// uses). Run before wiring this into kernel-bench as a provider.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK4_0 (32)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q4_0);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q4_0, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q4_0(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q4_0(halide_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q4_1.cpp b/apps/ggml/halide/test_q4_1.cpp
new file mode 100644
index 000000000000..bdbb4f9f0c5d
--- /dev/null
+++ b/apps/ggml/halide/test_q4_1.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide Q4_1 kernels vs
+// GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API (same accessor providers/ggml_provider.cpp
+// uses). Run before wiring this into kernel-bench as a provider.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK4_1 (32)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q4_1);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q4_1, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q4_1(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q4_1(halide_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q4_k.cpp b/apps/ggml/halide/test_q4_k.cpp
new file mode 100644
index 000000000000..62174d0e5e3d
--- /dev/null
+++ b/apps/ggml/halide/test_q4_k.cpp
@@ -0,0 +1,49 @@
+// Standalone round-trip check: the from-scratch Halide Q4_K dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp), so its
+// output is trivially identical to GGML's -- this test's real purpose is
+// exercising the from-scratch dequantize implementation against blocks
+// GGML itself produced.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q4_K);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q4_K, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q4_k(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q4_k(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q5_0.cpp b/apps/ggml/halide/test_q5_0.cpp
new file mode 100644
index 000000000000..d64bf692ff80
--- /dev/null
+++ b/apps/ggml/halide/test_q5_0.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide Q5_0 kernels vs
+// GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API (same accessor providers/ggml_provider.cpp
+// uses). Run before wiring this into kernel-bench as a provider.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK5_0 (32)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q5_0);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q5_0, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q5_0(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q5_0(halide_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q5_1.cpp b/apps/ggml/halide/test_q5_1.cpp
new file mode 100644
index 000000000000..2b096dfa3ebd
--- /dev/null
+++ b/apps/ggml/halide/test_q5_1.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide Q5_1 kernels vs
+// GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API (same accessor providers/ggml_provider.cpp
+// uses). Run before wiring this into kernel-bench as a provider.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK5_1 (32)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q5_1);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q5_1, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q5_1(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q5_1(halide_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q5_k.cpp b/apps/ggml/halide/test_q5_k.cpp
new file mode 100644
index 000000000000..d54e62c4f9dc
--- /dev/null
+++ b/apps/ggml/halide/test_q5_k.cpp
@@ -0,0 +1,49 @@
+// Standalone round-trip check: the from-scratch Halide Q5_K dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp), so its
+// output is trivially identical to GGML's -- this test's real purpose is
+// exercising the from-scratch dequantize implementation against blocks
+// GGML itself produced.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q5_K);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q5_K, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q5_k(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q5_k(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q6_k.cpp b/apps/ggml/halide/test_q6_k.cpp
new file mode 100644
index 000000000000..804fb2f35081
--- /dev/null
+++ b/apps/ggml/halide/test_q6_k.cpp
@@ -0,0 +1,49 @@
+// Standalone round-trip check: the from-scratch Halide Q6_K dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp), so its
+// output is trivially identical to GGML's -- this test's real purpose is
+// exercising the from-scratch dequantize implementation against blocks
+// GGML itself produced.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q6_K);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q6_K, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q6_k(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q6_k(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q8_0.cpp b/apps/ggml/halide/test_q8_0.cpp
new file mode 100644
index 000000000000..95135acf835b
--- /dev/null
+++ b/apps/ggml/halide/test_q8_0.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide Q8_0 kernels vs
+// GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API (same accessor providers/ggml_provider.cpp
+// uses). Run before wiring this into kernel-bench as a provider.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK8_0 (32)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q8_0);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q8_0, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q8_0(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_q8_0(halide_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q8_1.cpp b/apps/ggml/halide/test_q8_1.cpp
new file mode 100644
index 000000000000..fe570667642f
--- /dev/null
+++ b/apps/ggml/halide/test_q8_1.cpp
@@ -0,0 +1,37 @@
+// Standalone check: the from-scratch Halide Q8_1 quantize kernel vs GGML's
+// own reference implementation, reached via the public
+// ggml_get_type_traits() API. Q8_1 is an activation-only format -- GGML has
+// no public to_float for it, so there's no dequantize round-trip to check
+// here, only the quantize output.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK8_1 (32)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_Q8_1);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q8_1, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q8_1(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_q8_k.cpp b/apps/ggml/halide/test_q8_k.cpp
new file mode 100644
index 000000000000..5413714382c5
--- /dev/null
+++ b/apps/ggml/halide/test_q8_k.cpp
@@ -0,0 +1,40 @@
+// Standalone check: the from-scratch Halide Q8_K quantize kernel vs GGML's
+// own reference implementation. Unlike every other type here, Q8_K has no
+// public from_float_ref (see include/ggml.h's type_traits table) -- it's
+// reached the same way apps/ggml/providers/ggml_provider.cpp reaches it,
+// through the private quantize_row_q8_K_generic symbol declared in
+// ../providers/ggml_internal_abi.h (see that header's own comment for why
+// this one symbol needs the private ABI). Q8_K is activation-only, so
+// there's no dequantize round-trip to check, only the quantize output.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "../providers/ggml_internal_abi.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_Q8_K, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    quantize_row_q8_K_generic(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_q8_k(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_tq1_0.cpp b/apps/ggml/halide/test_tq1_0.cpp
new file mode 100644
index 000000000000..3c08bb576797
--- /dev/null
+++ b/apps/ggml/halide/test_tq1_0.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide TQ1_0 dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_TQ1_0);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_TQ1_0, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_tq1_0(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_tq1_0(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/test_tq2_0.cpp b/apps/ggml/halide/test_tq2_0.cpp
new file mode 100644
index 000000000000..19464cb6eab6
--- /dev/null
+++ b/apps/ggml/halide/test_tq2_0.cpp
@@ -0,0 +1,46 @@
+// Standalone round-trip check: the from-scratch Halide TQ2_0 dequantize
+// kernel vs GGML's own reference implementation, reached via the public
+// ggml_get_type_traits() API. Quantize here is scaffolding that itself
+// calls out to GGML's reference (see ggml_extern_quantize.cpp).
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <vector>
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_quants.h"
+
+int main() {
+    const int64_t k = 4096;  // multiple of QK_K (256)
+
+    std::vector<float> x(k);
+    generate_synthetic_data(x.data(), k);
+
+    const ggml_type_traits *tt = ggml_get_type_traits(GGML_TYPE_TQ2_0);
+    const size_t out_bytes = ggml_row_size(GGML_TYPE_TQ2_0, k);
+
+    std::vector<uint8_t> ref_blocks(out_bytes), halide_blocks(out_bytes);
+    tt->from_float_ref(x.data(), ref_blocks.data(), k);
+    ggml_quants_halide_quantize_tq2_0(x.data(), halide_blocks.data(), k);
+
+    if (std::memcmp(ref_blocks.data(), halide_blocks.data(), out_bytes) != 0) {
+        std::fprintf(stderr, "FAIL: quantize output does not match GGML's reference byte-for-byte\n");
+        return 1;
+    }
+
+    std::vector<float> ref_y(k), halide_y(k);
+    tt->to_float(ref_blocks.data(), ref_y.data(), k);
+    ggml_quants_halide_dequantize_tq2_0(ref_blocks.data(), halide_y.data(), k);
+
+    if (!floats_match(ref_y.data(), halide_y.data(), k)) {
+        std::fprintf(stderr, "FAIL: dequantize output does not match GGML's reference within tolerance\n");
+        return 1;
+    }
+
+    std::printf("Success!\n");
+    return 0;
+}
diff --git a/apps/ggml/halide/vec_dot_generator_base.h b/apps/ggml/halide/vec_dot_generator_base.h
new file mode 100644
index 000000000000..f246e2db434a
--- /dev/null
+++ b/apps/ggml/halide/vec_dot_generator_base.h
@@ -0,0 +1,583 @@
+#pragma once
+
+// Shared configure() scaffolding for every Approximation-based vec_dot
+// Generator (the extended SymmetricVecDotGenerator, KQuantVecDotGenerator,
+// LookupTableVecDotGenerator). All three build the same "naive fp32 dot
+// product -> approximate_by both operands -> compute_offline severs the
+// (already-quantized, Input-supplied) encode halves -> schedule" pipeline;
+// they differ only in which block-indexed codecs their build_vec_dot() picks.
+// This factors that shared body out via CRTP (Derived::build_vec_dot()) -- the
+// same static-polymorphism idiom as codec_generator_base.h's
+// CodecGeneratorBase.
+//
+// generate() never calls Approximation::encode()/decode() directly -- only
+// through Func::approximate_by()/Pipeline::compute_offline(), exactly like the
+// codec generators. The vec_dot is the point at which the framework's splice +
+// sever path is exercised for a dot product (matching
+// test/performance/matvec_offline_split.cpp).
+//
+// Usage:
+//   class FooVecDotGenerator : public VecDotGeneratorBase<FooVecDotGenerator> {
+//   public:
+//       GeneratorParam<...> family{...};
+//       VecDotSpec build_vec_dot() const { return {weight_codec, wbytes, act_codec, abytes, block_size, sched}; }
+//   };
+
+#include "Halide.h"
+
+#include "codec_generator_base.h"  // Direction/SchemeAndBytes live here; shared idiom
+#include "quant_components.h"
+#include "sdot_schedule.h"
+
+namespace ggml_halide {
+
+// Whether the per-block scale factors out to a single block-invariant scalar
+// (SDOT: hoist_invariants() + rfactor() + change_type(Int(32)) -> Int(32) inner
+// dot) or not (Float: a plain vectorized float accumulation -- affine offsets,
+// two-level sub-block scales, and per-group grid scales are not
+// single-per-block-invariant).
+enum class ScheduleKind { SDOT,
+                          Float };
+
+struct VecDotSpec {
+    // Both codecs decode to a block-indexed (kk, blk) Func at the SAME
+    // block_size, so the reduction below is uniform -- Wt(r.x, r.y) * Vec(r.x,
+    // r.y). When an activation is stored in a smaller block than the weight
+    // (e.g. Q1_0/NVFP4 x Q8_0), its codec is composed with a Reblock stage
+    // (see quant_components.h) that re-views it at the weight's block_size --
+    // the block-structure reconciliation is an Approximation, not something
+    // the Generator open-codes into the reduction.
+    std::unique_ptr<Halide::Approximation> weight_codec;
+    int weight_bytes;
+    std::unique_ptr<Halide::Approximation> act_codec;
+    int act_bytes;
+    int block_size;
+    ScheduleKind sched;
+    // Whether the SDOT schedule must multiply the per-block product out before
+    // hoisting. Set for the formats whose weight decode carries an offset (the
+    // affine family): (d*code + m) * (d_act*act) has no single scale to hoist
+    // until it is expanded. See sdot_schedule.h.
+    bool distribute_terms;
+    // When set (is_struct()), the corresponding operand's packed blocks are a
+    // first-class 1-D Type::Struct buffer (one struct per block) rather than a
+    // 2-D (byte, blk) UInt(8) buffer. Default-invalid keeps a codec on the byte
+    // path -- so an operand whose codec isn't struct-typed (e.g. a Reblock'd
+    // activation) just leaves its type unset.
+    Halide::Type weight_type;
+    Halide::Type act_type;
+    // Set when the activation stores a per-block scaled sum of its codes (Q8_1's
+    // `s` field). Together with distribute_terms (an affine weight), this lets
+    // configure() sever the offset term's sum(act) accumulator to a stored fp16
+    // field supplied as a third Input, instead of recomputing it -- ggml's own
+    // q4_1/q5_1 optimization. See SchemeAndBytes::has_block_sums.
+    bool act_has_block_sums = false;
+    // Number of adjacent blocks kept in flight by the SDOT schedule. The
+    // ordinary q4/q8 paths prefer four; q5's table-expansion live ranges make
+    // two faster and match GGML's paired-block loop.
+    int unroll_blocks = 4;
+    Halide::ApproximationStageKey reconstructed_codes_stage;
+    Halide::ApproximationStageKey packed_high_word_stage;
+    // Clean core-composed schemes can share one traced decode graph between
+    // the four-block main update and its tiny remainder update. Legacy schemes
+    // retain separate traces until their representation stages are migrated.
+    bool share_weight_tail = false;
+    bool share_act_tail = false;
+};
+
+template<typename Derived>
+class VecDotGeneratorBase : public Halide::Generator<Derived> {
+public:
+    void configure() {
+        using namespace Halide;
+        VecDotSpec spec = static_cast<Derived *>(this)->build_vec_dot();
+        int bs = spec.block_size;
+        const int unroll_blocks = spec.unroll_blocks;
+
+        // A struct-typed operand's packed blocks are a 1-D Type::Struct buffer
+        // (block index only); a byte-path operand is 2-D (byte, blk).
+        const bool wt_struct = spec.weight_type.is_struct();
+        const bool act_struct = spec.act_type.is_struct();
+        ImageParam x_blocks = wt_struct ? ImageParam(spec.weight_type, 1, "x_blocks") : ImageParam(UInt(8), 2, "x_blocks");
+        ImageParam y_blocks = act_struct ? ImageParam(spec.act_type, 1, "y_blocks") : ImageParam(UInt(8), 2, "y_blocks");
+
+        // Third Input, present only for the affine-x-Q8_1 pairings: the
+        // activation's stored per-block scaled code sum (`s`). configure() severs
+        // the offset term's sum(act) accumulator to this field rather than
+        // recomputing it -- a zero-copy 1-D fp16 view of the `s` slot the ABI
+        // wrapper passes (stride = block width). See the SDOT sever branch below.
+        const bool sever_sum = spec.sched == ScheduleKind::SDOT &&
+                               spec.distribute_terms && spec.act_has_block_sums;
+        ImageParam s_blocks = sever_sum ? ImageParam(Float(16), 1, "s_blocks") : ImageParam();
+
+        // The packed-block buffers are quantized GGML rows: their base pointers
+        // are cache-line aligned. Without this Halide assumes 1-byte alignment
+        // and lowers every strided / reinterpreted read (the interleaved fp16
+        // scales, the int8 codes) to byte-wise ld1.b + orr reassembly instead
+        // of wide vector loads -- which dominates these tiny vec_dots.
+        x_blocks.set_host_alignment(64);
+        y_blocks.set_host_alignment(64);
+
+        // Naive fp32 placeholders -- never realized; compute_offline() severs
+        // Acc from them entirely, and the real values come from the
+        // already-quantized x_blocks/y_blocks. Block-indexed (kk, blk) to match
+        // the codecs' block-indexed decode.
+        Var kk("kk"), blk("blk"), u("u");
+        Func Wt("wt_naive"), Vec("vec_naive");
+        Wt(kk, blk) = 0.0f;
+        Vec(kk, blk) = 0.0f;
+
+        // The SDOT schedule interleaves unroll_blocks blocks into independent
+        // accumulators, so it wants a block count divisible by that. Letting
+        // Halide's split produce the odd tail instead is not a local cost: the
+        // predicate it inserts makes the per-block sdot a dynamic-extent
+        // allocation that has to be zeroed and accumulated through memory,
+        // roughly doubling the cost of *every* block. So the main reduction gets
+        // an exactly divisible extent and a second update sweeps the remainder
+        // at the default schedule (at most unroll_blocks - 1 blocks).
+        const bool sdot = spec.sched == ScheduleKind::SDOT;
+        const bool keyed_q5 = spec.reconstructed_codes_stage.defined();
+        const bool share_weight_tail = keyed_q5 || spec.share_weight_tail;
+        const bool share_act_tail = spec.share_act_tail;
+        Expr nblocks = x_blocks.dim(wt_struct ? 0 : 1).extent();
+        Expr main_blocks = sdot ? (nblocks / unroll_blocks) * unroll_blocks : nblocks;
+
+        RDom r(0, bs, 0, main_blocks, "r");
+        Func Acc("acc");
+        Acc() = 0.0f;
+        Acc() += Wt(r.x, r.y) * Vec(r.x, r.y);
+
+        // The odd-block tail decodes through its OWN placeholder Funcs, given a
+        // separate decode chain by a second approximate_by below. This lets the
+        // main reduction materialize a reconstructed-codes leaf (Q5_x) via
+        // compute_at while the tail -- a different loop nest that could not see
+        // that per-block buffer -- reconstructs inline, at negligible cost (fewer
+        // than unroll_blocks blocks). Both chains read the same x_blocks/y_blocks.
+        Func WtT("wt_naive_tail"), VecT("vec_naive_tail");
+        WtT(kk, blk) = 0.0f;
+        VecT(kk, blk) = 0.0f;
+        RDom r_tail(0, bs, main_blocks, nblocks - main_blocks, "r_tail");
+        if (sdot) {
+            // q5_0 shares the main decode graph with its tiny odd-block tail.
+            // The tail update is eagerly inlined below before the reconstructed
+            // codes leaf is materialized for the main paired-block update.
+            Func tail_weight = share_weight_tail ? Wt : WtT;
+            Func tail_act = share_act_tail ? Vec : VecT;
+            Acc() += tail_weight(r_tail.x, r_tail.y) * tail_act(r_tail.x, r_tail.y);
+        }
+
+        ApproximationResult wt_r = Wt.approximate_by(*spec.weight_codec, {Acc});
+        ApproximationResult act_r = Vec.approximate_by(*spec.act_codec, {Acc});
+
+        // Both operands' encode halves are severed and bound to the real
+        // already-quantized Input buffers (same as symmetric_vec_dot). For the
+        // extern-delegated / SeveredEncode weight schemes the encode is likewise
+        // severed here, so its extern symbol is never computed or linked.
+        std::vector<Func> to_sever = wt_r.encoded;
+        to_sever.insert(to_sever.end(), act_r.encoded.begin(), act_r.encoded.end());
+        std::vector<ImageParam> bind_to = {x_blocks, y_blocks};
+        ApproximationResult wtT_r, actT_r;
+        if (sdot) {
+            if (!share_weight_tail) {
+                wtT_r = WtT.approximate_by(*spec.weight_codec, {Acc});
+                to_sever.insert(to_sever.end(), wtT_r.encoded.begin(), wtT_r.encoded.end());
+                bind_to.push_back(x_blocks);
+                for (Func h : wtT_r.handles) {
+                    if (h.has_update_definition()) {
+                        h.compute_root();
+                    }
+                }
+            }
+            if (!share_act_tail) {
+                actT_r = VecT.approximate_by(*spec.act_codec, {Acc});
+                to_sever.insert(to_sever.end(), actT_r.encoded.begin(), actT_r.encoded.end());
+                bind_to.push_back(y_blocks);
+                for (Func h : actT_r.handles) {
+                    if (h.has_update_definition()) {
+                        h.compute_root();
+                    }
+                }
+            }
+        }
+        Pipeline({Acc}).compute_offline(to_sever, bind_to);
+
+        // Q5_0/Q5_1 reconstruct each code from a nibble plus a per-element high
+        // bit read from the qh field's byte->bits expansion table (see
+        // PlanarBitPack::decode). That table read is only a *contiguous* 8-byte
+        // load -- matching ggml's table_b2b -- when the qh byte is a scalar and
+        // the 8 bit positions are the vector lanes. Inlined into the sdot it is
+        // the opposite (qh byte per lane -> a per-lane gather), so the SDOT
+        // branches materialize the reconstructed int8 codes per block (compute_at
+        // the block loop, kk split as (byte, pos): pos vectorizes the table load,
+        // byte unrolls to a scalar index). The odd-block tail decodes through its
+        // own inline chain (see above), so it does not need this buffer.
+        Func codes_leaf = wt_r.decoded_by(spec.reconstructed_codes_stage);
+        Func qh_leaf = wt_r.decoded_by(spec.packed_high_word_stage);
+        if (!keyed_q5) {
+            // Transitional q5_1 debt: its legacy composition has not yet
+            // exported stage keys. Keep its existing generated-name discovery
+            // local to that path; all scheduling boundaries below use Func
+            // identity once resolved.
+            for (const Func &h : wt_r.handles) {
+                if (h.name() == "combine_bits_code") {
+                    codes_leaf = h;
+                } else if (h.name() == "q5_struct_block_qh") {
+                    qh_leaf = h;
+                }
+            }
+        }
+        _halide_internal_assert(!keyed_q5 || (codes_leaf.defined() && qh_leaf.defined()));
+
+        if (share_weight_tail || share_act_tail) {
+            // Flatten shared decode stages into this update only. The main
+            // update is scheduled independently below, so its SDOT boundaries
+            // and any keyed materializations remain intact.
+            std::vector<Func> tail_inline;
+            if (share_weight_tail) {
+                tail_inline.push_back(wt_r.replacement);
+                for (const Func &h : wt_r.handles) {
+                    if (h.function().can_be_inlined()) {
+                        tail_inline.push_back(h);
+                    }
+                }
+            }
+            if (share_act_tail) {
+                tail_inline.push_back(act_r.replacement);
+                for (const Func &h : act_r.handles) {
+                    if (h.function().can_be_inlined()) {
+                        tail_inline.push_back(h);
+                    }
+                }
+            }
+            for (size_t pass = 0; pass < tail_inline.size(); ++pass) {
+                Acc.update(1).eager_inline(tail_inline);
+            }
+        }
+        auto schedule_codes = [&](LoopLevel level) {
+            // Split kk into (byte, pos): pos (8) vectorizes the contiguous table
+            // load, byte unrolls to a scalar index. store_in(Register) keeps the
+            // reconstructed codes in the vector register file straight into the
+            // sdot instead of round-tripping a stack buffer (all stores are at
+            // constant coordinates once ki is vectorized and ko unrolled).
+            Var kc = codes_leaf.args()[0], co("co"), ci("ci"), byte("byte"), pos("pos");
+            codes_leaf.compute_at(level)
+                .store_in(MemoryType::Register)
+                .split(kc, co, ci, 16)    // 16-code register unit = one sdot chunk
+                .split(ci, byte, pos, 8)  // within it, two qh bytes x 8 positions
+                .vectorize(pos, 8)
+                .unroll(byte)
+                .unroll(co);
+            if (qh_leaf.defined()) {
+                qh_leaf.compute_at(level).store_in(MemoryType::Register);
+            }
+        };
+
+        // Only handles with update definitions (per-block stat reductions) need
+        // explicit scheduling; pure pass-throughs stay inline (same reasoning as
+        // symmetric_vec_dot_generator.cpp).
+        for (Func h : wt_r.handles) {
+            if (h.has_update_definition()) {
+                h.compute_root();
+            }
+        }
+        for (Func h : act_r.handles) {
+            if (h.has_update_definition()) {
+                h.compute_root();
+            }
+        }
+
+        Func final_value = Acc;
+        if (sever_sum && codes_leaf.defined()) {
+            // Q5_1 needs two differently shaped reductions: a four-lane dot
+            // accumulator that survives across blocks, and the scalar m*s
+            // term supplied by Q8_1's stored sum. Express them independently,
+            // then fuse their paired-block loops. This mirrors GGML and avoids
+            // the generic sever path's horizontal Int(32) reduction per block.
+            Func min_leaf, scale_leaf;
+            for (const Func &h : wt_r.handles) {
+                if (h.name() == "q5_struct_block_min") {
+                    min_leaf = h;
+                } else if (h.name() == "q5_struct_block_scale") {
+                    scale_leaf = h;
+                }
+            }
+            Func codes_tail, scale_tail;
+            for (const Func &h : wtT_r.handles) {
+                if (h.name().find("combine_bits_code") == 0) {
+                    codes_tail = h;
+                } else if (h.name().find("q5_struct_block_scale") == 0) {
+                    scale_tail = h;
+                }
+            }
+            _halide_internal_assert(min_leaf.defined() && scale_leaf.defined() &&
+                                    codes_tail.defined() && scale_tail.defined());
+
+            Func wt_product("q5_1_product_weight");
+            wt_product(kk, blk) = cast<float>(codes_leaf(kk, blk)) * scale_leaf(blk);
+            Func wt_product_tail("q5_1_product_weight_tail");
+            wt_product_tail(kk, blk) = cast<float>(codes_tail(kk, blk)) * scale_tail(blk);
+
+            RDom rd(0, bs, 0, main_blocks, "rd");
+            RDom rd_tail(0, bs, main_blocks, nblocks - main_blocks, "rd_tail");
+            Func dot_acc("q5_1_dot_acc");
+            dot_acc() = 0.0f;
+            dot_acc() += wt_product(rd.x, rd.y) * act_r.replacement(rd.x, rd.y);
+            dot_acc() += wt_product_tail(rd_tail.x, rd_tail.y) * actT_r.replacement(rd_tail.x, rd_tail.y);
+
+            const int lanes = 4;
+            RVar rxc("rxc"), rxr("rxr"), rxo("rxo"), rxi("rxi");
+            dot_acc.update(0).split(rd.x, rxc, rxr, 4 * lanes);
+            dot_acc.update(0).split(rxr, rxo, rxi, 4);
+            dot_acc.update(0).eager_inline(wt_product);
+            Var lane("lane");
+            std::vector<Func> dot_i32 = sdot_partial(dot_acc, {{rxo, lane}, {rd.y, u}},
+                                                     {act_r}, false, {codes_leaf});
+
+            RVar ryo("ryo"), ryi("ryi");
+            Var lv("lv"), bacc("bacc");
+            dot_acc.update(0).split(rd.y, ryo, ryi, unroll_blocks);
+            Func acc_vec = dot_acc.update(0).rfactor({{rxo, lv}, {ryi, bacc}});
+            acc_vec.compute_root().vectorize(lv, lanes).unroll(bacc);
+            acc_vec.update().vectorize(lv, lanes).unroll(bacc);
+            for (Func &part : dot_i32) {
+                part.compute_at(acc_vec, bacc)
+                    .update()
+                    .atomic()
+                    .vectorize(rxi, 4)
+                    .vectorize(lane, lanes)
+                    .unroll(rxc);
+            }
+            schedule_codes(LoopLevel(acc_vec, bacc));
+
+            Var lv2("lv2");
+            Func dot_lanes = dot_acc.update(0).rfactor(rxo, lv2);
+            dot_lanes.compute_root().vectorize(lv2, lanes);
+            dot_lanes.update().vectorize(lv2, lanes).unroll(ryi);
+            dot_acc.update(0).atomic().vectorize(rxo, lanes);
+            dot_acc.update(1).unscheduled();
+
+            RDom rb(0, main_blocks, "rb");
+            RDom rb_tail(main_blocks, nblocks - main_blocks, "rb_tail");
+            Func offset_acc("q5_1_offset_acc");
+            offset_acc() = 0.0f;
+            offset_acc() += min_leaf(rb) * cast<float>(s_blocks(rb));
+            offset_acc() += min_leaf(rb_tail) * cast<float>(s_blocks(rb_tail));
+
+            RVar rbi("rbi");
+            Var offset_bacc("offset_bacc");
+            offset_acc.update(0).split(rb.x, ryo, rbi, unroll_blocks);
+            Func offset_vec = offset_acc.update(0).rfactor(rbi, offset_bacc);
+            offset_vec.compute_root().unroll(offset_bacc);
+            offset_vec.update().unroll(offset_bacc);
+            offset_vec.update().compute_with(acc_vec.update(), ryo, LoopAlignStrategy::AlignStart);
+            offset_acc.update(0).unroll(rbi);
+            offset_acc.update(1).unscheduled();
+
+            Func combined("q5_1_combined_acc");
+            combined() = dot_acc() + offset_acc();
+            final_value = combined;
+        } else if (sever_sum) {
+            // Affine weight x Q8_1: the per-block product (d*code + m)*(d_act*act)
+            // distributes into d*d_act*sum(code*act) + m*d_act*sum(act). ggml does
+            // not recompute the second sum -- it reads the `s` field Q8_1 stores.
+            // We do the same by severing that term's accumulator to the stored
+            // field (the third Input, s_blocks), leaving only the Int(32) dot.
+            //
+            // rfactor to whole-block partials (variant A), inlining only the
+            // WEIGHT's decode chain so the activation decode (Act) stays whole:
+            // the offset term's accumulator is then sum_k Act(k, blk), which *is*
+            // the stored `s`. The product term re-inlines Act's full chain and
+            // re-hoists to recover the scale-free Int(32) dot.
+            Func acc_dot = Acc.update().rfactor({{r.y, u}});
+
+            std::vector<Func> winl = {wt_r.replacement};
+            for (const Func &h : wt_r.handles) {
+                // Keep the materialized codes leaf (Q5_1) out of the flatten, so
+                // its qh table read stays a per-block contiguous load.
+                if (h.function().can_be_inlined() &&
+                    !(codes_leaf.defined() && h.name() == codes_leaf.name())) {
+                    winl.push_back(h);
+                }
+            }
+            for (size_t pass = 0; pass < winl.size(); pass++) {
+                acc_dot.update().eager_inline(winl);
+            }
+
+            acc_dot.update().distribute();
+            std::vector<Func> parts = acc_dot.update().hoist_invariants();
+            // parts[0] = product term (scale * sum code_w*Act); parts[1] = offset
+            // term (min * sum Act == stored s).
+
+            // Sever the offset term to the stored fp16 field. change_type(Float16)
+            // makes the severed accumulator's type match the data (the encoder
+            // rounds `s` to fp16 too, so this reproduces ggml's own rounding);
+            // compute_offline then replaces every call to it with a read of
+            // s_blocks and discards the recomputing reduction.
+            Func s16 = parts[1].change_type(Float(16));
+            Pipeline({Acc}).compute_offline({s16}, {s_blocks});
+
+            // Product term: flatten the activation's full decode chain and
+            // re-hoist to pull d_act out, leaving the scale-free Int(32) dot.
+            std::vector<Func> ainl = {act_r.replacement};
+            for (const Func &h : act_r.handles) {
+                if (h.function().can_be_inlined()) {
+                    ainl.push_back(h);
+                }
+            }
+            for (size_t pass = 0; pass < ainl.size(); pass++) {
+                parts[0].update().eager_inline(ainl);
+            }
+            Func prod_i32 = parts[0].update().hoist_invariants()[0].change_type(Int(32));
+
+            RVar ryo("ryo"), ryi("ryi");
+            Var bacc("bacc");
+            Acc.update(0).split(r.y, ryo, ryi, unroll_blocks);
+            Func acc_vec = Acc.update(0).rfactor(ryi, bacc);
+            acc_vec.compute_root().unroll(bacc);
+            acc_vec.update().unroll(bacc);
+
+            // Only the product dot is computed per block now; the offset term is
+            // a severed read of s_blocks (nothing to schedule).
+            prod_i32.compute_at(acc_vec, bacc).update().atomic().vectorize(r.x, bs);
+            if (codes_leaf.defined()) {
+                schedule_codes(LoopLevel(acc_vec, bacc));
+            }
+            Acc.update(1).unscheduled();
+        } else if (spec.sched == ScheduleKind::SDOT && getenv("GGML_PER_BLOCK_PROBE")) {
+            // PROBE (variant A): rfactor only the block index, so both terms are
+            // whole-block reductions. Costs a horizontal reduce per block and a
+            // scalar cross-block accumulator; kept for measuring the non-sum
+            // formats against the lane-split default below.
+            std::vector<Func> parts = sdot_partial(Acc, {{r.y, u}}, {wt_r, act_r}, spec.distribute_terms);
+
+            RVar ryo("ryo"), ryi("ryi");
+            Var bacc("bacc");
+            Acc.update(0).split(r.y, ryo, ryi, unroll_blocks);
+            Func acc_vec = Acc.update(0).rfactor(ryi, bacc);
+            acc_vec.compute_root().unroll(bacc);
+            acc_vec.update().unroll(bacc);
+
+            for (Func &part : parts) {
+                part.compute_at(acc_vec, bacc).update().atomic().vectorize(r.x, bs);
+            }
+            Acc.update(1).unscheduled();
+        } else if (spec.sched == ScheduleKind::SDOT) {
+            // The reduction is over (within-block r.x) x (block r.y). The lanes
+            // of the accumulator come from r.x, so the sdot's four Int(32) lanes
+            // survive all the way into the float accumulator and no block pays
+            // for a horizontal reduce. They must come from r.x rather than r.y:
+            // blocks are interleaved {scale, codes} records, so a lane per block
+            // would gather both the codes and the scales, while a lane per
+            // within-block group keeps every code load contiguous.
+            //
+            // One sdot consumes 16 int8s and lands in a 4-lane Int(32) register,
+            // so r.x is cut three ways: chunks of 16 (one sdot each, run
+            // serially so they accumulate into the *same* register), then within
+            // a chunk a 4-wide lane index and the 4 elements the lane sums.
+            // Reducing straight to 4 lanes instead would make Halide lower the
+            // wide reduce as two independent sdots plus an addp to merge them --
+            // an extra zeroing and an extra reduction per block.
+            const int lanes = 4;
+            const int chunk = 4 * lanes;
+            RVar rxc("rxc"), rxr("rxr"), rxo("rxo"), rxi("rxi");
+            Acc.update(0).split(r.x, rxc, rxr, chunk);
+            Acc.update(0).split(rxr, rxo, rxi, 4);
+
+            // sdot_partial() flattens both operands' decode chains and hoists
+            // their per-block scales out of the surviving rxi reduction, leaving
+            // the scale-free Int(32) dot. See sdot_schedule.h.
+            Var lane("lane");
+            std::vector<Func> keep_out;
+            if (codes_leaf.defined()) {
+                keep_out.push_back(codes_leaf);
+            }
+            std::vector<Func> Acc_i32 = sdot_partial(Acc, {{rxo, lane}, {r.y, u}}, {wt_r, act_r}, spec.distribute_terms, keep_out);
+
+            // Acc's update now reduces over (rxo, r.y). Peel rxo back off as the
+            // vector lanes, and peel unroll_blocks consecutive blocks off
+            // alongside it into separate accumulators. The accumulators have to
+            // be split over *blocks*: every lane of one accumulator advances on
+            // every block, so widening the vector does not shorten the
+            // multiply-add chain, only interleaving blocks does. At ~3-4 cycles
+            // of accumulate latency, an un-interleaved chain is what bounds the
+            // whole kernel.
+            RVar ryo("ryo"), ryi("ryi");
+            Var lv("lv"), bacc("bacc");
+            Acc.update(0).split(r.y, ryo, ryi, unroll_blocks);
+            Func acc_vec = Acc.update(0).rfactor({{rxo, lv}, {ryi, bacc}});
+            acc_vec.compute_root().vectorize(lv, lanes).unroll(bacc);
+            acc_vec.update().vectorize(lv, lanes).unroll(bacc);
+
+            // Inside the unrolled body, not at the block-group loop: at `bacc` the
+            // sdot is one block's worth of registers, whereas at `ryo` it is a
+            // unroll_blocks-long buffer that Halide has to allocate, zero, and
+            // accumulate through memory.
+            for (Func &part : Acc_i32) {
+                part.compute_at(acc_vec, bacc)
+                    .update()
+                    .atomic()
+                    .vectorize(rxi, 4)
+                    .vectorize(lane, lanes)
+                    .unroll(rxc);
+            }
+            if (codes_leaf.defined()) {
+                schedule_codes(LoopLevel(acc_vec, bacc));
+            }
+
+            // Collapsing the lanes x unrolled-blocks accumulators is a fixed
+            // cost, but at the row lengths GGML uses it is not a negligible one:
+            // left alone it is a serial chain of lanes*unroll_blocks scalar
+            // adds. Sum the blocks vectorially first, then reduce the lanes
+            // horizontally, so it costs a handful of vector ops instead.
+            Var lv2("lv2");
+            Func acc_lanes = Acc.update(0).rfactor(rxo, lv2);
+            acc_lanes.compute_root().vectorize(lv2, lanes);
+            acc_lanes.update().vectorize(lv2, lanes).unroll(ryi);
+            Acc.update(0).atomic().vectorize(rxo, lanes);
+
+            // The odd-block tail deliberately keeps the default schedule.
+            Acc.update(1).unscheduled();
+        }
+        // ScheduleKind::Float: leave the reduction at its default (legal) schedule
+        // -- correctness first; an interleave/sub-block-aware performance schedule
+        // is a separate step.
+
+        Func result("result");
+        result() = final_value();
+
+        // A byte-path operand's block stride is pinned to its block width: these
+        // are densely packed GGML rows, and leaving the stride dynamic costs a
+        // serial pointer-add chain per block instead of an immediate offset.
+        if (wt_struct) {
+            x_blocks.dim(0).set_min(0);
+        } else {
+            x_blocks.dim(0).set_bounds(0, spec.weight_bytes);
+            x_blocks.dim(1).set_min(0).set_stride(spec.weight_bytes);
+        }
+        if (act_struct) {
+            y_blocks.dim(0).set_min(0);
+        } else {
+            y_blocks.dim(0).set_bounds(0, spec.act_bytes);
+            y_blocks.dim(1).set_min(0).set_stride(spec.act_bytes);
+        }
+        if (sever_sum) {
+            // A gathered view of the `s` slot within each packed block: the field
+            // repeats every act_bytes, so the fp16 stride is act_bytes/2 (18 for
+            // Q8_1's 36-byte block). Pinning it makes the per-block read a
+            // compile-time immediate offset rather than a dynamic pointer chain.
+            s_blocks.dim(0).set_min(0).set_stride(spec.act_bytes / 2);
+        }
+
+        this->add_input(x_blocks);
+        this->add_input(y_blocks);
+        if (sever_sum) {
+            this->add_input(s_blocks);
+        }
+        this->add_output(result);
+    }
+
+    void generate() {
+        // configure() built the whole pipeline (add_input/add_output included).
+    }
+};
+
+}  // namespace ggml_halide
diff --git a/apps/ggml/include/kernel_registry.h b/apps/ggml/include/kernel_registry.h
new file mode 100644
index 000000000000..34ce7d3acb1b
--- /dev/null
+++ b/apps/ggml/include/kernel_registry.h
@@ -0,0 +1,156 @@
+#pragma once
+
+// Implementation-agnostic core of kernel-bench.
+//
+// A "kernel" here is identified by a ggml_type plus a category (quantize,
+// dequantize, vec_dot, repack quantize_mat/gemv/gemm). For each (category,
+// type) pair, exactly one *reference* implementation may be registered --
+// this is the correctness ground truth and baseline timing that every other
+// *candidate* implementation for that (category, type) is measured against.
+//
+// GGML's own routines are registered as the reference by providers/ggml_provider.cpp.
+// Nothing in this file knows anything about GGML internals: a future provider
+// (e.g. the user's own reference implementation) is just another call to
+// register_candidate() (or register_reference(), if it should replace GGML's
+// as the ground truth) with a function pointer matching the category's
+// signature. See providers/README.md for how to add one.
+
+#include <cstddef>
+#include <cstdint>
+#include <map>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include <ggml.h>
+
+// Function-pointer shapes shared by every provider. These mirror the layouts
+// used throughout ggml-cpu (quantize_row_*, ggml_vec_dot_*, ggml_gemv_*/ggml_gemm_*)
+// so that both GGML's own symbols and a from-scratch implementation can be
+// registered without adapters.
+using quantize_fn_t = void (*)(const float *GGML_RESTRICT x, void *GGML_RESTRICT y, int64_t k);
+using dequantize_fn_t = void (*)(const void *GGML_RESTRICT x, float *GGML_RESTRICT y, int64_t k);
+using vec_dot_fn_t = void (*)(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx,
+                              const void *GGML_RESTRICT vy, size_t by, int nrc);
+using gemx_fn_t = void (*)(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx,
+                           const void *GGML_RESTRICT vy, int nr, int nc);
+
+template<typename Fn>
+struct Impl {
+    std::string name;
+    Fn fn;
+};
+
+template<typename Fn>
+class Registry {
+public:
+    // Exactly one reference per type. Calling this twice for the same type
+    // replaces the previous reference (last call wins) -- useful if a later
+    // provider should become the new ground truth for that type.
+    void register_reference(ggml_type type, std::string name, Fn fn) {
+        entries_[type].reference = Impl<Fn>{std::move(name), fn};
+    }
+
+    // Zero or more per type.
+    void register_candidate(ggml_type type, std::string name, Fn fn) {
+        entries_[type].candidates.push_back(Impl<Fn>{std::move(name), fn});
+    }
+
+    const Impl<Fn> *reference(ggml_type type) const {
+        auto it = entries_.find(type);
+        if (it == entries_.end() || !it->second.reference.has_value()) {
+            return nullptr;
+        }
+        return &*it->second.reference;
+    }
+
+    const std::vector<Impl<Fn>> &candidates(ggml_type type) const {
+        static const std::vector<Impl<Fn>> empty;
+        auto it = entries_.find(type);
+        return it == entries_.end() ? empty : it->second.candidates;
+    }
+
+    std::vector<ggml_type> types_with_reference() const {
+        std::vector<ggml_type> out;
+        for (const auto &[type, entry] : entries_) {
+            if (entry.reference.has_value()) {
+                out.push_back(type);
+            }
+        }
+        return out;
+    }
+
+private:
+    struct Entry {
+        std::optional<Impl<Fn>> reference;
+        std::vector<Impl<Fn>> candidates;
+    };
+    std::map<ggml_type, Entry> entries_;
+};
+
+// Repack kernels are additionally keyed by the activation (vec_dot_type)
+// they were interleaved against, and by the interleave geometry -- carried
+// alongside the ggml_type key as a small identifying suffix (e.g. "4x4",
+// "8x8") so multiple repack variants can coexist for the same base type.
+struct RepackKey {
+    ggml_type base_type;
+    ggml_type act_type;
+    int inter_size;
+    int nb_cols;
+    std::string label;  // e.g. "q4_0_4x4_q8_0", used for display and as a stable map key
+
+    bool operator<(const RepackKey &other) const {
+        return label < other.label;
+    }
+};
+
+template<typename Fn>
+class RepackRegistry {
+public:
+    void register_reference(const RepackKey &key, std::string name, Fn fn) {
+        entries_[key.label].key = key;
+        entries_[key.label].reference = Impl<Fn>{std::move(name), fn};
+    }
+    void register_candidate(const RepackKey &key, std::string name, Fn fn) {
+        entries_[key.label].key = key;
+        entries_[key.label].candidates.push_back(Impl<Fn>{std::move(name), fn});
+    }
+    const Impl<Fn> *reference(const std::string &label) const {
+        auto it = entries_.find(label);
+        if (it == entries_.end() || !it->second.reference.has_value()) {
+            return nullptr;
+        }
+        return &*it->second.reference;
+    }
+    const std::vector<Impl<Fn>> &candidates(const std::string &label) const {
+        static const std::vector<Impl<Fn>> empty;
+        auto it = entries_.find(label);
+        return it == entries_.end() ? empty : it->second.candidates;
+    }
+    std::vector<RepackKey> keys() const {
+        std::vector<RepackKey> out;
+        for (const auto &[label, entry] : entries_) {
+            if (entry.reference.has_value()) {
+                out.push_back(entry.key);
+            }
+        }
+        return out;
+    }
+
+private:
+    struct Entry {
+        RepackKey key{};
+        std::optional<Impl<Fn>> reference;
+        std::vector<Impl<Fn>> candidates;
+    };
+    std::map<std::string, Entry> entries_;
+};
+
+struct KernelRegistries {
+    Registry<quantize_fn_t> quantize;
+    Registry<dequantize_fn_t> dequantize;
+    Registry<vec_dot_fn_t> vec_dot;
+    RepackRegistry<quantize_fn_t> repack_quantize_mat;
+    RepackRegistry<gemx_fn_t> repack_gemv;
+    RepackRegistry<gemx_fn_t> repack_gemm;
+};
diff --git a/apps/ggml/providers/README.md b/apps/ggml/providers/README.md
new file mode 100644
index 000000000000..279d8bdbf775
--- /dev/null
+++ b/apps/ggml/providers/README.md
@@ -0,0 +1,60 @@
+# Adding a provider
+
+A "provider" is anything that registers one or more implementations into a
+`KernelRegistries` (see `include/kernel_registry.h`). `ggml_provider.cpp` is the
+provider shipped today; it is the *only* file that knows GGML's internal symbols
+exist. A new provider -- e.g. your own from-scratch reference implementation,
+starting with dequantize per the project's stated goal -- is just another
+translation unit that does the same thing.
+
+## Steps
+
+1. Create `providers/<name>_provider.h` declaring one function:
+
+   ```cpp
+   void register_<name>_provider(KernelRegistries & registries);
+   ```
+
+2. Create `providers/<name>_provider.cpp` implementing it. For each
+   `(ggml_type, implementation)` pair you want benchmarked, call:
+
+   ```cpp
+   registries.dequantize.register_candidate(GGML_TYPE_Q4_0, "my-dequant", my_dequantize_q4_0);
+   ```
+
+   matching the category's function-pointer typedef from `kernel_registry.h`:
+
+   | category                          | typedef           | signature                                                                                     |
+   | --------------------------------- | ----------------- | --------------------------------------------------------------------------------------------- |
+   | `quantize`, `repack_quantize_mat` | `quantize_fn_t`   | `(const float* x, void* y, int64_t k)`                                                        |
+   | `dequantize`                      | `dequantize_fn_t` | `(const void* x, float* y, int64_t k)`                                                        |
+   | `vec_dot`                         | `vec_dot_fn_t`    | `(int n, float* s, size_t bs, const void* vx, size_t bx, const void* vy, size_t by, int nrc)` |
+   | `repack_gemv`, `repack_gemm`      | `gemx_fn_t`       | `(int n, float* s, size_t bs, const void* vx, const void* vy, int nr, int nc)`                |
+
+   Use `register_candidate()` if GGML's existing reference should remain the
+   correctness ground truth for that type (the common case: you want to see
+   whether your implementation agrees with GGML and how fast it is). Use
+   `register_reference()` instead if your implementation should *become* the new
+   ground truth other candidates are compared against for that type -- the
+   harness doesn't care which provider a reference comes from.
+
+3. Add one line to `src/main.cpp`:
+
+   ```cpp
+   register_<name>_provider(registries);
+   ```
+
+   next to the existing `register_ggml_provider(registries);` call. Nothing else
+   changes -- `bench_*.cpp`, the CLI, and the reporting code iterate whatever
+   ends up in the registries and don't know or care how many providers
+   contributed to them.
+
+## Repack keys
+
+`repack_quantize_mat`/`repack_gemv`/`repack_gemm` are keyed by `RepackKey` (base
+type, activation type, interleave geometry, label string) rather than by
+`ggml_type` alone, since several interleaved weight layouts can exist for the
+same base type (e.g. `q4_0_4x4_q8_0` vs `q4_0_8x8_q8_0`). Reuse the `RepackKey`
+values already registered by `ggml_provider.cpp` (see `k_repack_entries` in
+`ggml_provider.cpp`) if you're providing an alternative gemv/gemm for an
+existing layout; define your own `RepackKey` if you're introducing a new one.
diff --git a/apps/ggml/providers/ggml_internal_abi.h b/apps/ggml/providers/ggml_internal_abi.h
new file mode 100644
index 000000000000..a53ebf1bd905
--- /dev/null
+++ b/apps/ggml/providers/ggml_internal_abi.h
@@ -0,0 +1,295 @@
+#pragma once
+
+// PRIVATE, VERSION-PINNED ABI SURFACE -- READ BEFORE TOUCHING
+//
+// The functions declared below are internal implementation details of
+// ggml-cpu (declared in the *uninstalled* headers src/ggml-cpu/quants.h and
+// src/ggml-cpu/repack.h). GGML does not install those headers, does not
+// document these symbols, does not version them, and offers no ABI
+// stability guarantee for them whatsoever.
+//
+// They are reachable from an external application ONLY because ggml-cpu is
+// built without -fvisibility=hidden: every plain, non-static C function ends
+// up with default (exported) linker visibility by accident of the build
+// configuration, not by design. This header is a hand-copied snapshot of
+// the declarations in ggml (as of the commit this file was written against;
+// see README.md) -- if a future GGML release renames, removes, or changes
+// the signature of one of these functions, this header (and only this
+// header + ggml_provider.cpp) will need updating. No other part of
+// kernel-bench depends on GGML internals.
+//
+// Why we need this at all: GGML's public API (ggml_get_type_traits /
+// ggml_get_type_traits_cpu, see include/ggml.h and include/ggml-cpu.h)
+// exposes exactly one "reference" and one "dispatched" implementation per
+// type for quantize/dequantize, which is enough for those two categories
+// without touching anything private (see ggml_provider.cpp). It does NOT
+// expose the always-available pure-C fallback for vec_dot or for the repack
+// quantize_mat/gemv/gemm kernels -- the only way to reach those, and thus
+// the only way to compare them against the (possibly arch-optimized)
+// canonical symbol, is by declaring both names ourselves and letting the
+// linker resolve them.
+//
+// IMPORTANT -- the `_generic` name does not always exist as its own link
+// symbol: src/ggml-cpu/arch-fallback.h #defines it onto the canonical name
+// -- as a textual macro substitution inside GGML's own .c/.cpp files -- for
+// whichever functions the current architecture has no distinct optimized
+// version of, and there is then only one function in the binary. (A weak
+// C++ declaration doesn't paper over this: verified empirically that
+// Darwin's ld64 hard-fails on an undefined `weak_import` symbol that has
+// zero definitions anywhere in the link, and GNU ld's "resolve undefined
+// weak to null" behavior isn't something to rely on portably either.) So
+// this header mirrors arch-fallback.h's own collapsing, using the same
+// preprocessor guards, restricted to the subset of functions declared
+// below. When a `_generic` name collapses onto its canonical counterpart
+// here exactly as it does inside GGML itself, ggml_provider.cpp's
+// pointer-equality check (`generic_fn == canonical_fn`) naturally detects
+// "single implementation, nothing to compare" for that kernel on this
+// architecture. If GGML's own arch-fallback.h changes its collapsing list,
+// this block needs updating to match; this is the one part of this file
+// most likely to need attention when moving to a newer GGML.
+#if defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64)
+#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
+#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8
+#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
+#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0
+#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K
+#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
+#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0
+#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K
+#elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)
+#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0
+#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4
+#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
+#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0
+#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0
+#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K
+#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K
+#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K
+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K
+#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
+#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
+#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0
+#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0
+#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0
+#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0
+#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0
+#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K
+#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K
+#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K
+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K
+#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
+#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
+#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0
+#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0
+#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0
+#elif defined(__POWERPC__) || defined(__powerpc__) || defined(__loongarch64) || defined(__riscv) || \
+    defined(__s390x__) || defined(__wasm__)
+// PowerPC/LoongArch/RISC-V/s390x/wasm each collapse a large, differently-shaped
+// subset of quants.c/repack.cpp symbols (see arch-fallback.h) -- rather than
+// transcribe five more per-arch lists by hand, collapse everything this
+// header declares on these architectures. This is conservative in the safe
+// direction: on an arch that actually kept a real optimized/generic split
+// for some function, this makes that pairing look like "single
+// implementation" instead of reporting it, but it will never misreport two
+// genuinely different implementations as identical, and it will never
+// produce a link error.
+#define quantize_row_q8_K_generic quantize_row_q8_K
+#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0
+#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0
+#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0
+#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K
+#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K
+#define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K
+#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K
+#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K
+#define ggml_vec_dot_iq2_s_q8_K_generic ggml_vec_dot_iq2_s_q8_K
+#define ggml_vec_dot_iq3_xxs_q8_K_generic ggml_vec_dot_iq3_xxs_q8_K
+#define ggml_vec_dot_iq3_s_q8_K_generic ggml_vec_dot_iq3_s_q8_K
+#define ggml_vec_dot_iq1_s_q8_K_generic ggml_vec_dot_iq1_s_q8_K
+#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K
+#define ggml_vec_dot_iq4_nl_q8_0_generic ggml_vec_dot_iq4_nl_q8_0
+#define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K
+#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4
+#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8
+#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
+#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8
+#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0
+#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0
+#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0
+#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K
+#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K
+#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K
+#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K
+#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K
+#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K
+#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
+#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
+#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
+#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0
+#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0
+#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0
+#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0
+#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0
+#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0
+#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0
+#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K
+#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K
+#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K
+#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K
+#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K
+#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K
+#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
+#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
+#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
+#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0
+#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0
+#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0
+#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0
+#endif
+
+#include <cstddef>
+#include <cstdint>
+
+#include <ggml-backend.h>  // ggml_backend_buffer_type_t
+#include <ggml.h>          // GGML_RESTRICT
+
+// NOTE: declared with ordinary C++ (mangled) linkage, matching the real
+// src/ggml-cpu/repack.h -- this one declaration sits *before* that header's
+// `extern "C" { ... }` block, unlike every quantize_row_*/vec_dot_*/gemv/gemm
+// declaration below.
+ggml_backend_buffer_type_t ggml_backend_cpu_repack_buffer_type(void);
+
+extern "C" {
+
+// -- src/ggml-cpu/quants.h: pure-C reference quantizer for Q8_K. Unlike
+// every other quantized type, GGML_TYPE_Q8_K has no `from_float_ref` in the
+// public ggml_get_type_traits() table (src/ggml.c) at all -- Q8_K is purely
+// an internal activation format for K-quant vec_dot/gemv/gemm, never a
+// row-conversion target -- so it needs this private symbol as its
+// reference; see the special case in ggml_provider.cpp.
+void quantize_row_q8_K_generic(const float *GGML_RESTRICT x, void *GGML_RESTRICT y, int64_t k);
+
+// -- src/ggml-quants.h: whole-matrix quantizers for the importance-matrix-
+// only codebook types (IQ2_XXS, IQ2_XS, IQ1_S, IQ1_M). Unlike every other
+// quantized type, these have no `from_float_ref` in the public
+// ggml_get_type_traits() table at all -- GGML only exposes them through
+// this differently-shaped `(src, dst, nrows, n_per_row, imatrix)` signature
+// (nrows/n_per_row instead of a flat element count k, and an optional
+// importance-matrix pointer), used only by the model-quantization tool.
+// Called here with nrows=1, n_per_row=k, imatrix=nullptr to get a plain
+// per-row reference, matching the shape every other type's from_float_ref
+// already has; see the special case in ggml_provider.cpp. Declared to
+// return size_t per GGML's real signature (the number of bytes written).
+size_t quantize_iq2_xxs(const float *GGML_RESTRICT src, void *GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float *GGML_RESTRICT imatrix);
+size_t quantize_iq2_xs(const float *GGML_RESTRICT src, void *GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float *GGML_RESTRICT imatrix);
+size_t quantize_iq1_s(const float *GGML_RESTRICT src, void *GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float *GGML_RESTRICT imatrix);
+size_t quantize_iq1_m(const float *GGML_RESTRICT src, void *GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float *GGML_RESTRICT imatrix);
+
+// -- src/ggml-cpu/quants.h: vec_dot, pure-C reference (collapsed onto the
+// canonical, arch-dispatched symbol above on architectures with no distinct
+// optimized implementation for that type) --
+void ggml_vec_dot_q1_0_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q4_0_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q4_1_q8_1_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q5_0_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q5_1_q8_1_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q8_0_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_mxfp4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_nvfp4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_tq1_0_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_tq2_0_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q2_K_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q3_K_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q4_K_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q5_K_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q6_K_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq2_xxs_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq2_xs_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq2_s_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq3_xxs_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq3_s_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq1_s_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq1_m_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq4_nl_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq4_xs_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, size_t bx, const void *GGML_RESTRICT vy, size_t by, int nrc);
+
+// -- src/ggml-cpu/repack.h: activation packing (float -> interleaved q8 blocks), canonical + reference --
+void ggml_quantize_mat_q8_0_4x4(const float *GGML_RESTRICT x, void *GGML_RESTRICT vy, int64_t k);
+void ggml_quantize_mat_q8_0_4x8(const float *GGML_RESTRICT x, void *GGML_RESTRICT vy, int64_t k);
+void ggml_quantize_mat_q8_K_4x4(const float *GGML_RESTRICT x, void *GGML_RESTRICT vy, int64_t k);
+void ggml_quantize_mat_q8_K_4x8(const float *GGML_RESTRICT x, void *GGML_RESTRICT vy, int64_t k);
+void ggml_quantize_mat_q8_0_4x4_generic(const float *GGML_RESTRICT x, void *GGML_RESTRICT vy, int64_t k);
+void ggml_quantize_mat_q8_0_4x8_generic(const float *GGML_RESTRICT x, void *GGML_RESTRICT vy, int64_t k);
+void ggml_quantize_mat_q8_K_4x4_generic(const float *GGML_RESTRICT x, void *GGML_RESTRICT vy, int64_t k);
+void ggml_quantize_mat_q8_K_4x8_generic(const float *GGML_RESTRICT x, void *GGML_RESTRICT vy, int64_t k);
+
+// -- src/ggml-cpu/repack.h: gemv/gemm over packed weight blocks, canonical + reference --
+void ggml_gemv_q4_0_4x4_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_0_4x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_0_8x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q2_K_8x8_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_K_8x4_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_K_8x8_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q5_K_8x4_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q5_K_8x8_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q6_K_8x4_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q6_K_8x8_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_iq4_nl_4x4_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_iq4_nl_8x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_mxfp4_4x4_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_mxfp4_8x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q8_0_4x4_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q8_0_4x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+
+void ggml_gemm_q4_0_4x4_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_0_4x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_0_8x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q2_K_8x8_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_K_8x4_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_K_8x8_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q5_K_8x4_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q5_K_8x8_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q6_K_8x4_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q6_K_8x8_q8_K(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_iq4_nl_4x4_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_iq4_nl_8x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_mxfp4_4x4_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_mxfp4_8x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q8_0_4x4_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q8_0_4x8_q8_0(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+
+void ggml_gemv_q4_0_4x4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_0_4x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_0_8x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q2_K_8x8_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_K_8x4_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_K_8x8_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q5_K_8x4_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q5_K_8x8_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q6_K_8x4_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q6_K_8x8_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_iq4_nl_4x4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_iq4_nl_8x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_mxfp4_4x4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_mxfp4_8x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q8_0_4x4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q8_0_4x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+
+void ggml_gemm_q4_0_4x4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_0_4x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_0_8x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q2_K_8x8_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_K_8x4_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_K_8x8_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q5_K_8x4_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q5_K_8x8_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q6_K_8x4_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q6_K_8x8_q8_K_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_iq4_nl_4x4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_iq4_nl_8x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_mxfp4_4x4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_mxfp4_8x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q8_0_4x4_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q8_0_4x8_q8_0_generic(int n, float *GGML_RESTRICT s, size_t bs, const void *GGML_RESTRICT vx, const void *GGML_RESTRICT vy, int nr, int nc);
+
+}  // extern "C"
diff --git a/apps/ggml/providers/ggml_provider.cpp b/apps/ggml/providers/ggml_provider.cpp
new file mode 100644
index 000000000000..928233cdf13f
--- /dev/null
+++ b/apps/ggml/providers/ggml_provider.cpp
@@ -0,0 +1,293 @@
+#include "ggml_provider.h"
+#include "ggml_internal_abi.h"
+
+#include <ggml-cpu.h>
+#include <ggml.h>
+
+#include <vector>
+
+namespace {
+
+// type -> pure-C reference vec_dot (src/ggml-cpu/quants.h `_generic` symbols).
+// The canonical (possibly arch-optimized) candidate is obtained separately,
+// through the PUBLIC ggml_get_type_traits_cpu(type)->vec_dot.
+struct VecDotRef {
+    ggml_type type;
+    vec_dot_fn_t fn;
+};
+
+const VecDotRef k_vec_dot_refs[] = {
+    {GGML_TYPE_Q1_0, ggml_vec_dot_q1_0_q8_0_generic},
+    {GGML_TYPE_Q4_0, ggml_vec_dot_q4_0_q8_0_generic},
+    {GGML_TYPE_Q4_1, ggml_vec_dot_q4_1_q8_1_generic},
+    {GGML_TYPE_Q5_0, ggml_vec_dot_q5_0_q8_0_generic},
+    {GGML_TYPE_Q5_1, ggml_vec_dot_q5_1_q8_1_generic},
+    {GGML_TYPE_Q8_0, ggml_vec_dot_q8_0_q8_0_generic},
+    {GGML_TYPE_MXFP4, ggml_vec_dot_mxfp4_q8_0_generic},
+    {GGML_TYPE_NVFP4, ggml_vec_dot_nvfp4_q8_0_generic},
+    {GGML_TYPE_Q2_K, ggml_vec_dot_q2_K_q8_K_generic},
+    {GGML_TYPE_Q3_K, ggml_vec_dot_q3_K_q8_K_generic},
+    {GGML_TYPE_Q4_K, ggml_vec_dot_q4_K_q8_K_generic},
+    {GGML_TYPE_Q5_K, ggml_vec_dot_q5_K_q8_K_generic},
+    {GGML_TYPE_Q6_K, ggml_vec_dot_q6_K_q8_K_generic},
+    {GGML_TYPE_TQ1_0, ggml_vec_dot_tq1_0_q8_K_generic},
+    {GGML_TYPE_TQ2_0, ggml_vec_dot_tq2_0_q8_K_generic},
+    {GGML_TYPE_IQ2_XXS, ggml_vec_dot_iq2_xxs_q8_K_generic},
+    {GGML_TYPE_IQ2_XS, ggml_vec_dot_iq2_xs_q8_K_generic},
+    {GGML_TYPE_IQ2_S, ggml_vec_dot_iq2_s_q8_K_generic},
+    {GGML_TYPE_IQ3_XXS, ggml_vec_dot_iq3_xxs_q8_K_generic},
+    {GGML_TYPE_IQ3_S, ggml_vec_dot_iq3_s_q8_K_generic},
+    {GGML_TYPE_IQ1_S, ggml_vec_dot_iq1_s_q8_K_generic},
+    {GGML_TYPE_IQ1_M, ggml_vec_dot_iq1_m_q8_K_generic},
+    {GGML_TYPE_IQ4_NL, ggml_vec_dot_iq4_nl_q8_0_generic},
+    {GGML_TYPE_IQ4_XS, ggml_vec_dot_iq4_xs_q8_K_generic},
+};
+
+// The 9 repack combinations enumerated in
+// ggml_repack_get_optimal_repack_type() (src/ggml-cpu/repack.cpp:4528-4560).
+struct RepackEntry {
+    RepackKey key;
+    quantize_fn_t quantize_mat;
+    quantize_fn_t quantize_mat_generic;
+    gemx_fn_t gemv;
+    gemx_fn_t gemv_generic;
+    gemx_fn_t gemm;
+    gemx_fn_t gemm_generic;
+};
+
+const RepackEntry k_repack_entries[] = {
+    {{GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 4, 4, "q4_0_4x4_q8_0"},
+     ggml_quantize_mat_q8_0_4x4,
+     ggml_quantize_mat_q8_0_4x4_generic,
+     ggml_gemv_q4_0_4x4_q8_0,
+     ggml_gemv_q4_0_4x4_q8_0_generic,
+     ggml_gemm_q4_0_4x4_q8_0,
+     ggml_gemm_q4_0_4x4_q8_0_generic},
+    {{GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 8, 4, "q4_0_4x8_q8_0"},
+     ggml_quantize_mat_q8_0_4x8,
+     ggml_quantize_mat_q8_0_4x8_generic,
+     ggml_gemv_q4_0_4x8_q8_0,
+     ggml_gemv_q4_0_4x8_q8_0_generic,
+     ggml_gemm_q4_0_4x8_q8_0,
+     ggml_gemm_q4_0_4x8_q8_0_generic},
+    {{GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 8, 8, "q4_0_8x8_q8_0"},
+     ggml_quantize_mat_q8_0_4x8,
+     ggml_quantize_mat_q8_0_4x8_generic,
+     ggml_gemv_q4_0_8x8_q8_0,
+     ggml_gemv_q4_0_8x8_q8_0_generic,
+     ggml_gemm_q4_0_8x8_q8_0,
+     ggml_gemm_q4_0_8x8_q8_0_generic},
+    {{GGML_TYPE_Q4_K, GGML_TYPE_Q8_K, 8, 4, "q4_K_8x4_q8_K"},
+     ggml_quantize_mat_q8_K_4x4,
+     ggml_quantize_mat_q8_K_4x4_generic,
+     ggml_gemv_q4_K_8x4_q8_K,
+     ggml_gemv_q4_K_8x4_q8_K_generic,
+     ggml_gemm_q4_K_8x4_q8_K,
+     ggml_gemm_q4_K_8x4_q8_K_generic},
+    {{GGML_TYPE_Q4_K, GGML_TYPE_Q8_K, 8, 8, "q4_K_8x8_q8_K"},
+     ggml_quantize_mat_q8_K_4x8,
+     ggml_quantize_mat_q8_K_4x8_generic,
+     ggml_gemv_q4_K_8x8_q8_K,
+     ggml_gemv_q4_K_8x8_q8_K_generic,
+     ggml_gemm_q4_K_8x8_q8_K,
+     ggml_gemm_q4_K_8x8_q8_K_generic},
+    {{GGML_TYPE_Q5_K, GGML_TYPE_Q8_K, 8, 4, "q5_K_8x4_q8_K"},
+     ggml_quantize_mat_q8_K_4x4,
+     ggml_quantize_mat_q8_K_4x4_generic,
+     ggml_gemv_q5_K_8x4_q8_K,
+     ggml_gemv_q5_K_8x4_q8_K_generic,
+     ggml_gemm_q5_K_8x4_q8_K,
+     ggml_gemm_q5_K_8x4_q8_K_generic},
+    {{GGML_TYPE_Q5_K, GGML_TYPE_Q8_K, 8, 8, "q5_K_8x8_q8_K"},
+     ggml_quantize_mat_q8_K_4x8,
+     ggml_quantize_mat_q8_K_4x8_generic,
+     ggml_gemv_q5_K_8x8_q8_K,
+     ggml_gemv_q5_K_8x8_q8_K_generic,
+     ggml_gemm_q5_K_8x8_q8_K,
+     ggml_gemm_q5_K_8x8_q8_K_generic},
+    {{GGML_TYPE_Q6_K, GGML_TYPE_Q8_K, 8, 4, "q6_K_8x4_q8_K"},
+     ggml_quantize_mat_q8_K_4x4,
+     ggml_quantize_mat_q8_K_4x4_generic,
+     ggml_gemv_q6_K_8x4_q8_K,
+     ggml_gemv_q6_K_8x4_q8_K_generic,
+     ggml_gemm_q6_K_8x4_q8_K,
+     ggml_gemm_q6_K_8x4_q8_K_generic},
+    {{GGML_TYPE_Q6_K, GGML_TYPE_Q8_K, 8, 8, "q6_K_8x8_q8_K"},
+     ggml_quantize_mat_q8_K_4x8,
+     ggml_quantize_mat_q8_K_4x8_generic,
+     ggml_gemv_q6_K_8x8_q8_K,
+     ggml_gemv_q6_K_8x8_q8_K_generic,
+     ggml_gemm_q6_K_8x8_q8_K,
+     ggml_gemm_q6_K_8x8_q8_K_generic},
+    {{GGML_TYPE_Q2_K, GGML_TYPE_Q8_K, 8, 8, "q2_K_8x8_q8_K"},
+     ggml_quantize_mat_q8_K_4x8,
+     ggml_quantize_mat_q8_K_4x8_generic,
+     ggml_gemv_q2_K_8x8_q8_K,
+     ggml_gemv_q2_K_8x8_q8_K_generic,
+     ggml_gemm_q2_K_8x8_q8_K,
+     ggml_gemm_q2_K_8x8_q8_K_generic},
+    {{GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0, 4, 4, "iq4_nl_4x4_q8_0"},
+     ggml_quantize_mat_q8_0_4x4,
+     ggml_quantize_mat_q8_0_4x4_generic,
+     ggml_gemv_iq4_nl_4x4_q8_0,
+     ggml_gemv_iq4_nl_4x4_q8_0_generic,
+     ggml_gemm_iq4_nl_4x4_q8_0,
+     ggml_gemm_iq4_nl_4x4_q8_0_generic},
+    {{GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0, 8, 8, "iq4_nl_8x8_q8_0"},
+     ggml_quantize_mat_q8_0_4x8,
+     ggml_quantize_mat_q8_0_4x8_generic,
+     ggml_gemv_iq4_nl_8x8_q8_0,
+     ggml_gemv_iq4_nl_8x8_q8_0_generic,
+     ggml_gemm_iq4_nl_8x8_q8_0,
+     ggml_gemm_iq4_nl_8x8_q8_0_generic},
+    {{GGML_TYPE_MXFP4, GGML_TYPE_Q8_0, 4, 4, "mxfp4_4x4_q8_0"},
+     ggml_quantize_mat_q8_0_4x4,
+     ggml_quantize_mat_q8_0_4x4_generic,
+     ggml_gemv_mxfp4_4x4_q8_0,
+     ggml_gemv_mxfp4_4x4_q8_0_generic,
+     ggml_gemm_mxfp4_4x4_q8_0,
+     ggml_gemm_mxfp4_4x4_q8_0_generic},
+    {{GGML_TYPE_MXFP4, GGML_TYPE_Q8_0, 8, 8, "mxfp4_8x8_q8_0"},
+     ggml_quantize_mat_q8_0_4x8,
+     ggml_quantize_mat_q8_0_4x8_generic,
+     ggml_gemv_mxfp4_8x8_q8_0,
+     ggml_gemv_mxfp4_8x8_q8_0_generic,
+     ggml_gemm_mxfp4_8x8_q8_0,
+     ggml_gemm_mxfp4_8x8_q8_0_generic},
+    {{GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, 4, 4, "q8_0_4x4_q8_0"},
+     ggml_quantize_mat_q8_0_4x4,
+     ggml_quantize_mat_q8_0_4x4_generic,
+     ggml_gemv_q8_0_4x4_q8_0,
+     ggml_gemv_q8_0_4x4_q8_0_generic,
+     ggml_gemm_q8_0_4x4_q8_0,
+     ggml_gemm_q8_0_4x4_q8_0_generic},
+    {{GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, 8, 4, "q8_0_4x8_q8_0"},
+     ggml_quantize_mat_q8_0_4x8,
+     ggml_quantize_mat_q8_0_4x8_generic,
+     ggml_gemv_q8_0_4x8_q8_0,
+     ggml_gemv_q8_0_4x8_q8_0_generic,
+     ggml_gemm_q8_0_4x8_q8_0,
+     ggml_gemm_q8_0_4x8_q8_0_generic},
+};
+
+// Thin adapters from GGML's whole-matrix `(src, dst, nrows, n_per_row,
+// imatrix)` quantizer signature (see ggml_internal_abi.h) down to
+// quantize_fn_t's flat `(x, y, k)` shape, matching what every other type's
+// from_float_ref already looks like: one row, no importance weighting. All
+// four of these implementations actually require a non-null quant_weights
+// pointer (a real GGML_ASSERT for IQ2_XXS/IQ2_XS/IQ1_S; commented out but
+// still exercised for IQ1_M) -- a uniform (all-1.0) weighting is passed so
+// every element is treated as equally important, the closest equivalent to
+// "no importance weighting" these quantizers support.
+void quantize_iq2_xxs_row(const float *x, void *y, int64_t k) {
+    std::vector<float> w(k, 1.0f);
+    quantize_iq2_xxs(x, y, 1, k, w.data());
+}
+void quantize_iq2_xs_row(const float *x, void *y, int64_t k) {
+    std::vector<float> w(k, 1.0f);
+    quantize_iq2_xs(x, y, 1, k, w.data());
+}
+void quantize_iq1_s_row(const float *x, void *y, int64_t k) {
+    std::vector<float> w(k, 1.0f);
+    quantize_iq1_s(x, y, 1, k, w.data());
+}
+void quantize_iq1_m_row(const float *x, void *y, int64_t k) {
+    std::vector<float> w(k, 1.0f);
+    quantize_iq1_m(x, y, 1, k, w.data());
+}
+
+}  // namespace
+
+void register_ggml_provider(KernelRegistries &registries) {
+    ggml_cpu_init();
+
+    // -- quantize / dequantize: fully public API --
+    for (int t = 0; t < GGML_TYPE_COUNT; ++t) {
+        const ggml_type type = static_cast<ggml_type>(t);
+        const ggml_type_traits *tt = ggml_get_type_traits(type);
+        const ggml_type_traits_cpu *tc = ggml_get_type_traits_cpu(type);
+        if (!tt || !tc) {
+            continue;
+        }
+        // Deliberately NOT calling ggml_quantize_init(type) here: for
+        // IQ2_XXS/IQ2_XS/IQ2_S/IQ1_S/IQ1_M/IQ3_XXS/IQ3_S it builds a nearest-
+        // neighbor lookup table via an O(43692 * grid_size) search with a
+        // qsort per row (src/ggml-quants.c, iq2xs_init_impl/iq3xs_init_impl)
+        // -- genuinely slow (hundreds of ms), and doing it here for all 42
+        // types unconditionally at registration time means paying for it
+        // before any benchmark has printed a single row, even for runs
+        // (e.g. --repack) that never touch these types at all. Each bench_*.cpp
+        // calls it lazily, once, right before it first actually invokes one
+        // of these types' quantize/dequantize/vec_dot functions instead.
+
+        if (tt->from_float_ref) {
+            registries.quantize.register_reference(type, "ggml-ref", tt->from_float_ref);
+            if (tc->from_float) {
+                registries.quantize.register_candidate(type, "ggml-cpu", tc->from_float);
+            }
+        }
+        if (tt->to_float) {
+            registries.dequantize.register_reference(type, "ggml-ref", tt->to_float);
+            // No candidate registered yet: GGML has exactly one dequantize
+            // implementation per type (src/ggml-quants.c, arch-independent).
+            // This is where a from-scratch dequantizer plugs in later.
+        }
+    }
+
+    // GGML_TYPE_Q8_K has no public from_float_ref (see quantize_row_q8_K_generic's
+    // doc comment in ggml_internal_abi.h) -- it's the activation format for every
+    // K-quant vec_dot/gemv/gemm, so without this, all of those silently have no
+    // valid input to quantize into and get skipped by the benchmarks.
+    {
+        const ggml_type_traits_cpu *tc = ggml_get_type_traits_cpu(GGML_TYPE_Q8_K);
+        registries.quantize.register_reference(GGML_TYPE_Q8_K, "ggml-generic", quantize_row_q8_K_generic);
+        if (tc && tc->from_float) {
+            registries.quantize.register_candidate(GGML_TYPE_Q8_K, "ggml-cpu", tc->from_float);
+        }
+    }
+
+    // GGML_TYPE_IQ2_XXS/IQ2_XS/IQ1_S/IQ1_M have no public from_float_ref
+    // either (see quantize_iq2_xxs's doc comment in ggml_internal_abi.h) --
+    // they're importance-matrix-only codebook types whose only public
+    // quantizer takes a different, whole-matrix signature. Without this,
+    // these 4 types would never appear in the quantize/dequantize
+    // benchmarks at all (bench_dequantize.cpp requires both a quantize and
+    // a dequantize reference to exist before it will test a type).
+    registries.quantize.register_reference(GGML_TYPE_IQ2_XXS, "ggml-ref", quantize_iq2_xxs_row);
+    registries.quantize.register_reference(GGML_TYPE_IQ2_XS, "ggml-ref", quantize_iq2_xs_row);
+    registries.quantize.register_reference(GGML_TYPE_IQ1_S, "ggml-ref", quantize_iq1_s_row);
+    registries.quantize.register_reference(GGML_TYPE_IQ1_M, "ggml-ref", quantize_iq1_m_row);
+
+    // arch-fallback.h #defines a `_generic` symbol onto its canonical
+    // counterpart -- inside GGML's own source files -- for whichever
+    // functions the current architecture has no distinct optimized version
+    // of, and which functions that applies to varies by architecture. The
+    // `_generic` declarations in ggml_internal_abi.h are marked
+    // GGML_BENCH_WEAK precisely so that case resolves to a null function
+    // pointer here instead of a link error: when null, there is only one
+    // real implementation, so it becomes the reference with no candidate,
+    // rather than fabricating a comparison against nothing.
+    auto register_pair = [](auto &registry, const auto &key, auto generic_fn, auto canonical_fn) {
+        if (generic_fn) {
+            registry.register_reference(key, "ggml-generic", generic_fn);
+            if (canonical_fn) {
+                registry.register_candidate(key, "ggml-cpu", canonical_fn);
+            }
+        } else if (canonical_fn) {
+            registry.register_reference(key, "ggml-cpu", canonical_fn);
+        }
+    };
+
+    // -- vec_dot: public candidate, private (possibly weak-null) reference --
+    for (const auto &ref : k_vec_dot_refs) {
+        const ggml_type_traits_cpu *tc = ggml_get_type_traits_cpu(ref.type);
+        register_pair(registries.vec_dot, ref.type, ref.fn, tc ? tc->vec_dot : nullptr);
+    }
+
+    // -- repack: private reference and candidate (no public accessor exists) --
+    for (const auto &e : k_repack_entries) {
+        register_pair(registries.repack_quantize_mat, e.key, e.quantize_mat_generic, e.quantize_mat);
+        register_pair(registries.repack_gemv, e.key, e.gemv_generic, e.gemv);
+        register_pair(registries.repack_gemm, e.key, e.gemm_generic, e.gemm);
+    }
+}
diff --git a/apps/ggml/providers/ggml_provider.h b/apps/ggml/providers/ggml_provider.h
new file mode 100644
index 000000000000..bca2c7ec42fb
--- /dev/null
+++ b/apps/ggml/providers/ggml_provider.h
@@ -0,0 +1,21 @@
+#pragma once
+
+#include "kernel_registry.h"
+
+// Registers GGML's own implementations into `registries`:
+//
+//   quantize / dequantize -- entirely via GGML's PUBLIC API
+//   (ggml_get_type_traits / ggml_get_type_traits_cpu, see include/ggml.h and
+//   include/ggml-cpu.h): `from_float_ref`/`to_float` (GGML's own documented
+//   "reference" routines) become the Registry reference, and the
+//   CPU-dispatched `from_float` becomes a candidate. No private header used.
+//
+//   vec_dot / repack (quantize_mat, gemv, gemm) -- these categories have no
+//   public way to reach the always-available pure-C fallback, so the
+//   `_generic`-suffixed symbol (declared in ggml_internal_abi.h) is
+//   registered as the reference and the canonical symbol (reached publicly
+//   for vec_dot via ggml_get_type_traits_cpu, and privately for repack,
+//   which has no public accessor at all) is registered as a candidate.
+//
+// This is the only file in kernel-bench that knows GGML exists.
+void register_ggml_provider(KernelRegistries &registries);
diff --git a/apps/ggml/providers/halide_provider.cpp b/apps/ggml/providers/halide_provider.cpp
new file mode 100644
index 000000000000..09df8bcc2333
--- /dev/null
+++ b/apps/ggml/providers/halide_provider.cpp
@@ -0,0 +1,236 @@
+#include "halide_provider.h"
+
+#include <ggml_quants.h>
+
+void register_halide_provider(KernelRegistries &registries) {
+    registries.quantize.register_candidate(GGML_TYPE_Q4_0, "halide", ggml_quants_halide_quantize_q4_0);
+    registries.dequantize.register_candidate(GGML_TYPE_Q4_0, "halide", ggml_quants_halide_dequantize_q4_0);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q4_0, "halide", ggml_quants_halide_vec_dot_q4_0_q8_0);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q4_1, "halide", ggml_quants_halide_quantize_q4_1);
+    registries.dequantize.register_candidate(GGML_TYPE_Q4_1, "halide", ggml_quants_halide_dequantize_q4_1);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q4_1, "halide", ggml_quants_halide_vec_dot_q4_1_q8_1);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q5_0, "halide", ggml_quants_halide_quantize_q5_0);
+    registries.dequantize.register_candidate(GGML_TYPE_Q5_0, "halide", ggml_quants_halide_dequantize_q5_0);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q5_0, "halide", ggml_quants_halide_vec_dot_q5_0_q8_0);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q5_1, "halide", ggml_quants_halide_quantize_q5_1);
+    registries.dequantize.register_candidate(GGML_TYPE_Q5_1, "halide", ggml_quants_halide_dequantize_q5_1);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q5_1, "halide", ggml_quants_halide_vec_dot_q5_1_q8_1);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q8_0, "halide", ggml_quants_halide_quantize_q8_0);
+    registries.dequantize.register_candidate(GGML_TYPE_Q8_0, "halide", ggml_quants_halide_dequantize_q8_0);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q8_0, "halide", ggml_quants_halide_vec_dot_q8_0_q8_0);
+
+    // Q8_1 is activation-only (GGML has no public to_float for it, so
+    // ggml_provider.cpp registers no dequantize reference either -- the
+    // harness's bench_dequantize.cpp already skips types with no reference).
+    registries.quantize.register_candidate(GGML_TYPE_Q8_1, "halide", ggml_quants_halide_quantize_q8_1);
+
+    // Q8_K is also activation-only, but unlike Q8_1, GGML doesn't even
+    // register a public from_float_ref for it -- ggml_provider.cpp's
+    // special case (using the private quantize_row_q8_K_generic) is what
+    // supplies the reference this candidate is compared against.
+    registries.quantize.register_candidate(GGML_TYPE_Q8_K, "halide", ggml_quants_halide_quantize_q8_k);
+
+    // Q2_K, Q6_K: dequantize is a genuine from-scratch Halide candidate.
+    // Quantize is scaffolding that itself calls out to GGML's own
+    // reference (see halide/ggml_extern_quantize.cpp) pending a
+    // from-scratch port of GGML's iterative scale search -- it's still
+    // registered as a candidate so the harness's plumbing is exercised
+    // end-to-end, but it will trivially match (same underlying code path).
+    // vec_dot is a genuine from-scratch candidate (against Q8_K).
+    registries.quantize.register_candidate(GGML_TYPE_Q2_K, "halide", ggml_quants_halide_quantize_q2_k);
+    registries.dequantize.register_candidate(GGML_TYPE_Q2_K, "halide", ggml_quants_halide_dequantize_q2_k);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q2_K, "halide", ggml_quants_halide_vec_dot_q2_k_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q6_K, "halide", ggml_quants_halide_quantize_q6_k);
+    registries.dequantize.register_candidate(GGML_TYPE_Q6_K, "halide", ggml_quants_halide_dequantize_q6_k);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q6_K, "halide", ggml_quants_halide_vec_dot_q6_k_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q4_K, "halide", ggml_quants_halide_quantize_q4_k);
+    registries.dequantize.register_candidate(GGML_TYPE_Q4_K, "halide", ggml_quants_halide_dequantize_q4_k);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q4_K, "halide", ggml_quants_halide_vec_dot_q4_k_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q5_K, "halide", ggml_quants_halide_quantize_q5_k);
+    registries.dequantize.register_candidate(GGML_TYPE_Q5_K, "halide", ggml_quants_halide_dequantize_q5_k);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q5_K, "halide", ggml_quants_halide_vec_dot_q5_k_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q3_K, "halide", ggml_quants_halide_quantize_q3_k);
+    registries.dequantize.register_candidate(GGML_TYPE_Q3_K, "halide", ggml_quants_halide_dequantize_q3_k);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q3_K, "halide", ggml_quants_halide_vec_dot_q3_k_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_Q1_0, "halide", ggml_quants_halide_quantize_q1_0);
+    registries.dequantize.register_candidate(GGML_TYPE_Q1_0, "halide", ggml_quants_halide_dequantize_q1_0);
+    registries.vec_dot.register_candidate(GGML_TYPE_Q1_0, "halide", ggml_quants_halide_vec_dot_q1_0_q8_0);
+
+    registries.quantize.register_candidate(GGML_TYPE_MXFP4, "halide", ggml_quants_halide_quantize_mxfp4);
+    registries.dequantize.register_candidate(GGML_TYPE_MXFP4, "halide", ggml_quants_halide_dequantize_mxfp4);
+    registries.vec_dot.register_candidate(GGML_TYPE_MXFP4, "halide", ggml_quants_halide_vec_dot_mxfp4_q8_0);
+
+    registries.quantize.register_candidate(GGML_TYPE_NVFP4, "halide", ggml_quants_halide_quantize_nvfp4);
+    registries.dequantize.register_candidate(GGML_TYPE_NVFP4, "halide", ggml_quants_halide_dequantize_nvfp4);
+    registries.vec_dot.register_candidate(GGML_TYPE_NVFP4, "halide", ggml_quants_halide_vec_dot_nvfp4_q8_0);
+
+    registries.quantize.register_candidate(GGML_TYPE_IQ4_NL, "halide", ggml_quants_halide_quantize_iq4_nl);
+    registries.dequantize.register_candidate(GGML_TYPE_IQ4_NL, "halide", ggml_quants_halide_dequantize_iq4_nl);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ4_NL, "halide", ggml_quants_halide_vec_dot_iq4_nl_q8_0);
+
+    registries.quantize.register_candidate(GGML_TYPE_IQ4_XS, "halide", ggml_quants_halide_quantize_iq4_xs);
+    registries.dequantize.register_candidate(GGML_TYPE_IQ4_XS, "halide", ggml_quants_halide_dequantize_iq4_xs);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ4_XS, "halide", ggml_quants_halide_vec_dot_iq4_xs_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_TQ1_0, "halide", ggml_quants_halide_quantize_tq1_0);
+    registries.dequantize.register_candidate(GGML_TYPE_TQ1_0, "halide", ggml_quants_halide_dequantize_tq1_0);
+    registries.vec_dot.register_candidate(GGML_TYPE_TQ1_0, "halide", ggml_quants_halide_vec_dot_tq1_0_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_TQ2_0, "halide", ggml_quants_halide_quantize_tq2_0);
+    registries.dequantize.register_candidate(GGML_TYPE_TQ2_0, "halide", ggml_quants_halide_dequantize_tq2_0);
+    registries.vec_dot.register_candidate(GGML_TYPE_TQ2_0, "halide", ggml_quants_halide_vec_dot_tq2_0_q8_k);
+
+    // IQ2_XXS: dequantize only (see ggml_quants.h for why); vec_dot is still
+    // a genuine from-scratch candidate (GGML's own reference quantizer is
+    // used to produce the test/benchmark input, via ggml_provider.cpp).
+    registries.dequantize.register_candidate(GGML_TYPE_IQ2_XXS, "halide", ggml_quants_halide_dequantize_iq2_xxs);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ2_XXS, "halide", ggml_quants_halide_vec_dot_iq2_xxs_q8_k);
+
+    registries.dequantize.register_candidate(GGML_TYPE_IQ2_XS, "halide", ggml_quants_halide_dequantize_iq2_xs);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ2_XS, "halide", ggml_quants_halide_vec_dot_iq2_xs_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_IQ2_S, "halide", ggml_quants_halide_quantize_iq2_s);
+    registries.dequantize.register_candidate(GGML_TYPE_IQ2_S, "halide", ggml_quants_halide_dequantize_iq2_s);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ2_S, "halide", ggml_quants_halide_vec_dot_iq2_s_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_IQ3_XXS, "halide", ggml_quants_halide_quantize_iq3_xxs);
+    registries.dequantize.register_candidate(GGML_TYPE_IQ3_XXS, "halide", ggml_quants_halide_dequantize_iq3_xxs);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ3_XXS, "halide", ggml_quants_halide_vec_dot_iq3_xxs_q8_k);
+
+    registries.quantize.register_candidate(GGML_TYPE_IQ3_S, "halide", ggml_quants_halide_quantize_iq3_s);
+    registries.dequantize.register_candidate(GGML_TYPE_IQ3_S, "halide", ggml_quants_halide_dequantize_iq3_s);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ3_S, "halide", ggml_quants_halide_vec_dot_iq3_s_q8_k);
+
+    registries.dequantize.register_candidate(GGML_TYPE_IQ1_S, "halide", ggml_quants_halide_dequantize_iq1_s);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ1_S, "halide", ggml_quants_halide_vec_dot_iq1_s_q8_k);
+
+    registries.dequantize.register_candidate(GGML_TYPE_IQ1_M, "halide", ggml_quants_halide_dequantize_iq1_m);
+    registries.vec_dot.register_candidate(GGML_TYPE_IQ1_M, "halide", ggml_quants_halide_vec_dot_iq1_m_q8_k);
+
+    // F16, BF16: plain float casts, not "quantized" types, but both
+    // directions are fully native Halide. No vec_dot: not part of the
+    // quantized-format vec_dot sweep this directory otherwise covers.
+    registries.quantize.register_candidate(GGML_TYPE_F16, "halide", ggml_quants_halide_quantize_f16);
+    registries.dequantize.register_candidate(GGML_TYPE_F16, "halide", ggml_quants_halide_dequantize_f16);
+
+    registries.quantize.register_candidate(GGML_TYPE_BF16, "halide", ggml_quants_halide_quantize_bf16);
+    registries.dequantize.register_candidate(GGML_TYPE_BF16, "halide", ggml_quants_halide_dequantize_bf16);
+
+    // Repack quantize_mat: GGML itself only has 4 distinct implementations
+    // (2 activation formats x 2 interleave widths), reused across every
+    // repack weight type that shares one -- see k_repack_entries in
+    // ggml_provider.cpp, which this table mirrors label-for-label so the
+    // registered RepackKey (used by bench_repack.cpp for act_type/base_type)
+    // matches exactly. gemv/gemm repack candidates for every weight family
+    // GGML registers a repack entry for (Q4_0, Q8_0, IQ4_NL, MXFP4, Q4_K,
+    // Q5_K, Q6_K, Q2_K) are registered below.
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 4, 4, "q4_0_4x4_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x4);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 8, 4, "q4_0_4x8_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x8);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 8, 8, "q4_0_8x8_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x8);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q4_K, GGML_TYPE_Q8_K, 8, 4, "q4_K_8x4_q8_K"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_k_4x4);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q4_K, GGML_TYPE_Q8_K, 8, 8, "q4_K_8x8_q8_K"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_k_4x8);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q5_K, GGML_TYPE_Q8_K, 8, 4, "q5_K_8x4_q8_K"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_k_4x4);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q5_K, GGML_TYPE_Q8_K, 8, 8, "q5_K_8x8_q8_K"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_k_4x8);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q6_K, GGML_TYPE_Q8_K, 8, 4, "q6_K_8x4_q8_K"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_k_4x4);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q6_K, GGML_TYPE_Q8_K, 8, 8, "q6_K_8x8_q8_K"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_k_4x8);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q2_K, GGML_TYPE_Q8_K, 8, 8, "q2_K_8x8_q8_K"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_k_4x8);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0, 4, 4, "iq4_nl_4x4_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x4);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0, 8, 8, "iq4_nl_8x8_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x8);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_MXFP4, GGML_TYPE_Q8_0, 4, 4, "mxfp4_4x4_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x4);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_MXFP4, GGML_TYPE_Q8_0, 8, 8, "mxfp4_8x8_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x8);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, 4, 4, "q8_0_4x4_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x4);
+    registries.repack_quantize_mat.register_candidate({GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, 8, 4, "q8_0_4x8_q8_0"},
+                                                      "halide", ggml_quants_halide_repack_quantize_mat_q8_0_4x8);
+
+    // gemv/gemm: same RepackKeys as the quantize_mat table above (base type,
+    // activation type, blocklen, ncols_interleaved, label must match
+    // label-for-label so bench_repack.cpp's per-key lookups line up).
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 4, 4, "q4_0_4x4_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q4_0_4x4_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 4, 4, "q4_0_4x4_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q4_0_4x4_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 8, 4, "q4_0_4x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q4_0_4x8_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 8, 4, "q4_0_4x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q4_0_4x8_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 8, 8, "q4_0_8x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q4_0_8x8_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, 8, 8, "q4_0_8x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q4_0_8x8_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, 4, 4, "q8_0_4x4_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q8_0_4x4_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, 4, 4, "q8_0_4x4_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q8_0_4x4_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, 8, 4, "q8_0_4x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q8_0_4x8_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, 8, 4, "q8_0_4x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q8_0_4x8_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0, 4, 4, "iq4_nl_4x4_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_iq4_nl_4x4_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0, 4, 4, "iq4_nl_4x4_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_iq4_nl_4x4_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0, 8, 8, "iq4_nl_8x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_iq4_nl_8x8_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0, 8, 8, "iq4_nl_8x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_iq4_nl_8x8_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_MXFP4, GGML_TYPE_Q8_0, 4, 4, "mxfp4_4x4_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_mxfp4_4x4_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_MXFP4, GGML_TYPE_Q8_0, 4, 4, "mxfp4_4x4_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_mxfp4_4x4_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_MXFP4, GGML_TYPE_Q8_0, 8, 8, "mxfp4_8x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemv_mxfp4_8x8_q8_0);
+    registries.repack_gemm.register_candidate({GGML_TYPE_MXFP4, GGML_TYPE_Q8_0, 8, 8, "mxfp4_8x8_q8_0"}, "halide",
+                                              ggml_quants_halide_repack_gemm_mxfp4_8x8_q8_0);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q4_K, GGML_TYPE_Q8_K, 8, 4, "q4_K_8x4_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q4_k_8x4_q8_k);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q4_K, GGML_TYPE_Q8_K, 8, 4, "q4_K_8x4_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q4_k_8x4_q8_k);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q4_K, GGML_TYPE_Q8_K, 8, 8, "q4_K_8x8_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q4_k_8x8_q8_k);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q4_K, GGML_TYPE_Q8_K, 8, 8, "q4_K_8x8_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q4_k_8x8_q8_k);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q5_K, GGML_TYPE_Q8_K, 8, 4, "q5_K_8x4_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q5_k_8x4_q8_k);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q5_K, GGML_TYPE_Q8_K, 8, 4, "q5_K_8x4_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q5_k_8x4_q8_k);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q5_K, GGML_TYPE_Q8_K, 8, 8, "q5_K_8x8_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q5_k_8x8_q8_k);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q5_K, GGML_TYPE_Q8_K, 8, 8, "q5_K_8x8_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q5_k_8x8_q8_k);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q6_K, GGML_TYPE_Q8_K, 8, 4, "q6_K_8x4_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q6_k_8x4_q8_k);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q6_K, GGML_TYPE_Q8_K, 8, 4, "q6_K_8x4_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q6_k_8x4_q8_k);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q6_K, GGML_TYPE_Q8_K, 8, 8, "q6_K_8x8_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q6_k_8x8_q8_k);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q6_K, GGML_TYPE_Q8_K, 8, 8, "q6_K_8x8_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q6_k_8x8_q8_k);
+    registries.repack_gemv.register_candidate({GGML_TYPE_Q2_K, GGML_TYPE_Q8_K, 8, 8, "q2_K_8x8_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemv_q2_k_8x8_q8_k);
+    registries.repack_gemm.register_candidate({GGML_TYPE_Q2_K, GGML_TYPE_Q8_K, 8, 8, "q2_K_8x8_q8_K"}, "halide",
+                                              ggml_quants_halide_repack_gemm_q2_k_8x8_q8_k);
+}
diff --git a/apps/ggml/providers/halide_provider.h b/apps/ggml/providers/halide_provider.h
new file mode 100644
index 000000000000..02b80d345536
--- /dev/null
+++ b/apps/ggml/providers/halide_provider.h
@@ -0,0 +1,8 @@
+#pragma once
+
+#include "kernel_registry.h"
+
+// Registers the from-scratch Halide reimplementation of GGML's Q4_0
+// quantize/dequantize kernels (see ../halide/) as candidates against GGML's
+// own reference, which register_ggml_provider() registers first.
+void register_halide_provider(KernelRegistries &registries);
diff --git a/apps/ggml/src/bench_dequantize.cpp b/apps/ggml/src/bench_dequantize.cpp
new file mode 100644
index 000000000000..260645b80656
--- /dev/null
+++ b/apps/ggml/src/bench_dequantize.cpp
@@ -0,0 +1,67 @@
+#include "benchmarks.h"
+
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "timing.h"
+
+namespace {
+constexpr int64_t kTargetElements = 4096;
+}  // namespace
+
+BenchReport run_dequantize_benchmarks(const KernelRegistries &registries) {
+    BenchReport report{"dequantize_row", "GB/s", {}};
+    print_report_header(report.title, report.throughput_unit);
+
+    for (int t = 0; t < GGML_TYPE_COUNT; ++t) {
+        const ggml_type type = static_cast<ggml_type>(t);
+        const Impl<dequantize_fn_t> *ref = registries.dequantize.reference(type);
+        const Impl<quantize_fn_t> *qref = registries.quantize.reference(type);
+        if (!ref || !qref) {
+            continue;
+        }
+        ggml_quantize_init(type);  // one-time, cheap after the first call for this type; see ggml_provider.cpp
+
+        const int64_t blck = ggml_blck_size(type);
+        const int64_t n = ((kTargetElements + blck - 1) / blck) * blck;
+
+        AlignedBuffer src(n * sizeof(float));
+        generate_synthetic_data(src.as<float>(), n);
+
+        AlignedBuffer quantized(ggml_row_size(type, n));
+        qref->fn(src.as<float>(), quantized.data(), n);
+
+        AlignedBuffer ref_out(n * sizeof(float));
+        ref->fn(quantized.data(), ref_out.as<float>(), n);
+        const TimingResult ref_time = time_calls([&] { ref->fn(quantized.data(), ref_out.as<float>(), n); });
+
+        BenchRow row;
+        row.label = ggml_type_name(type);
+        row.ref_name = ref->name;
+        row.ref_ns = ref_time.median_ns;
+        row.ref_throughput = bytes_per_sec(n * sizeof(float), ref_time.median_ns) / 1e9;
+
+        for (const auto &cand : registries.dequantize.candidates(type)) {
+            BenchCandidate bc;
+            bc.name = cand.name;
+            bc.identical = (cand.fn == ref->fn);
+            if (!bc.identical) {
+                AlignedBuffer cand_out(n * sizeof(float));
+                cand.fn(quantized.data(), cand_out.as<float>(), n);
+                bc.correct = floats_match(ref_out.as<float>(), cand_out.as<float>(), n);
+
+                const TimingResult cand_time = time_calls([&] { cand.fn(quantized.data(), cand_out.as<float>(), n); });
+                bc.ns = cand_time.median_ns;
+                bc.throughput = bytes_per_sec(n * sizeof(float), cand_time.median_ns) / 1e9;
+                bc.speedup = ref_time.median_ns / cand_time.median_ns;
+            }
+            row.candidates.push_back(bc);
+        }
+
+        print_row(row, report.throughput_unit);
+        report.rows.push_back(std::move(row));
+    }
+
+    return report;
+}
diff --git a/apps/ggml/src/bench_quantize.cpp b/apps/ggml/src/bench_quantize.cpp
new file mode 100644
index 000000000000..56d60dcd7f71
--- /dev/null
+++ b/apps/ggml/src/bench_quantize.cpp
@@ -0,0 +1,65 @@
+#include "benchmarks.h"
+
+#include <cstring>
+
+#include <ggml.h>
+
+#include "data_gen.h"
+#include "timing.h"
+
+namespace {
+constexpr int64_t kTargetElements = 4096;
+}
+
+BenchReport run_quantize_benchmarks(const KernelRegistries &registries) {
+    BenchReport report{"quantize_row", "GB/s", {}};
+    print_report_header(report.title, report.throughput_unit);
+
+    for (int t = 0; t < GGML_TYPE_COUNT; ++t) {
+        const ggml_type type = static_cast<ggml_type>(t);
+        const Impl<quantize_fn_t> *ref = registries.quantize.reference(type);
+        if (!ref) {
+            continue;
+        }
+        ggml_quantize_init(type);  // one-time, cheap after the first call for this type; see ggml_provider.cpp
+
+        const int64_t blck = ggml_blck_size(type);
+        const int64_t n = ((kTargetElements + blck - 1) / blck) * blck;
+        const size_t out_bytes = ggml_row_size(type, n);
+
+        AlignedBuffer src(n * sizeof(float));
+        generate_synthetic_data(src.as<float>(), n);
+
+        AlignedBuffer ref_out(out_bytes);
+        ref->fn(src.as<float>(), ref_out.data(), n);
+        const TimingResult ref_time = time_calls([&] { ref->fn(src.as<float>(), ref_out.data(), n); });
+
+        BenchRow row;
+        row.label = ggml_type_name(type);
+        row.ref_name = ref->name;
+        row.ref_ns = ref_time.median_ns;
+        row.ref_throughput = bytes_per_sec(n * sizeof(float), ref_time.median_ns) / 1e9;
+
+        for (const auto &cand : registries.quantize.candidates(type)) {
+            BenchCandidate bc;
+            bc.name = cand.name;
+            bc.identical = (cand.fn == ref->fn);
+            if (!bc.identical) {
+                AlignedBuffer cand_out(out_bytes);
+                cand.fn(src.as<float>(), cand_out.data(), n);
+                bc.correct = (std::memcmp(ref_out.data(), cand_out.data(), out_bytes) == 0);
+
+                const TimingResult cand_time = time_calls([&] { cand.fn(src.as<float>(), cand_out.data(), n); });
+                bc.ns = cand_time.median_ns;
+                bc.throughput = bytes_per_sec(n * sizeof(float), cand_time.median_ns) / 1e9;
+                bc.speedup = ref_time.median_ns / cand_time.median_ns;
+            }
+            row.candidates.push_back(bc);
+        }
+
+        print_row(row, report.throughput_unit);
+        report.rows.push_back(std::move(row));
+    }
+
+    return report;
+}
diff --git a/apps/ggml/src/bench_repack.cpp b/apps/ggml/src/bench_repack.cpp
new file mode 100644
index 000000000000..18ca3b675222
--- /dev/null
+++ b/apps/ggml/src/bench_repack.cpp
@@ -0,0 +1,269 @@
+#include "benchmarks.h"
+
+#include <cstring>
+#include <vector>
+
+#include <ggml-alloc.h>
+#include <ggml-backend.h>
+#include <ggml-cpu.h>
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "ggml_internal_abi.h"  // ggml_backend_cpu_repack_buffer_type()
+#include "timing.h"
+
+namespace {
+
+// K (reduction dim): divisible by every block size in play (32 and 256).
+constexpr int64_t kK = 4096;
+// Output columns: divisible by every NB_COLS in play (4 and 8).
+constexpr int kNC = 32;
+// Activation rows for the gemm (batched) path; must be a multiple of 4 (the
+// row-group size ggml_quantize_mat_* always packs), and > 3 so production
+// code would pick gemm over gemv for this many rows (see repack.cpp's
+// forward_mul_mat_one_chunk: "if there are more than three rows in src1,
+// use gemm; otherwise, use gemv").
+constexpr int kGemmRows = 32;
+
+// Builds a packed weight buffer for `base_type`, shape [kK, kNC], by
+// quantizing a row-major staging buffer through the reference quantizer and
+// then letting the repack buffer type's set_tensor callback do the actual
+// interleaving (src/ggml-cpu/repack.cpp:4733). This avoids reimplementing
+// the private block<K,N> interleave layout by hand -- everything here is
+// public API plus the one declared-ourselves accessor for the buffer type.
+struct PackedWeight {
+    ggml_context *ctx = nullptr;
+    ggml_backend_buffer_t buffer = nullptr;
+    ggml_tensor *tensor = nullptr;
+
+    PackedWeight() = default;
+    PackedWeight(const PackedWeight &) = delete;
+    PackedWeight &operator=(const PackedWeight &) = delete;
+    PackedWeight(PackedWeight &&other) noexcept
+        : ctx(other.ctx), buffer(other.buffer), tensor(other.tensor) {
+        other.ctx = nullptr;
+        other.buffer = nullptr;
+        other.tensor = nullptr;
+    }
+    PackedWeight &operator=(PackedWeight &&other) noexcept {
+        if (this != &other) {
+            if (buffer) ggml_backend_buffer_free(buffer);
+            if (ctx) ggml_free(ctx);
+            ctx = other.ctx;
+            buffer = other.buffer;
+            tensor = other.tensor;
+            other.ctx = nullptr;
+            other.buffer = nullptr;
+            other.tensor = nullptr;
+        }
+        return *this;
+    }
+    ~PackedWeight() {
+        if (buffer) ggml_backend_buffer_free(buffer);
+        if (ctx) ggml_free(ctx);
+    }
+
+    // ggml_repack_get_optimal_repack_type() (src/ggml-cpu/repack.cpp) picks
+    // ONE interleave layout per (type, CPU features, column count) -- it is
+    // GGML's own "best kernel for this hardware" heuristic, not something
+    // this benchmark can steer towards a specific registered combo. When it
+    // finds none (e.g. Q2_K has no ARM branch at all, only AVX512/RISC-V --
+    // see that function), the buffer type's init_tensor callback leaves
+    // tensor->extra null, and calling set_tensor on it would dereference a
+    // null tensor_traits pointer. There is no supported way to force a
+    // different combo through this public mechanism, so such combos are
+    // skipped rather than benchmarked with fabricated data.
+    bool supported_on_this_cpu() const {
+        return tensor && tensor->extra != nullptr;
+    }
+};
+
+PackedWeight build_packed_weight(ggml_type base_type, const Impl<quantize_fn_t> &row_quant, int64_t k, int nc) {
+    PackedWeight pw;
+    ggml_init_params params{/*.mem_size=*/ggml_tensor_overhead() + 256, /*.mem_buffer=*/nullptr,
+                            /*.no_alloc=*/true};
+    pw.ctx = ggml_init(params);
+    pw.tensor = ggml_new_tensor_2d(pw.ctx, base_type, k, nc);
+    pw.buffer = ggml_backend_alloc_ctx_tensors_from_buft(pw.ctx, ggml_backend_cpu_repack_buffer_type());
+    if (!pw.supported_on_this_cpu()) {
+        return pw;
+    }
+
+    AlignedBuffer staging(ggml_row_size(base_type, k) * nc);
+    AlignedBuffer col(k * sizeof(float));
+    const size_t row_bytes = ggml_row_size(base_type, k);
+    for (int c = 0; c < nc; ++c) {
+        generate_synthetic_data(col.as<float>(), k, static_cast<float>(c));
+        row_quant.fn(col.as<float>(), staging.as<uint8_t>() + c * row_bytes, k);
+    }
+    ggml_backend_tensor_set(pw.tensor, staging.data(), 0, staging.size());
+    return pw;
+}
+
+}  // namespace
+
+std::vector<BenchReport> run_repack_benchmarks(const KernelRegistries &registries) {
+    BenchReport quant_mat_report{"repack_quantize_mat", "GB/s", {}};
+    BenchReport gemv_report{"repack_gemv", "GFLOP/s", {}};
+    BenchReport gemm_report{"repack_gemm", "GFLOP/s", {}};
+
+    print_report_header(quant_mat_report.title, quant_mat_report.throughput_unit);
+    for (const RepackKey &key : registries.repack_quantize_mat.keys()) {
+        const Impl<quantize_fn_t> *qm_ref = registries.repack_quantize_mat.reference(key.label);
+        if (!qm_ref) {
+            continue;
+        }
+        const int64_t blck = ggml_blck_size(key.act_type);
+        const int64_t k = ((kK + blck - 1) / blck) * blck;
+
+        // ggml_quantize_mat_* always consumes exactly 4 rows (see
+        // src/ggml-cpu/repack.cpp: ggml_quantize_mat_t<...> asserts nrow==4),
+        // regardless of the interleave geometry in the name.
+        AlignedBuffer src(4 * k * sizeof(float));
+        generate_synthetic_data(src.as<float>(), 4 * k);
+        const size_t out_bytes = 4 * ggml_row_size(key.act_type, k);
+
+        AlignedBuffer ref_out(out_bytes);
+        qm_ref->fn(src.as<float>(), ref_out.data(), k);
+        const TimingResult ref_time = time_calls([&] { qm_ref->fn(src.as<float>(), ref_out.data(), k); });
+
+        BenchRow row;
+        row.label = key.label;
+        row.ref_name = qm_ref->name;
+        row.ref_ns = ref_time.median_ns;
+        row.ref_throughput = bytes_per_sec(4 * k * sizeof(float), ref_time.median_ns) / 1e9;
+
+        for (const auto &cand : registries.repack_quantize_mat.candidates(key.label)) {
+            BenchCandidate bc;
+            bc.name = cand.name;
+            bc.identical = (cand.fn == qm_ref->fn);
+            if (!bc.identical) {
+                AlignedBuffer cand_out(out_bytes);
+                cand.fn(src.as<float>(), cand_out.data(), k);
+                bc.correct = (std::memcmp(ref_out.data(), cand_out.data(), out_bytes) == 0);
+
+                const TimingResult cand_time = time_calls([&] { cand.fn(src.as<float>(), cand_out.data(), k); });
+                bc.ns = cand_time.median_ns;
+                bc.throughput = bytes_per_sec(4 * k * sizeof(float), cand_time.median_ns) / 1e9;
+                bc.speedup = ref_time.median_ns / cand_time.median_ns;
+            }
+            row.candidates.push_back(bc);
+        }
+        print_row(row, quant_mat_report.throughput_unit);
+        quant_mat_report.rows.push_back(std::move(row));
+    }
+
+    print_report_header(gemv_report.title, gemv_report.throughput_unit);
+    for (const RepackKey &key : registries.repack_gemv.keys()) {
+        const Impl<gemx_fn_t> *gemv_ref = registries.repack_gemv.reference(key.label);
+        const Impl<quantize_fn_t> *w_quant = registries.quantize.reference(key.base_type);
+        const Impl<quantize_fn_t> *a_quant = registries.quantize.reference(key.act_type);
+        if (!gemv_ref || !w_quant || !a_quant) {
+            continue;
+        }
+        const int64_t blck = ggml_blck_size(key.base_type);
+        const int64_t k = ((kK + blck - 1) / blck) * blck;
+
+        PackedWeight weight = build_packed_weight(key.base_type, *w_quant, k, kNC);
+        if (!weight.supported_on_this_cpu()) {
+            continue;  // no repack kernel for this (type, CPU) combo -- see PackedWeight::supported_on_this_cpu
+        }
+
+        AlignedBuffer a_src(k * sizeof(float));
+        generate_synthetic_data(a_src.as<float>(), k, 3.0f);
+        AlignedBuffer vy(ggml_row_size(key.act_type, k));
+        a_quant->fn(a_src.as<float>(), vy.data(), k);
+
+        AlignedBuffer ref_out(kNC * sizeof(float));
+        gemv_ref->fn(k, ref_out.as<float>(), kNC, weight.tensor->data, vy.data(), 1, kNC);
+        const TimingResult ref_time =
+            time_calls([&] { gemv_ref->fn(k, ref_out.as<float>(), kNC, weight.tensor->data, vy.data(), 1, kNC); });
+        const double ref_flops = 2.0 * k * kNC;
+
+        BenchRow row;
+        row.label = key.label;
+        row.ref_name = gemv_ref->name;
+        row.ref_ns = ref_time.median_ns;
+        row.ref_throughput = gflops(ref_flops, ref_time.median_ns);
+
+        for (const auto &cand : registries.repack_gemv.candidates(key.label)) {
+            BenchCandidate bc;
+            bc.name = cand.name;
+            bc.identical = (cand.fn == gemv_ref->fn);
+            if (!bc.identical) {
+                AlignedBuffer cand_out(kNC * sizeof(float));
+                cand.fn(k, cand_out.as<float>(), kNC, weight.tensor->data, vy.data(), 1, kNC);
+                bc.correct = floats_match(ref_out.as<float>(), cand_out.as<float>(), kNC);
+
+                const TimingResult cand_time =
+                    time_calls([&] { cand.fn(k, cand_out.as<float>(), kNC, weight.tensor->data, vy.data(), 1, kNC); });
+                bc.ns = cand_time.median_ns;
+                bc.throughput = gflops(ref_flops, cand_time.median_ns);
+                bc.speedup = ref_time.median_ns / cand_time.median_ns;
+            }
+            row.candidates.push_back(bc);
+        }
+        print_row(row, gemv_report.throughput_unit);
+        gemv_report.rows.push_back(std::move(row));
+    }
+
+    print_report_header(gemm_report.title, gemm_report.throughput_unit);
+    for (const RepackKey &key : registries.repack_gemm.keys()) {
+        const Impl<gemx_fn_t> *gemm_ref = registries.repack_gemm.reference(key.label);
+        const Impl<quantize_fn_t> *w_quant = registries.quantize.reference(key.base_type);
+        const Impl<quantize_fn_t> *qm_ref = registries.repack_quantize_mat.reference(key.label);
+        if (!gemm_ref || !w_quant || !qm_ref) {
+            continue;
+        }
+        const int64_t blck = ggml_blck_size(key.base_type);
+        const int64_t k = ((kK + blck - 1) / blck) * blck;
+
+        PackedWeight weight = build_packed_weight(key.base_type, *w_quant, k, kNC);
+        if (!weight.supported_on_this_cpu()) {
+            continue;  // no repack kernel for this (type, CPU) combo -- see PackedWeight::supported_on_this_cpu
+        }
+
+        AlignedBuffer a_src(kGemmRows * k * sizeof(float));
+        generate_synthetic_data(a_src.as<float>(), kGemmRows * k, 5.0f);
+        const size_t group_bytes = 4 * ggml_row_size(key.act_type, k);
+        AlignedBuffer vy(group_bytes * (kGemmRows / 4));
+        for (int g = 0; g < kGemmRows / 4; ++g) {
+            qm_ref->fn(a_src.as<float>() + g * 4 * k, vy.as<uint8_t>() + g * group_bytes, k);
+        }
+
+        AlignedBuffer ref_out(kGemmRows * kNC * sizeof(float));
+        gemm_ref->fn(k, ref_out.as<float>(), kNC, weight.tensor->data, vy.data(), kGemmRows, kNC);
+        const TimingResult ref_time = time_calls(
+            [&] { gemm_ref->fn(k, ref_out.as<float>(), kNC, weight.tensor->data, vy.data(), kGemmRows, kNC); });
+        const double ref_flops = 2.0 * k * kNC * kGemmRows;
+
+        BenchRow row;
+        row.label = key.label;
+        row.ref_name = gemm_ref->name;
+        row.ref_ns = ref_time.median_ns;
+        row.ref_throughput = gflops(ref_flops, ref_time.median_ns);
+
+        for (const auto &cand : registries.repack_gemm.candidates(key.label)) {
+            BenchCandidate bc;
+            bc.name = cand.name;
+            bc.identical = (cand.fn == gemm_ref->fn);
+            if (!bc.identical) {
+                AlignedBuffer cand_out(kGemmRows * kNC * sizeof(float));
+                cand.fn(k, cand_out.as<float>(), kNC, weight.tensor->data, vy.data(), kGemmRows, kNC);
+                bc.correct = floats_match(ref_out.as<float>(), cand_out.as<float>(), kGemmRows * kNC);
+
+                const TimingResult cand_time = time_calls(
+                    [&] { cand.fn(k, cand_out.as<float>(), kNC, weight.tensor->data, vy.data(), kGemmRows, kNC); });
+                bc.ns = cand_time.median_ns;
+                bc.throughput = gflops(ref_flops, cand_time.median_ns);
+                bc.speedup = ref_time.median_ns / cand_time.median_ns;
+            }
+            row.candidates.push_back(bc);
+        }
+        print_row(row, gemm_report.throughput_unit);
+        gemm_report.rows.push_back(std::move(row));
+    }
+
+    return {quant_mat_report, gemv_report, gemm_report};
+}
diff --git a/apps/ggml/src/bench_vecdot.cpp b/apps/ggml/src/bench_vecdot.cpp
new file mode 100644
index 000000000000..c2e522794d28
--- /dev/null
+++ b/apps/ggml/src/bench_vecdot.cpp
@@ -0,0 +1,116 @@
+#include "benchmarks.h"
+
+#include <ggml-cpu.h>
+#include <ggml.h>
+
+#include "compare.h"
+#include "data_gen.h"
+#include "timing.h"
+
+#include <cstdlib>
+#include <cstring>
+
+namespace {
+// Divisible by every quant block size in play (32 for the q4/q5/q8 family, 256 for the k-quants).
+// Dev iteration aid: KERNEL_BENCH_N overrides it (must stay a multiple of 256),
+// which is how per-call overhead is separated from per-block cost.
+int64_t elements() {
+    const char *n = std::getenv("KERNEL_BENCH_N");
+    return (n && *n) ? std::atoll(n) : 4096;
+}
+const int64_t kElements = elements();
+
+// Dev iteration aid: KERNEL_BENCH_FILTER=q4_0,q8_0 restricts the sweep to
+// matching type names (substring match, comma-separated). Empty => all.
+bool passes_filter(const char *name) {
+    const char *f = std::getenv("KERNEL_BENCH_FILTER");
+    if (!f || !*f) {
+        return true;
+    }
+    std::string filt(f), item;
+    size_t pos = 0;
+    while (pos <= filt.size()) {
+        size_t comma = filt.find(',', pos);
+        if (comma == std::string::npos) comma = filt.size();
+        item = filt.substr(pos, comma - pos);
+        if (!item.empty() && std::strstr(name, item.c_str())) {
+            return true;
+        }
+        pos = comma + 1;
+    }
+    return false;
+}
+}  // namespace
+
+BenchReport run_vecdot_benchmarks(const KernelRegistries &registries) {
+    BenchReport report{"vec_dot", "GB/s", {}};
+    print_report_header(report.title, report.throughput_unit);
+
+    for (int t = 0; t < GGML_TYPE_COUNT; ++t) {
+        const ggml_type type = static_cast<ggml_type>(t);
+        const Impl<vec_dot_fn_t> *ref = registries.vec_dot.reference(type);
+        if (!ref) {
+            continue;
+        }
+        if (!passes_filter(ggml_type_name(type))) {
+            continue;
+        }
+
+        const ggml_type_traits_cpu *tc = ggml_get_type_traits_cpu(type);
+        if (!tc) {
+            continue;
+        }
+        const ggml_type act_type = tc->vec_dot_type;
+
+        const Impl<quantize_fn_t> *x_quant = registries.quantize.reference(type);
+        const Impl<quantize_fn_t> *y_quant = registries.quantize.reference(act_type);
+        if (!x_quant || !y_quant) {
+            continue;  // shouldn't happen for any type reachable through the CPU backend
+        }
+        ggml_quantize_init(type);  // one-time, cheap after the first call for this type; see ggml_provider.cpp
+
+        AlignedBuffer x_src(kElements * sizeof(float));
+        AlignedBuffer y_src(kElements * sizeof(float));
+        generate_synthetic_data(x_src.as<float>(), kElements, 0.0f);
+        generate_synthetic_data(y_src.as<float>(), kElements, 7.0f);  // different phase so x != y
+
+        AlignedBuffer vx(ggml_row_size(type, kElements));
+        AlignedBuffer vy(ggml_row_size(act_type, kElements));
+        x_quant->fn(x_src.as<float>(), vx.data(), kElements);
+        y_quant->fn(y_src.as<float>(), vy.data(), kElements);
+
+        float ref_result = 0.0f;
+        ref->fn(kElements, &ref_result, 0, vx.data(), 0, vy.data(), 0, 1);
+        const TimingResult ref_time =
+            time_calls([&] { ref->fn(kElements, &ref_result, 0, vx.data(), 0, vy.data(), 0, 1); });
+
+        BenchRow row;
+        row.label = ggml_type_name(type);
+        row.ref_name = ref->name;
+        row.ref_ns = ref_time.min_ns;
+        row.ref_throughput = bytes_per_sec(vx.size() + vy.size(), ref_time.min_ns) / 1e9;
+
+        for (const auto &cand : registries.vec_dot.candidates(type)) {
+            BenchCandidate bc;
+            bc.name = cand.name;
+            bc.identical = (cand.fn == ref->fn);
+            if (!bc.identical) {
+                float cand_result = 0.0f;
+                cand.fn(kElements, &cand_result, 0, vx.data(), 0, vy.data(), 0, 1);
+                bc.correct = floats_match(&ref_result, &cand_result, 1);
+
+                const TimingResult cand_time =
+                    time_calls([&] { cand.fn(kElements, &cand_result, 0, vx.data(), 0, vy.data(), 0, 1); });
+                bc.ns = cand_time.min_ns;
+                bc.throughput = bytes_per_sec(vx.size() + vy.size(), cand_time.min_ns) / 1e9;
+                bc.speedup = ref_time.min_ns / cand_time.min_ns;
+            }
+            row.candidates.push_back(bc);
+        }
+
+        print_row(row, report.throughput_unit);
+        report.rows.push_back(std::move(row));
+    }
+
+    return report;
+}
diff --git a/apps/ggml/src/benchmarks.h b/apps/ggml/src/benchmarks.h
new file mode 100644
index 000000000000..75029adbc44d
--- /dev/null
+++ b/apps/ggml/src/benchmarks.h
@@ -0,0 +1,13 @@
+#pragma once
+
+#include <vector>
+
+#include "kernel_registry.h"
+#include "report.h"
+
+BenchReport run_quantize_benchmarks(const KernelRegistries &registries);
+BenchReport run_dequantize_benchmarks(const KernelRegistries &registries);
+BenchReport run_vecdot_benchmarks(const KernelRegistries &registries);
+
+// One report each for quantize_mat, gemv, gemm.
+std::vector<BenchReport> run_repack_benchmarks(const KernelRegistries &registries);
diff --git a/apps/ggml/src/compare.h b/apps/ggml/src/compare.h
new file mode 100644
index 000000000000..b056c3c0d8fe
--- /dev/null
+++ b/apps/ggml/src/compare.h
@@ -0,0 +1,20 @@
+#pragma once
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+
+// Relative-error comparison for floating point outputs (dequantize, vec_dot,
+// gemv/gemm results). Quantize/quantize_mat outputs are compared with an
+// exact memcmp instead (see bench_quantize.cpp) since those algorithms are
+// specified to be bit-identical across implementations.
+inline bool floats_match(const float *a, const float *b, int64_t n, float rel_tol = 1e-2f) {
+    for (int64_t i = 0; i < n; ++i) {
+        const float diff = std::fabs(a[i] - b[i]);
+        const float scale = std::max(std::fabs(a[i]), 1e-6f);
+        if (diff / scale > rel_tol) {
+            return false;
+        }
+    }
+    return true;
+}
diff --git a/apps/ggml/src/data_gen.h b/apps/ggml/src/data_gen.h
new file mode 100644
index 000000000000..22075c369abf
--- /dev/null
+++ b/apps/ggml/src/data_gen.h
@@ -0,0 +1,60 @@
+#pragma once
+
+// Deterministic synthetic data + aligned buffers, following the conventions
+// of tests/test-quantize-perf.cpp (same generator, same rationale: a fixed
+// seedless formula so every implementation under comparison sees byte-identical
+// input without carrying a PRNG dependency).
+
+#include <cmath>
+#include <cstdint>
+#include <cstdlib>
+#include <memory>
+#include <vector>
+
+inline void generate_synthetic_data(float *dst, size_t n, float offset = 0.0f) {
+    for (size_t i = 0; i < n; ++i) {
+        dst[i] = 0.1f + 2.0f * cosf(static_cast<float>(i) + offset);
+    }
+}
+
+// 64-byte aligned heap buffer (covers every SIMD width in use: SSE/AVX/AVX-512/NEON/SVE).
+class AlignedBuffer {
+public:
+    explicit AlignedBuffer(size_t bytes) : size_(bytes) {
+        constexpr size_t alignment = 64;
+        size_t padded = (bytes + alignment - 1) / alignment * alignment;
+        if (padded == 0) {
+            padded = alignment;
+        }
+        ptr_ = nullptr;
+        posix_memalign(&ptr_, alignment, padded);
+    }
+    ~AlignedBuffer() {
+        std::free(ptr_);
+    }
+
+    AlignedBuffer(const AlignedBuffer &) = delete;
+    AlignedBuffer &operator=(const AlignedBuffer &) = delete;
+
+    void *data() {
+        return ptr_;
+    }
+    const void *data() const {
+        return ptr_;
+    }
+    template<typename T>
+    T *as() {
+        return static_cast<T *>(ptr_);
+    }
+    template<typename T>
+    const T *as() const {
+        return static_cast<const T *>(ptr_);
+    }
+    size_t size() const {
+        return size_;
+    }
+
+private:
+    void *ptr_;
+    size_t size_;
+};
diff --git a/apps/ggml/src/main.cpp b/apps/ggml/src/main.cpp
new file mode 100644
index 000000000000..24cb441426f8
--- /dev/null
+++ b/apps/ggml/src/main.cpp
@@ -0,0 +1,122 @@
+#include <cstdio>
+#include <fstream>
+#include <string>
+
+#include <ggml-cpu.h>
+
+#include "benchmarks.h"
+#include "ggml_provider.h"
+#include "halide_provider.h"
+#include "kernel_registry.h"
+
+namespace {
+
+// The repack buffer type logs a GGML_LOG_DEBUG line on every repack (see
+// src/ggml-cpu/repack.cpp:4733) -- benign, but this benchmark triggers many
+// of them (one per gemv/gemm sample weight built), so drop DEBUG/INFO noise
+// and keep only warnings/errors.
+void quiet_log_callback(ggml_log_level level, const char *text, void *) {
+    if (level >= GGML_LOG_LEVEL_WARN) {
+        std::fputs(text, stderr);
+    }
+}
+
+void print_ggml_version() {
+#ifdef KERNEL_BENCH_GGML_VERSION
+    std::printf("GGML version: %s\n", KERNEL_BENCH_GGML_VERSION);
+#else
+    std::printf("GGML version: unknown (GGML_VERSION not set by ggml-config.cmake)\n");
+#endif
+}
+
+void print_cpu_features() {
+    std::printf("CPU features:");
+#if defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)
+    if (ggml_cpu_has_avx()) std::printf(" avx");
+    if (ggml_cpu_has_avx2()) std::printf(" avx2");
+    if (ggml_cpu_has_avx512()) std::printf(" avx512");
+    if (ggml_cpu_has_avx512_vnni()) std::printf(" avx512_vnni");
+    if (ggml_cpu_has_fma()) std::printf(" fma");
+    if (ggml_cpu_has_f16c()) std::printf(" f16c");
+    if (ggml_cpu_has_amx_int8()) std::printf(" amx_int8");
+#endif
+#if defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64)
+    if (ggml_cpu_has_neon()) std::printf(" neon");
+    if (ggml_cpu_has_dotprod()) std::printf(" dotprod");
+    if (ggml_cpu_has_matmul_int8()) std::printf(" matmul_int8");
+    if (ggml_cpu_has_fp16_va()) std::printf(" fp16_va");
+    if (ggml_cpu_has_sve()) std::printf(" sve(%d bytes)", ggml_cpu_get_sve_cnt());
+    if (ggml_cpu_has_sme()) std::printf(" sme");  // codespell:ignore sme
+#endif
+    std::printf("\n");
+}
+
+void usage(const char *argv0) {
+    std::printf("usage: %s [--quantize] [--dequantize] [--vecdot] [--repack] [--all] [--csv FILE]\n", argv0);
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+    ggml_log_set(quiet_log_callback, nullptr);
+
+    bool do_quantize = false, do_dequantize = false, do_vecdot = false, do_repack = false;
+    std::string csv_path;
+
+    for (int i = 1; i < argc; ++i) {
+        const std::string arg = argv[i];
+        if (arg == "--quantize") do_quantize = true;
+        else if (arg == "--dequantize")
+            do_dequantize = true;
+        else if (arg == "--vecdot")
+            do_vecdot = true;
+        else if (arg == "--repack")
+            do_repack = true;
+        else if (arg == "--all")
+            do_quantize = do_dequantize = do_vecdot = do_repack = true;
+        else if (arg == "--csv" && i + 1 < argc)
+            csv_path = argv[++i];
+        else if (arg == "--help" || arg == "-h") {
+            usage(argv[0]);
+            return 0;
+        } else {
+            std::fprintf(stderr, "unknown argument: %s\n", arg.c_str());
+            usage(argv[0]);
+            return 1;
+        }
+    }
+    if (!do_quantize && !do_dequantize && !do_vecdot && !do_repack) {
+        do_quantize = do_dequantize = do_vecdot = do_repack = true;  // default: --all
+    }
+
+    print_ggml_version();
+    print_cpu_features();
+
+    KernelRegistries registries;
+    register_ggml_provider(registries);
+    register_halide_provider(registries);
+
+    // Each run_*_benchmarks() call prints its own header and streams a row
+    // to stdout as soon as that row is computed (see report.h/print_row) --
+    // results become visible immediately rather than only after everything
+    // finishes. The returned reports are only needed here for --csv.
+    std::vector<BenchReport> reports;
+    if (do_quantize) reports.push_back(run_quantize_benchmarks(registries));
+    if (do_dequantize) reports.push_back(run_dequantize_benchmarks(registries));
+    if (do_vecdot) reports.push_back(run_vecdot_benchmarks(registries));
+    if (do_repack) {
+        for (auto &r : run_repack_benchmarks(registries)) {
+            reports.push_back(std::move(r));
+        }
+    }
+
+    if (!csv_path.empty()) {
+        std::ofstream out(csv_path);
+        for (const auto &report : reports) {
+            write_report_csv(report, out);
+        }
+        std::printf("\nwrote %s\n", csv_path.c_str());
+    }
+
+    return 0;
+}
diff --git a/apps/ggml/src/report.cpp b/apps/ggml/src/report.cpp
new file mode 100644
index 000000000000..6527b10967ed
--- /dev/null
+++ b/apps/ggml/src/report.cpp
@@ -0,0 +1,50 @@
+#include "report.h"
+
+#include <cstdio>
+#include <ostream>
+
+void print_report_header(const std::string &title, const std::string &throughput_unit) {
+    std::printf("\n=== %s (%s) ===\n", title.c_str(), throughput_unit.c_str());
+    std::fflush(stdout);
+}
+
+void print_row(const BenchRow &row, const std::string &throughput_unit) {
+    std::printf("  %-20s  reference=%-10s %8.1f ns  %8.2f %s\n", row.label.c_str(), row.ref_name.c_str(), row.ref_ns,
+                row.ref_throughput, throughput_unit.c_str());
+    if (row.candidates.empty()) {
+        std::printf("  %-20s    (no candidates registered yet)\n", "");
+    }
+    for (const auto &c : row.candidates) {
+        if (c.identical) {
+            std::printf("  %-20s    %-12s  identical to reference\n", "", c.name.c_str());
+            continue;
+        }
+        std::printf("  %-20s    %-12s  %8.1f ns  %8.2f %s  %6.2fx%s\n", "", c.name.c_str(), c.ns, c.throughput,
+                    throughput_unit.c_str(), c.speedup, c.correct ? "" : "  [MISMATCH vs reference]");
+    }
+    std::fflush(stdout);
+}
+
+void print_report(const BenchReport &report) {
+    print_report_header(report.title, report.throughput_unit);
+    if (report.rows.empty()) {
+        std::printf("  (nothing registered)\n");
+        std::fflush(stdout);
+        return;
+    }
+    for (const auto &row : report.rows) {
+        print_row(row, report.throughput_unit);
+    }
+}
+
+void write_report_csv(const BenchReport &report, std::ostream &out) {
+    out << "table,label,role,name,ns,throughput_" << report.throughput_unit << ",speedup,identical,correct\n";
+    for (const auto &row : report.rows) {
+        out << report.title << ',' << row.label << ",reference," << row.ref_name << ',' << row.ref_ns << ','
+            << row.ref_throughput << ",1.0,0,1\n";
+        for (const auto &c : row.candidates) {
+            out << report.title << ',' << row.label << ",candidate," << c.name << ',' << c.ns << ',' << c.throughput
+                << ',' << c.speedup << ',' << (c.identical ? 1 : 0) << ',' << (c.correct ? 1 : 0) << '\n';
+        }
+    }
+}
diff --git a/apps/ggml/src/report.h b/apps/ggml/src/report.h
new file mode 100644
index 000000000000..265cab2e10eb
--- /dev/null
+++ b/apps/ggml/src/report.h
@@ -0,0 +1,42 @@
+#pragma once
+
+#include <iosfwd>
+#include <string>
+#include <vector>
+
+struct BenchCandidate {
+    std::string name;
+    double ns = 0.0;
+    double throughput = 0.0;
+    double speedup = 0.0;    // reference.ns / ns
+    bool identical = false;  // candidate fn pointer == reference fn pointer
+    bool correct = true;     // output matched the reference within tolerance
+};
+
+struct BenchRow {
+    std::string label;  // type name, or repack key label
+    double ref_ns = 0.0;
+    double ref_throughput = 0.0;
+    std::string ref_name;
+    std::vector<BenchCandidate> candidates;
+};
+
+struct BenchReport {
+    std::string title;
+    std::string throughput_unit;  // "GB/s" or "GFLOP/s"
+    std::vector<BenchRow> rows;
+};
+
+// Incremental printing: call print_report_header() once, then print_row()
+// as each row is computed (bench_*.cpp interleaves this with the actual
+// benchmarking so results stream out immediately instead of only appearing
+// after the whole category finishes). Both flush stdout so the stream is
+// visible immediately even when redirected/piped, not just on a tty.
+void print_report_header(const std::string &title, const std::string &throughput_unit);
+void print_row(const BenchRow &row, const std::string &throughput_unit);
+
+// Convenience wrapper for a fully-built report (used for the "nothing
+// registered" case, and anywhere the whole report is already in hand).
+void print_report(const BenchReport &report);
+
+void write_report_csv(const BenchReport &report, std::ostream &out);
diff --git a/apps/ggml/src/timing.h b/apps/ggml/src/timing.h
new file mode 100644
index 000000000000..ad78e859c6b8
--- /dev/null
+++ b/apps/ggml/src/timing.h
@@ -0,0 +1,73 @@
+#pragma once
+
+#include <algorithm>
+#include <chrono>
+#include <cstdint>
+#include <functional>
+#include <vector>
+
+struct TimingResult {
+    double min_ns = 0.0;
+    double median_ns = 0.0;
+};
+
+// Runs `fn` `warmup` times (discarded), then calibrates a batch size large
+// enough that timing a whole batch back-to-back amortizes clock overhead and
+// resolution (individual quantize_row/vec_dot calls on fast SIMD kernels can
+// complete in a few nanoseconds -- timing them one at a time, even with a
+// high-resolution clock, is dominated by noise), then times `iters` such
+// batches and returns the min/median per-call latency in nanoseconds.
+inline TimingResult time_calls(const std::function<void()> &fn, int warmup = 5, int iters = 20) {
+    using clock = std::chrono::steady_clock;
+
+    for (int i = 0; i < warmup; ++i) {
+        fn();
+    }
+
+    constexpr double kMinBatchNs = 200000.0;  // 0.2ms per batch
+    int batch = 1;
+    for (;;) {
+        const auto t0 = clock::now();
+        for (int i = 0; i < batch; ++i) {
+            fn();
+        }
+        const auto t1 = clock::now();
+        const double batch_ns = std::chrono::duration<double, std::nano>(t1 - t0).count();
+        if (batch_ns >= kMinBatchNs || batch >= (1 << 20)) {
+            break;
+        }
+        batch *= 4;
+    }
+
+    std::vector<double> samples_ns;
+    samples_ns.reserve(iters);
+    for (int i = 0; i < iters; ++i) {
+        const auto t0 = clock::now();
+        for (int b = 0; b < batch; ++b) {
+            fn();
+        }
+        const auto t1 = clock::now();
+        const double batch_ns = std::chrono::duration<double, std::nano>(t1 - t0).count();
+        samples_ns.push_back(batch_ns / batch);
+    }
+
+    std::sort(samples_ns.begin(), samples_ns.end());
+    TimingResult result;
+    result.min_ns = samples_ns.front();
+    result.median_ns = samples_ns[samples_ns.size() / 2];
+    return result;
+}
+
+inline double bytes_per_sec(size_t bytes, double ns) {
+    if (ns <= 0.0) {
+        return 0.0;
+    }
+    return static_cast<double>(bytes) / (ns * 1e-9);
+}
+
+inline double gflops(double flops, double ns) {
+    if (ns <= 0.0) {
+        return 0.0;
+    }
+    return flops / (ns * 1e-9) / 1e9;
+}
diff --git a/apps/ggml/vcpkg-configuration.json b/apps/ggml/vcpkg-configuration.json
new file mode 100644
index 000000000000..a0daf83101ff
--- /dev/null
+++ b/apps/ggml/vcpkg-configuration.json
@@ -0,0 +1,5 @@
+{
+  "overlay-ports": [
+    "../vcpkg/ports"
+  ]
+}
diff --git a/apps/ggml/vcpkg.json b/apps/ggml/vcpkg.json
new file mode 100644
index 000000000000..bac5a1fefe0f
--- /dev/null
+++ b/apps/ggml/vcpkg.json
@@ -0,0 +1,9 @@
+{
+  "name": "halide-ggml-app",
+  "version": "22.0.0",
+  "license": "MIT",
+  "builtin-baseline": "66c0373dc7fca549e5803087b9487edfe3aca0a1",
+  "dependencies": [
+    "ggml"
+  ]
+}
diff --git a/apps/vcpkg.json b/apps/vcpkg.json
index 1c3c5805f7ca..381ba6e211ac 100644
--- a/apps/vcpkg.json
+++ b/apps/vcpkg.json
@@ -10,6 +10,7 @@
       "platform": "(windows & x64 & !uwp & !xbox) | (linux & x64) | (linux & arm64)"
     },
     "eigen3",
+    "ggml",
     "libjpeg-turbo",
     "libpng",
     "onnx",
diff --git a/apps/vcpkg/ports/ggml/portfile.cmake b/apps/vcpkg/ports/ggml/portfile.cmake
new file mode 100644
index 000000000000..942b99f5f5b4
--- /dev/null
+++ b/apps/vcpkg/ports/ggml/portfile.cmake
@@ -0,0 +1,26 @@
+vcpkg_check_linkage(ONLY_STATIC_LIBRARY)
+
+vcpkg_from_github(
+    OUT_SOURCE_PATH SOURCE_PATH
+    REPO ggml-org/ggml
+    REF eced84c86f8b012c752c016f7fe789adea168e1e  # v0.15.3
+    SHA512 3295c064aff295b0387249d5dec7860b620de82c8361197888df186be18270ede253ab7bce3358b1fb3020f11d01f0f8a29f7d268bff44666f8a8f3ea832781e
+)
+
+# We set GGML_BACKEND_DL=OFF to keep the CPU backend linked, not dlopen'ed,
+# because apps/ggml needs ggml-cpu's internal symbols at link time.
+vcpkg_cmake_configure(
+    SOURCE_PATH "${SOURCE_PATH}"
+    OPTIONS
+    -DBUILD_SHARED_LIBS=OFF
+    -DGGML_BACKEND_DL=OFF
+    -DGGML_BUILD_TESTS=OFF
+    -DGGML_BUILD_EXAMPLES=OFF
+)
+
+vcpkg_cmake_install()
+vcpkg_cmake_config_fixup(PACKAGE_NAME ggml CONFIG_PATH lib/cmake/ggml)
+
+vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE" "${SOURCE_PATH}/AUTHORS")
+
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include" "${CURRENT_PACKAGES_DIR}/debug/share")
diff --git a/apps/vcpkg/ports/ggml/vcpkg.json b/apps/vcpkg/ports/ggml/vcpkg.json
new file mode 100644
index 000000000000..043c2b93892d
--- /dev/null
+++ b/apps/vcpkg/ports/ggml/vcpkg.json
@@ -0,0 +1,17 @@
+{
+  "name": "ggml",
+  "version": "0.15.3",
+  "description": "Tensor library for machine learning",
+  "homepage": "https://github.com/ggml-org/ggml",
+  "license": "MIT",
+  "dependencies": [
+    {
+      "name": "vcpkg-cmake",
+      "host": true
+    },
+    {
+      "name": "vcpkg-cmake-config",
+      "host": true
+    }
+  ]
+}
diff --git a/doc/ApproximationDesign.md b/doc/ApproximationDesign.md
new file mode 100644
index 000000000000..0d87bea2df2d
--- /dev/null
+++ b/doc/ApproximationDesign.md
@@ -0,0 +1,424 @@
+# Approximation: a core concept for lossy, quantified Func substitution
+
+Status: design draft, no implementation yet.
+
+## Motivation
+
+Two pieces of prior work motivate this:
+
+1. `apps/ggml/halide/*_generators.cpp` hand-implements ~24 GGML quantized weight
+   formats as pairs of Halide Generators (quantize, dequantize), each file
+   independently encoding its own byte layout, scale/bias math, and (for
+   K-quants and a few others) a scaffolded extern-stage call to GGML's own
+   reference quantizer pending a native port. This works, but every type
+   duplicates the same shape of logic (block layout, scale search, bit-packing)
+   by hand, and the "quantize happens once offline, dequantize happens on every
+   inference call" relationship between the two directions is enforced by
+   nothing but convention and comments.
+
+2. A private Python research prototype explores building the same set of formats
+   compositionally instead: a small `Approximation` ABC (`encode`/`decode`, each
+   operating on Halide `Func`s) with a handful of primitives (block layout
+   reshaping, a linear integer quantizer, a shift-by-min helper for affine
+   schemes, bit-packers) that compose to reconstruct all of Q2_K..Q8_K. It's
+   Python-only, JIT-only, and only covers quantize/dequantize round trips (no
+   vec_dot, no repack, no scheduling story).
+
+This document proposes promoting the core of that prototype (`Approximation`)
+into a first-class Halide C++ concept, plus two small pieces of surrounding API
+(`approximate_by`, `compute_offline`) needed to wire an `Approximation` into a
+real pipeline's call graph and, optionally, split it across a compile-time
+boundary.
+
+## Why this can't just be ordinary scheduling
+
+Halide's algorithm/schedule separation depends on schedule directives being
+meaning-preserving: `.compute_root()` vs `.compute_at()` never changes what a
+pipeline computes, only how. An `Approximation` is the opposite by design — it
+deliberately changes the *value* computed (a real weight becomes a
+quantized-then-dequantized approximation of itself), in a bounded, quantified
+way. Wiring that in by disguising it as an ordinary Func substitution (e.g. a
+custom `.in()` wrapper with no other marking) would make a semantics-changing
+operation look, to any future reader, like a semantics-preserving one. It needs
+its own footing in the API.
+
+## Core concept: `Approximation`
+
+```cpp
+// encode()/decode() each return (funcs, handles): the "public" Func(s) that
+// participate in the signature contract, plus extra intermediate Funcs
+// (reduction accumulators, per-block stats, etc.) that have no meaning
+// outside scheduling but still need someone to schedule them -- see
+// "approximate_by" below for why silently dropping handles is a real bug,
+// not a simplification.
+struct EncodeResult {
+    std::vector<Func> encoded;  // the signature-contract output(s)
+    std::vector<Func> handles;  // scheduling-only, no semantic meaning
+};
+struct DecodeResult {
+    std::vector<Func> decoded;  // decoded[0] is the round-trip replacement
+    std::vector<Func> handles;
+};
+
+class Approximation {
+public:
+    virtual ~Approximation() = default;
+
+    // Both operate purely on Funcs -- no opinion about placement (compute
+    // root vs fused, offline vs online). See "Scope: placement is not
+    // semantics" below for why that split matters.
+    virtual EncodeResult encode(Func f) = 0;
+    virtual DecodeResult decode(std::vector<Func> encoded) = 0;
+};
+```
+
+Virtual dispatch is used specifically because composed Approximations
+(`Compose`/`Apply`, below) need to hold a runtime-heterogeneous list of
+`Approximation` references and call `encode`/`decode` on each polymorphically.
+(A templated/CRTP alternative was considered and rejected for this reason: it
+would make composed, heterogeneous chains require type erasure some other way,
+for no benefit here.)
+
+**Signature contract.** `decode(encode(f).encoded).decoded[0]` must reproduce
+`f`'s arg list and value type exactly — that's what makes it valid to splice
+back into a call graph in place of `f`. This is not proposed to be enforced
+generically by the base class in v1; each concrete `Approximation` is
+responsible for it, treating round-trip error and convergence as a testable
+property rather than a type-level guarantee. `approximate_by`'s substitution
+step (see below) does do a shape/type check at the point of substitution, which
+catches violations, just not at `Approximation`-definition time.
+
+**`encode`'s output arity is scheme-dependent, and that's fine.** This follows
+directly from a memory-layout choice each `Approximation` makes:
+
+- *Packed*: one opaque `Buffer<uint8_t>`, fields recovered via
+  `reinterpret<T>()` at fixed byte offsets inside `decode`. This is what every
+  type in `apps/ggml/halide` already does today (`Buffer<uint8_t, 2>` dim 0 =
+  byte-within-block), because Halide has no struct type to give a K-quant's
+  `(d, dmin, scales[12], qs[128])` a real typed signature.
+- *Planar*: multiple typed Funcs (e.g. a separate `float16_t` delta Func, a
+  separate `int8_t` quants Func), more Halide-native and type-safe, but the
+  Generator's public signature grows with the scheme's field count.
+
+Both are legitimate; `Approximation` doesn't pick one. A Generator wrapping an
+`Approximation` (see below) ends up with a scheme-dependent public signature
+either way — accepted as a consequence of this choice, not something the
+framework tries to paper over.
+
+## `approximate_by`: wiring an `Approximation` into a call graph
+
+### Rejected first approach: building on `Func::in`
+
+`Func::in(g)` (`src/Func.cpp:2456`) looked like a natural fit initially: it
+registers a wrapper Function in `f`'s `FuncSchedule::wrappers()` map, which
+`WrapCalls.cpp::wrap_func_calls` later rewrites `g`'s `Call` nodes against. The
+problem is *later*: that rewrite only happens during `lower()`, well after
+`configure()`/`generate()`/`schedule()` have all already run. Anything that
+needs to reason about "what does `g` actually call" before that point — in
+particular, `compute_offline`'s `configure()`-time branching — would be looking
+at stale, pre-substitution state. Using `.in()` here would make `approximate_by`
+and `compute_offline` fundamentally unable to compose in the same pipeline.
+
+### The actual mechanism: eager, destructive, like `rfactor`
+
+`Stage::rfactor` (`src/Func.cpp:1001`) is the right precedent instead. It never
+defers to a lowering pass: it builds a new `Func intm(...)`, calls
+`intm.function().define_update(...)` immediately, and rewrites the *original*
+Function's own definition via `substitute_self_reference` — all synchronously,
+as part of the `.rfactor()` call itself. By the time `.rfactor()` returns, the
+graph already reflects the change, which is why a caller can immediately turn
+around and schedule `intm` in the same breath.
+
+`approximate_by` should behave the same way, using the same class of primitive
+`WrapCalls.cpp` already relies on internally —
+`Function::substitute_calls(SubstitutionMap)` — but invoked immediately,
+directly on an explicitly-given set of consumers, instead of registered for a
+later pass:
+
+`approximate_by` is a member of `Func` (`f.approximate_by(p, consumers)`), not a
+free function — it's a graph-editing operation on `f` in exactly the same sense
+`f.in(...)`/`f.rfactor(...)` are, so it should read like the rest of that family
+instead of standing apart as a free function taking `f` as its first argument:
+
+```cpp
+struct ApproximationResult {
+    Func replacement;           // decode's round-trip output; already
+                                 // spliced into every Func in `consumers`
+    std::vector<Func> handles;  // encode's output(s) + encode's handles +
+                                 // decode's handles -- all need scheduling,
+                                 // none are part of the signature contract
+};
+
+// Func.h: ApproximationResult approximate_by(Approximation &p, const std::vector<Func> &consumers);
+ApproximationResult Func::approximate_by(Approximation &p, const std::vector<Func> &consumers) {
+    EncodeResult enc = p.encode(*this);
+    DecodeResult dec = p.decode(enc.encoded);
+    Func round_trip = dec.decoded[0];  // signature contract: matches *this exactly
+
+    for (const Func &g : consumers) {
+        // eager, destructive -- happens now, not at lowering time
+        g.function().substitute_calls(func, round_trip.function());
+    }
+
+    std::vector<Func> handles = enc.encoded;
+    handles.insert(handles.end(), enc.handles.begin(), enc.handles.end());
+    handles.insert(handles.end(), dec.handles.begin(), dec.handles.end());
+    return {round_trip, handles};
+}
+```
+
+(`func` is `Func`'s own private `Internal::Function` member; `g.function()` is
+the public accessor for a different `Func`'s. `Function::substitute_calls`
+already has a single-pair overload taking `(orig, substitute)` directly, so no
+manual `SubstitutionMap` construction is needed.)
+
+This needs **no new public primitive at all**, which is a smaller change than
+either of the two things earlier drafts of this design proposed (a `Func::in`
+overload, or a public wrapper around `substitute_calls`):
+`Function::substitute_calls` is already an ordinary method on the internal
+`Function` class, and `approximate_by` is implemented inside libHalide itself
+(`Approximation.h`/`Func.cpp`), so it can call it directly — the "new public
+entry point" concern only would have applied if this were being built as
+external, non-core code.
+
+**Returning `handles` is not optional.** Both `encode` and `decode` can
+introduce intermediate Funcs with update definitions (per-block reductions, a
+shift-by-min helper's own min-reduction, etc.). Left unscheduled, Halide doesn't
+error on these — it computes them at the innermost valid loop level by default
+(a Func can only fail to compile this way if something explicitly forces
+`.compute_inline()` on it, which is illegal for a Func with an update
+definition). But that default placement is exactly that, a default: the caller
+has no way to override it, or to apply the fusion patterns from "Scope:
+placement is not semantics" below (e.g. `compute_at`-ing `enc.encoded` into a
+producer for dynamic activation requantization). The struct above bundles every
+encode/decode handle together for exactly this reason — so the caller can
+schedule all of them, not just the primary output.
+
+### Consequence: consumers must already exist
+
+Because the substitution is eager, `approximate_by` can only rewrite Funcs that
+are already built at the point of the call — there is no equivalent of `.in()`'s
+global mode (redirect *every* current and future consumer, including ones not
+yet written). This is a real capability loss relative to `.in()`, but it's the
+same scoping `rfactor` already lives with (it never retroactively touches
+definitions that don't exist yet either), and it matches how Generator code is
+actually written: within `generate()`, `f` and its consumer(s) are typically
+built in the same breath, so passing `consumers` explicitly costs nothing there.
+It only forecloses a genuinely different use case — transparently intercepting
+calls inside some large, opaque, externally-authored algorithm without being
+able to enumerate its call sites — which is out of scope for this design.
+
+## Scope: placement is not semantics
+
+An `Approximation`'s `encode`/`decode` never make any claim about *where* or
+*when* they're computed relative to the rest of the pipeline. This was initially
+proposed otherwise — an early draft of this design suggested that `encode`'s
+output could just always be treated as "the offline half" — and was rejected on
+a concrete counterexample: **dynamic activation requantization**.
+
+Contrast:
+
+- **Static weight quantization**: `encode` (e.g. Q4_0 quantize) runs exactly
+  once, ever, fully decoupled from any inference call — a genuine
+  compile-time/compilation-unit boundary. `decode` gets fused inline into the
+  consumer's inner loop — already exactly what every `*VecDotGenerator` in
+  `apps/ggml/halide` does (e.g. `ggml_halide::q8_0_value(y_blocks_, r)` called
+  inline inside `sum()`, never materialized as its own Func).
+- **Dynamic activation requantization**: `encode` (quantize a just-computed
+  activation tile) needs to be fused into the *producer's* schedule — same
+  granularity, same loop nest, no separate storage, recomputed every call.
+  `decode` is fused into the consumer's tiles exactly as before.
+
+Same `Approximation`, opposite treatment of where `encode` is computed. If
+"encode ⇒ offline" were baked into the interface, the activation case would need
+an escape hatch to override it — at which point the shortcut has bought nothing,
+and worse, it would invite tooling to assume every quantize step is safe to
+hoist to conversion time, which is a real correctness trap for anything computed
+at inference time.
+
+**Consequence, and a scope reduction**: the "fuse encode into producer" / "fuse
+decode into consumer tiles" cases need *no new Halide feature at all*. Ordinary
+`.compute_at()` / `.compute_inline()` on `ApproximationResult`'s `replacement`
+and `handles` (in particular `enc.encoded`, the piece that needs to be fused
+into the producer for the activation case) already achieves this, since they're
+just regular Funcs sitting in the call graph. `compute_offline` (below) is
+needed only for the strictly narrower case of actually severing the graph into
+two separately-compiled artifacts — the static-weight case.
+
+## `compute_offline`: v1 scope
+
+**Decision: "seam exposure," not automatic pipeline splitting.** Given one Func
+`f` that should become a compile-time boundary, `compute_offline`-style support
+means:
+
+- `f`'s computation exists in one compile, as a normal `Output` of a Generator
+  (the "offline" artifact).
+- A same-shaped `Input` (e.g. an opaque `Buffer<uint8_t>` for a packed layout)
+  exists in another compile, standing in for `f` (the "online" pipeline).
+- Both are ordinary, statically-declared Generator I/O — no dynamic discovery of
+  new ports mid-`generate()`.
+
+The alternative (true automatic splitting: one Generator definition, two
+artifacts emitted automatically, no extra static I/O declared by the author) was
+considered and rejected for v1: it would require a Generator to discover an
+extra Input/Output *during* `generate()`, based on the structure of a Func graph
+that doesn't exist yet when `configure()` runs and declares I/O — a
+phase-ordering problem, not just an ergonomics one.
+
+### Why this doesn't need new Generator machinery
+
+Generators already support dynamic I/O declared *before* `generate()` runs:
+`configure()` (`src/Generator.h:192`) exists specifically so `add_input<>()`/
+`add_output<>()` (`src/Generator.h:3236` ff.) can be called based on
+`GeneratorParam` values decided earlier. The only in-repo precedent
+(`apps/hannk/halide/conv_generator.cpp:75-81`) uses `configure()` narrowly (to
+pick a `Target`-dependent filter type), but the mechanism generalizes directly:
+`compute_offline`'s "seam" is just another `GeneratorParam`-driven decision
+about which ports to declare.
+
+### Composability with `approximate_by`
+
+Now that `approximate_by`'s substitution is eager (see above), this composes
+cleanly: whichever order `compute_offline`'s `configure()`-time branching and an
+`approximate_by` call run in, within the same `configure()`/`generate()`, the
+graph state at every point *is* the true state — no later lowering pass can
+silently change what a Func calls out from under code that already ran. In
+practice, a Generator authoring an op from scratch (the shape below) usually
+doesn't need `approximate_by` at all — it can just build `enc`/`dec` and
+reference `dec.decoded[0]` directly wherever the op's math needs the value,
+since it's writing the consumer fresh anyway. `approximate_by` earns its keep
+when the consumer already exists as written code the author doesn't want to edit
+by hand (e.g. a shared reduction reused across several ops) — and because it's
+now eager, nothing stops using it *inside* the same `generate()` that also does
+`compute_offline`-style branching, in either order.
+
+### Non-goal for v1: provenance checking
+
+Nothing proposed here guarantees that the `Approximation` used to produce the
+offline artifact in one compile is *actually* the same one the online compile
+expects when decoding it — correctness rests entirely on both sides referencing
+the same registry name (see the Generator shape below). Building real guarantees
+on top of that (e.g. embedding a scheme fingerprint in the artifact, checked at
+load time) is explicitly deferred to a future version; v1 accepts this as a user
+obligation, the same way `apps/ggml/halide`'s quantize/dequantize split already
+relies on convention today.
+
+## Generator shape
+
+There's very little in-repo precedent for a Generator whose `configure()` does
+real work beyond a `Target`/`GeneratorParam` type tweak (`conv_generator.cpp` is
+the only example). The proposed shape leans on the existing three-phase
+lifecycle (`call_configure()` → `call_generate()` → `call_schedule()`,
+`src/Generator.h:3826-3829`) much harder than any existing Generator does:
+
+- **`configure()`** is the real "compiler driver": it looks up `Approximation`
+  objects by name from an independent, non-Generator registry
+  (`make_approximation(name)`, a lookup table of named schemes), decides which
+  Inputs/Outputs to declare based on `GeneratorParam`s (per-argument scheme
+  choice, a `quantize_only` mode, a `weight_precomputed` flag selecting between
+  the split and combined-artifact shapes), and stores the selected
+  `Approximation`s as members.
+- **`generate()`** must exist (`static_assert(has_generate_method_v<T>)`,
+  `src/Generator.h:3901`) but can be a thin pass-through: it builds the actual
+  op math using the `Approximation`s selected in `configure()` (weight value via
+  `decode`, activation `encode`d and fused at whatever granularity, the op's own
+  reduction) and assigns the result to the `Output<>` members declared in
+  `configure()`.
+- **`schedule()`** is a thin dispatch reading further `GeneratorParam`s to pick
+  a named tiling/fusion strategy, rather than scheduling being written inline in
+  `generate()` (the style every existing Generator in this repo uses).
+
+```cpp
+class MatMulOp : public Generator<MatMulOp> {
+public:
+    GeneratorParam<std::string> weight_scheme_{"weight_scheme", "q4_0"};
+    GeneratorParam<std::string> activation_scheme_{"activation_scheme", "q8_0"};
+    GeneratorParam<std::string> quantize_only_{"quantize_only", ""};  // "" | "weight" | "activation"
+    GeneratorParam<bool> weight_precomputed_{"weight_precomputed", true};
+
+    void configure() {
+        weight_approx_ = make_approximation(weight_scheme_);
+        activation_approx_ = make_approximation(activation_scheme_);
+
+        if (!quantize_only_.value().empty()) {
+            x_ = add_input<Buffer<float>>("x", 1);
+            EncodeResult enc = (quantize_only_ == "weight" ? weight_approx_ : activation_approx_)->encode(Func(*x_));
+            for (size_t i = 0; i < enc.encoded.size(); i++) {
+                *add_output<Buffer<void>>("packed_" + std::to_string(i), enc.encoded[i].dimensions()) = enc.encoded[i];
+            }
+            // enc.handles (e.g. per-block reduction Funcs) still need scheduling here too.
+            return;
+        }
+
+        if (weight_precomputed_) {
+            weight_packed_ = add_input<Buffer<void>>("weight_packed", 2);
+        } else {
+            weight_fp32_ = add_input<Buffer<float>>("weight_fp32", 2);
+        }
+        activation_ = add_input<Buffer<float>>("activation", 1);
+        result_ = add_output<Buffer<float>>("result", 1);
+    }
+
+    void generate() {
+        Func weight_value = weight_precomputed_
+            ? weight_approx_->decode({Func(*weight_packed_)}).decoded[0]
+            : weight_approx_->decode(weight_approx_->encode(Func(*weight_fp32_)).encoded).decoded[0];
+        EncodeResult activation_enc = activation_approx_->encode(Func(*activation_));
+        // activation_enc.encoded / .handles get fused at whatever granularity
+        // schedule() picks -- see "Scope: placement is not semantics" above.
+        *result_ = /* the actual matmul reduction, calling weight_value / activation_enc.encoded inline */;
+    }
+
+    void schedule() {
+        // e.g. pick a named tiling strategy based on another GeneratorParam.
+    }
+
+private:
+    std::unique_ptr<Approximation> weight_approx_, activation_approx_;
+    Input<Buffer<float>> *x_ = nullptr, *weight_fp32_ = nullptr, *activation_ = nullptr;
+    Input<Buffer<void>> *weight_packed_ = nullptr;
+    Output<Buffer<float>> *result_ = nullptr;
+};
+```
+
+Known rough edge, accepted for now: `add_input`/`add_output` return raw
+pointers, needing member-pointer bookkeeping (`Input<Buffer<float>> *x_`) that
+the static `Input<>`/`Output<>` member style used elsewhere in this repo doesn't
+need. Revisiting this ergonomics gap is out of scope until the rest of this
+design is validated.
+
+## Summary of decisions and open items
+
+| Item                                                                                        | Status                                                                                                       |
+| ------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ |
+| `Approximation`: virtual-dispatch abstract class, operates on `Func`s only                  | Decided                                                                                                      |
+| `Approximation` makes no placement claims (offline vs fused)                                | Decided                                                                                                      |
+| `encode`/`decode` return `(funcs, handles)`, not bare `vector<Func>`                        | Decided; handles are scheduling-only intermediates, kept separate from the signature-contract output         |
+| `decode(encode(f).encoded).decoded[0]` signature contract                                   | Decided; enforced only at `approximate_by`'s substitution time in v1, not at `Approximation`-definition time |
+| `encode`'s output arity/layout (packed vs planar)                                           | Left to each `Approximation`; not decided by the framework                                                   |
+| `approximate_by`: eager, destructive `substitute_calls`, not `Func::in`                     | Decided; needed so it composes with `compute_offline` — see rejected-`Func::in` writeup above                |
+| `approximate_by` is a `Func` member (`f.approximate_by(p, consumers)`), not a free function | Decided; reads like the rest of the `in()`/`rfactor()`/`clone_in()` graph-editing family                     |
+| `approximate_by` requires explicit, already-existing `consumers`                            | Decided trade-off; no more "global" wrapper covering not-yet-written Funcs, same scoping as `rfactor`        |
+| `compute_offline`: seam exposure via static, `GeneratorParam`-driven `configure()`-time I/O | Decided for v1                                                                                               |
+| `compute_offline`: true automatic pipeline splitting                                        | Rejected for v1 (phase-ordering conflict with `configure()`/`generate()`)                                    |
+| `compute_offline`: cross-compile provenance checking                                        | Explicitly deferred to v2; v1 relies on "same registry name"                                                 |
+| `compute_offline` + `approximate_by` used together in one pipeline                          | Supported — both are eager/destructive, sequenced by ordinary program order, like `rfactor` + scheduling     |
+| Fusing `encode`/`decode` into neighboring stages (activation requantization)                | No new mechanism needed — ordinary `.compute_at()`/`.compute_inline()`                                       |
+| Generator I/O ergonomics (`add_input`/`add_output` pointer bookkeeping)                     | Accepted rough edge, deferred                                                                                |
+
+## Prior art referenced
+
+- `apps/ggml/halide/*_generators.cpp` — the manually-duplicated
+  quantize/dequantize/vec_dot implementations this design generalizes.
+- A private Python research prototype exploring the same compositional
+  `Approximation` idea, including its `(funcs, handles)` return convention — not
+  a public artifact, referenced here only for context.
+- `apps/hannk/halide/conv_generator.cpp` — sole in-repo precedent for a
+  non-trivial `configure()`.
+- `src/Func.cpp: Stage::rfactor` — the eager, destructive graph-editing
+  precedent `approximate_by` follows instead of `.in()`.
+- `src/Func.cpp`, `src/Function.cpp`, `src/WrapCalls.cpp` — origin of
+  `Function::substitute_calls`, the primitive `approximate_by` calls directly
+  and eagerly instead of through the deferred wrapper map.
+- `src/Generator.h` — the `configure()`/`generate()`/`schedule()` lifecycle
+  `compute_offline` and the Generator shape build on.
diff --git a/src/Approximation.cpp b/src/Approximation.cpp
new file mode 100644
index 000000000000..ba02ed6f1644
--- /dev/null
+++ b/src/Approximation.cpp
@@ -0,0 +1,114 @@
+#include "Approximation.h"
+
+#include "Error.h"
+
+namespace Halide {
+
+namespace {
+
+Func find_stage_output(const std::vector<ApproximationStageOutputs> &outputs,
+                       const ApproximationStageKey &stage, size_t port) {
+    if (!stage.defined()) {
+        return Func();
+    }
+    for (auto it = outputs.rbegin(); it != outputs.rend(); ++it) {
+        if (it->stage == stage) {
+            return port < it->ports.size() ? it->ports[port] : Func();
+        }
+    }
+    return Func();
+}
+
+}  // namespace
+
+Func ApproximationResult::encoded_by(const ApproximationStageKey &stage, size_t port) const {
+    return find_stage_output(encoded_stage_outputs, stage, port);
+}
+
+Func ApproximationResult::decoded_by(const ApproximationStageKey &stage, size_t port) const {
+    return find_stage_output(decoded_stage_outputs, stage, port);
+}
+
+EncodeResult Compose::encode(std::vector<Func> inputs) {
+    user_assert(!stages_.empty()) << "Compose::encode: no stages\n";
+
+    std::vector<Func> handles;
+    std::vector<ApproximationStageOutputs> stage_outputs;
+    std::vector<Func> current = std::move(inputs);
+    for (int i = (int)stages_.size() - 1; i >= 0; i--) {
+        EncodeResult r = stages_[i]->encode(std::move(current));
+        stage_outputs.insert(stage_outputs.end(), r.stage_outputs.begin(), r.stage_outputs.end());
+        stage_outputs.push_back({stages_[i]->stage_key(), r.encoded});
+        if (i > 0) {
+            // Not the final (outermost) stage -- its encoded output is an
+            // intermediate between stages, so it needs scheduling like any
+            // other handle, but isn't part of the signature contract this
+            // Compose itself returns.
+            handles.insert(handles.end(), r.encoded.begin(), r.encoded.end());
+        }
+        handles.insert(handles.end(), r.handles.begin(), r.handles.end());
+        current = std::move(r.encoded);
+    }
+    return {current, handles, stage_outputs};
+}
+
+DecodeResult Compose::decode(std::vector<Func> encoded) {
+    user_assert(!stages_.empty()) << "Compose::decode: no stages\n";
+
+    std::vector<Func> handles;
+    std::vector<ApproximationStageOutputs> stage_outputs;
+    std::vector<Func> current = std::move(encoded);
+    for (int i = 0; i < (int)stages_.size(); i++) {
+        DecodeResult r = stages_[i]->decode(std::move(current));
+        stage_outputs.insert(stage_outputs.end(), r.stage_outputs.begin(), r.stage_outputs.end());
+        stage_outputs.push_back({stages_[i]->stage_key(), r.decoded});
+        if (i + 1 < (int)stages_.size()) {
+            handles.insert(handles.end(), r.decoded.begin(), r.decoded.end());
+        }
+        handles.insert(handles.end(), r.handles.begin(), r.handles.end());
+        current = std::move(r.decoded);
+    }
+    return {current, handles, stage_outputs};
+}
+
+EncodeResult Apply::encode(std::vector<Func> inputs) {
+    user_assert(idx_ + encode_arity_ <= (int)inputs.size())
+        << "Apply::encode: idx (" << idx_ << ") + encode_arity (" << encode_arity_
+        << ") exceeds the input count (" << inputs.size() << ")\n";
+    std::vector<Func> target(inputs.begin() + idx_, inputs.begin() + idx_ + encode_arity_);
+    EncodeResult inner_result = inner_->encode(std::move(target));
+
+    std::vector<Func> encoded(inputs.begin(), inputs.begin() + idx_);
+    encoded.insert(encoded.end(), inner_result.encoded.begin(), inner_result.encoded.end());
+    encoded.insert(encoded.end(), inputs.begin() + idx_ + encode_arity_, inputs.end());
+    inner_result.stage_outputs.push_back({inner_->stage_key(), inner_result.encoded});
+    return {encoded, inner_result.handles, inner_result.stage_outputs};
+}
+
+DecodeResult Apply::decode(std::vector<Func> encoded) {
+    user_assert(idx_ + decode_arity_ <= (int)encoded.size())
+        << "Apply::decode: idx (" << idx_ << ") + decode_arity (" << decode_arity_
+        << ") exceeds the input count (" << encoded.size() << ")\n";
+    std::vector<Func> target(encoded.begin() + idx_, encoded.begin() + idx_ + decode_arity_);
+    DecodeResult inner_result = inner_->decode(std::move(target));
+
+    std::vector<Func> decoded(encoded.begin(), encoded.begin() + idx_);
+    decoded.insert(decoded.end(), inner_result.decoded.begin(), inner_result.decoded.end());
+    decoded.insert(decoded.end(), encoded.begin() + idx_ + decode_arity_, encoded.end());
+    inner_result.stage_outputs.push_back({inner_->stage_key(), inner_result.decoded});
+    return {decoded, inner_result.handles, inner_result.stage_outputs};
+}
+
+EncodeResult TrustedInverse::encode(std::vector<Func> inputs) {
+    EncodeResult r = encoder_->encode(std::move(inputs));
+    r.stage_outputs.push_back({encoder_->stage_key(), r.encoded});
+    return r;
+}
+
+DecodeResult TrustedInverse::decode(std::vector<Func> encoded) {
+    DecodeResult r = decoder_->decode(std::move(encoded));
+    r.stage_outputs.push_back({decoder_->stage_key(), r.decoded});
+    return r;
+}
+
+}  // namespace Halide
diff --git a/src/Approximation.h b/src/Approximation.h
new file mode 100644
index 000000000000..fc4d5e7dbb5c
--- /dev/null
+++ b/src/Approximation.h
@@ -0,0 +1,393 @@
+#ifndef HALIDE_APPROXIMATION_H
+#define HALIDE_APPROXIMATION_H
+
+/** \file
+ * Defines Approximation, a core interface for lossy, quantified
+ * Func-to-Func transformations (e.g. a quantize/dequantize round trip), and
+ * Compose/Apply, which build larger Approximations out of smaller ones. See
+ * Func::approximate_by(), which splices such a round trip into an existing
+ * call graph, and doc/ApproximationDesign.md for the design rationale.
+ */
+
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "Func.h"
+
+namespace Halide {
+
+/** An opaque identity for one Approximation stage. Keys are cheap to copy and
+ * remain stable when an Approximation is moved into Compose/Apply. A copied
+ * Approximation deliberately retains its key: the key identifies the logical
+ * stage selected by the caller, not a particular C++ address. */
+class ApproximationStageKey {
+public:
+    ApproximationStageKey() = default;
+
+    bool defined() const {
+        return token_ != nullptr;
+    }
+
+    friend bool operator==(const ApproximationStageKey &a, const ApproximationStageKey &b) {
+        return a.token_ == b.token_;
+    }
+    friend bool operator!=(const ApproximationStageKey &a, const ApproximationStageKey &b) {
+        return !(a == b);
+    }
+
+private:
+    explicit ApproximationStageKey(std::shared_ptr<const char> token)
+        : token_(std::move(token)) {
+    }
+
+    std::shared_ptr<const char> token_;
+    friend class Approximation;
+};
+
+/** The ports produced by one stage during encode or decode. This trace is
+ * supplemental scheduling metadata; it does not alter the signature contract
+ * or the legacy flat handle lists. */
+struct ApproximationStageOutputs {
+    ApproximationStageKey stage;
+    std::vector<Func> ports;
+};
+
+/** The result of Approximation::encode(): the Func(s) that make up the
+ * signature contract other code is expected to consume, plus any extra
+ * intermediate Funcs ("handles") that have no meaning outside scheduling
+ * (e.g. per-block reduction Funcs) but must still be scheduled by whoever
+ * calls encode(). */
+struct EncodeResult {
+    std::vector<Func> encoded;
+    std::vector<Func> handles;
+    std::vector<ApproximationStageOutputs> stage_outputs;
+};
+
+/** The result of Approximation::decode(): decoded is the round-trip
+ * replacement for whatever Func(s) were originally encoded, plus any
+ * additional scheduling-only handles. When an Approximation is used
+ * directly with Func::approximate_by(), decoded must contain exactly one
+ * Func; when it's used as one stage of a larger Compose/Apply chain,
+ * decoded may contain however many Funcs the next stage down expects. */
+struct DecodeResult {
+    std::vector<Func> decoded;
+    std::vector<Func> handles;
+    std::vector<ApproximationStageOutputs> stage_outputs;
+};
+
+/** Approximation is the base class for a lossy, quantified transformation
+ * of one or more Funcs' values -- e.g. quantize-then-dequantize. Unlike an
+ * ordinary schedule directive, an Approximation deliberately changes the
+ * *value* computed, not just how or where it's computed: decode(encode(f))
+ * is expected to approximately reproduce f, not exactly reproduce it.
+ *
+ * encode()/decode() take and return a *vector* of Funcs, not a single Func,
+ * even though the common case (a leaf Approximation like a plain quantizer)
+ * only ever uses one. This is what makes Compose and Apply below possible:
+ * a composed Approximation's inner stage can produce multiple Funcs (e.g. a
+ * quantized-values Func plus a separate scale Func), and the next stage
+ * needs to be able to consume all of them, or select just one to act on.
+ *
+ * An Approximation makes no claim about *where* or *when* encode/decode are
+ * computed relative to the rest of a pipeline (offline vs fused inline,
+ * compute_root vs compute_at) -- that is a scheduling decision, orthogonal
+ * to the semantics defined here. Concretely: the same Approximation can be
+ * used with encode() computed once, offline, ahead of any other stage (a
+ * static weight quantizer) or fused into a producer's inner loop and
+ * recomputed on every call (dynamic activation requantization) -- nothing
+ * about the interface favors one over the other. See Func::approximate_by()
+ * for splicing an Approximation into an existing call graph. */
+class Approximation {
+public:
+    Approximation()
+        : stage_key_(std::make_shared<const char>(0)) {
+    }
+    virtual ~Approximation() = default;
+
+    ApproximationStageKey stage_key() const {
+        return stage_key_;
+    }
+
+    /** Produce the encoded form of `inputs`. EncodeResult::encoded's
+     * elements are not required to have the same type, dimensionality, or
+     * count as `inputs` -- an Approximation is free to choose a packed
+     * representation (a single opaque byte buffer, fields recovered via
+     * reinterpret<>() inside decode) or a planar one (multiple typed Funcs,
+     * one per field). Either is legitimate; the framework does not
+     * decide. */
+    virtual EncodeResult encode(std::vector<Func> inputs) = 0;
+
+    /** Reconstruct an approximation of the original Func(s) from their
+     * encoded form. See DecodeResult for the constraint on `decoded`'s
+     * size, which depends on how this Approximation is used. */
+    virtual DecodeResult decode(std::vector<Func> encoded) = 0;
+
+private:
+    ApproximationStageKey stage_key_;
+};
+
+namespace Internal {
+
+/** Not for direct use. Type-erases an Approximation-derived value (or an
+ * already-type-erased std::unique_ptr<Approximation>, for the rare case
+ * where the concrete type is only known at runtime, e.g. chosen by an
+ * if/else) into an owned std::unique_ptr<Approximation> -- what lets
+ * Compose/Apply's constructors accept a plain mix of concrete Approximation
+ * values while still handling runtime-chosen ones, without exposing that
+ * distinction as something a caller has to think about. */
+// @{
+template<typename T>
+std::unique_ptr<Approximation> approximation_ptr(T &&value) {
+    return std::make_unique<std::decay_t<T>>(std::forward<T>(value));
+}
+inline std::unique_ptr<Approximation> approximation_ptr(std::unique_ptr<Approximation> value) {
+    return value;
+}
+// @}
+
+}  // namespace Internal
+
+/** The result of Func::approximate_by(): the primary replacement Func
+ * (already spliced into every Func in `consumers`), plus every
+ * intermediate Func produced by encode()/decode() along the way that needs
+ * scheduling (compute_root, compute_at, etc.) -- none of `handles` are
+ * part of the Approximation's signature contract, but Halide still
+ * requires Funcs with update definitions to be scheduled, and the fusion
+ * patterns described on Approximation above (e.g. compute_at-ing the
+ * encoded Func into a producer) are only possible if the caller has a
+ * handle to schedule. */
+struct ApproximationResult {
+    Func replacement;
+    /** The Func(s) produced by encode() -- the signature-contract boundary
+     * between the original values and their approximated form (e.g. a
+     * quantizer's packed byte buffer). This is a subset of `handles` (kept
+     * there too, so existing code that schedules everything in `handles`
+     * doesn't need to change), broken out separately so callers can act on
+     * exactly this boundary -- e.g. Pipeline::compute_offline(result.encoded)
+     * -- without calling Approximation::encode() themselves. */
+    std::vector<Func> encoded;
+    std::vector<Func> handles;
+    std::vector<ApproximationStageOutputs> encoded_stage_outputs;
+    std::vector<ApproximationStageOutputs> decoded_stage_outputs;
+
+    /** Return a stage output port, or an undefined Func if the key was not
+     * invoked in this direction or the port is out of range. */
+    Func encoded_by(const ApproximationStageKey &stage, size_t port = 0) const;
+    Func decoded_by(const ApproximationStageKey &stage, size_t port = 0) const;
+};
+
+/** Sequentially composes any number of Approximations into a pipeline:
+ * encode() runs `stages` back-to-front (the last stage first, on the
+ * original inputs), feeding each stage's encoded output to the one before
+ * it; decode() runs the mirror image, front-to-back. So `stages[0]` is the
+ * "outermost" stage -- the one whose encode() output is this Compose's own
+ * encoded result, and whose decode() input is this Compose's own encoded
+ * argument -- and `stages.back()` is "innermost", closest to the original
+ * values. This generalizes what used to be a fixed two-stage
+ * `Compose(outer, inner)`; that's just the two-element case.
+ *
+ * Compose owns every stage: each constructor argument is moved into (or, if
+ * already a std::unique_ptr<Approximation> -- e.g. because the concrete
+ * type was only known at runtime -- taken as) internal storage, so callers
+ * don't need to keep named locals alive alongside the Compose itself:
+ *
+ * \code
+ * Compose scheme{
+ *     StructPack{...},
+ *     Apply{1, 1, 1, Fp16Pack{}},
+ *     SymmetricAffineQuantize{block_size, qmax, rounding, anchor},
+ * };
+ * \endcode
+ */
+class Compose : public Approximation {
+public:
+    explicit Compose(std::vector<std::unique_ptr<Approximation>> stages)
+        : stages_(std::move(stages)) {
+    }
+
+    template<typename... Stages>
+    explicit Compose(Stages &&...stages) {
+        stages_.reserve(sizeof...(Stages));
+        (stages_.push_back(Internal::approximation_ptr(std::forward<Stages>(stages))), ...);
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override;
+    DecodeResult decode(std::vector<Func> encoded) override;
+
+private:
+    std::vector<std::unique_ptr<Approximation>> stages_;
+};
+
+class ComposeBuilder {
+public:
+    template<typename Stage>
+    ComposeBuilder &add(Stage &&stage) {
+        stages_.emplace_back(Internal::approximation_ptr(std::forward<Stage>(stage)));
+        return *this;
+    }
+
+    [[nodiscard]] std::unique_ptr<Approximation> build() {
+        return std::make_unique<Compose>(std::move(stages_));
+    }
+
+private:
+    std::vector<std::unique_ptr<Approximation>> stages_;
+};
+
+/** Applies `inner` to just the sub-range `[idx, idx + arity)` of a Func
+ * vector, passing every other element through unchanged -- e.g. applying a
+ * quantizer to just the "shifted" component of an affine (shift + scale)
+ * scheme's encoded output while leaving the shift amount itself untouched.
+ * `encode_arity`/`decode_arity` (how many Funcs `inner` consumes at that
+ * position for each direction) must be given explicitly, since C++ has no
+ * way to infer them generically from `inner` itself. Apply owns `inner` --
+ * moved in (or taken directly, if already a std::unique_ptr<Approximation>)
+ * -- the same way Compose owns its stages. */
+class Apply : public Approximation {
+public:
+    template<typename Inner>
+    Apply(int idx, int encode_arity, int decode_arity, Inner &&inner)
+        : idx_(idx), encode_arity_(encode_arity), decode_arity_(decode_arity),
+          inner_(Internal::approximation_ptr(std::forward<Inner>(inner))) {
+    }
+
+    template<typename Inner>
+    Apply(int idx, Inner &&inner)
+        : Apply(idx, 1, 1, std::forward<Inner>(inner)) {
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override;
+    DecodeResult decode(std::vector<Func> encoded) override;
+
+private:
+    int idx_, encode_arity_, decode_arity_;
+    std::unique_ptr<Approximation> inner_;
+};
+
+/** Routes encode() to one Approximation and decode() to another, taking each
+ * direction from a *different* source. This is the deliberate backdoor out of
+ * the structural guarantee Compose provides.
+ *
+ * Every Approximation is meant to be an approximate identity, factored into a
+ * decode-after-encode pair (decode(encode(f)) ~= f). Compose preserves that by
+ * construction: it interleaves its stages' encode()s and decode()s in mirror
+ * order, so the composed round trip (d1 . d2) . (e2 . e1) is *guaranteed* to be
+ * an approximate identity for the same structural reason each stage is -- the
+ * two halves provably come from one stage list. TrustedInverse pairs an encode
+ * and a decode from unrelated Approximations, so nothing structural guarantees
+ * they compose to an identity: the caller is *trusted* to have supplied a true
+ * inverse pair. Hence the name -- "trusted" as in "taken on trust", not "known
+ * safe".
+ *
+ * The motivating case: a scheme whose forward map (quantize) is an opaque
+ * offline black box -- a per-block codeword search, a transcendental scale fit,
+ * typically an extern call -- that no composition of Halide Funcs reproduces
+ * bit-for-bit, but whose reverse map (dequantize) *is* an ordinary Compose of
+ * invertible primitives. Compose can't express that pairing; TrustedInverse
+ * can, keeping the decode side a clean composition while the encode side is
+ * whatever opaque Approximation actually produces the encoded form:
+ *
+ * \code
+ * TrustedInverse{
+ *     ExternQuantize{"q4_k_quantize_via_ggml"},   // encode(): values -> bytes
+ *     Compose{                                      // decode(): bytes -> values
+ *         StructPack{...}, Apply{...}, ..., BlockReshape{block_size},
+ *     },
+ * };
+ * \endcode
+ *
+ * The unused half of each side is never called (here, the ExternQuantize's
+ * decode() and the Compose's encode()); supplying an Approximation whose
+ * relevant half is a stub is expected. TrustedInverse owns both sides the same
+ * way Compose/Apply own their stages -- moved in, or taken directly if already
+ * a std::unique_ptr<Approximation>. */
+class TrustedInverse : public Approximation {
+public:
+    template<typename Enc, typename Dec>
+    TrustedInverse(Enc &&encoder, Dec &&decoder)
+        : encoder_(Internal::approximation_ptr(std::forward<Enc>(encoder))),
+          decoder_(Internal::approximation_ptr(std::forward<Dec>(decoder))) {
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override;
+    DecodeResult decode(std::vector<Func> encoded) override;
+
+private:
+    std::unique_ptr<Approximation> encoder_, decoder_;
+};
+
+/** Picks one of two Approximations at construction time based on `cond` */
+class Choose : public Approximation {
+public:
+    template<typename True, typename False>
+    Choose(bool cond, True &&if_true, False &&if_false)
+        : chosen_(cond ? Internal::approximation_ptr(std::forward<True>(if_true)) :
+                         Internal::approximation_ptr(std::forward<False>(if_false))) {
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) {
+        EncodeResult r = chosen_->encode(std::move(inputs));
+        r.stage_outputs.push_back({chosen_->stage_key(), r.encoded});
+        return r;
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) {
+        DecodeResult r = chosen_->decode(std::move(encoded));
+        r.stage_outputs.push_back({chosen_->stage_key(), r.decoded});
+        return r;
+    }
+
+private:
+    std::unique_ptr<Approximation> chosen_;
+};
+
+class Identity : public Approximation {
+public:
+    EncodeResult encode(std::vector<Func> inputs) override {
+        return {inputs, {}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        return {encoded, {}};
+    }
+};
+
+class Permute : public Approximation {
+public:
+    explicit Permute(std::vector<int> permutation) : forward_(std::move(permutation)) {
+        backward_.resize(forward_.size());
+        for (int i = 0; i < (int)forward_.size(); i++) {
+            backward_[forward_[i]] = i;
+        }
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == forward_.size()) << "Permutation size does not match input size";
+        std::vector<Func> result;
+        result.reserve(inputs.size());
+        for (int i = 0; i < (int)inputs.size(); i++) {
+            result.push_back(inputs[forward_[i]]);
+        }
+        return {result, {}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == forward_.size()) << "Permutation size does not match encoded size";
+        std::vector<Func> result;
+        result.reserve(encoded.size());
+        for (int i = 0; i < (int)encoded.size(); i++) {
+            result.push_back(encoded[backward_[i]]);
+        }
+        return {result, {}};
+    }
+
+private:
+    std::vector<int> forward_, backward_;
+};
+
+}  // namespace Halide
+
+#endif
diff --git a/src/ApproximationComponents.h b/src/ApproximationComponents.h
new file mode 100644
index 000000000000..12d617ed302c
--- /dev/null
+++ b/src/ApproximationComponents.h
@@ -0,0 +1,564 @@
+#ifndef HALIDE_APPROXIMATION_COMPONENTS_H
+#define HALIDE_APPROXIMATION_COMPONENTS_H
+
+/** \file
+ * Reusable Approximation building blocks for typed records, scalar storage,
+ * fixed-width bit fields, and block quantization.
+ */
+
+#include <cstdint>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "Approximation.h"
+#include "Buffer.h"
+#include "IROperator.h"
+#include "RDom.h"
+
+namespace Halide {
+
+namespace Internal {
+
+inline std::vector<Var> approximation_component_vars(int dimensions, const std::string &prefix) {
+    std::vector<Var> vars;
+    vars.reserve(dimensions);
+    for (int i = 0; i < dimensions; ++i) {
+        vars.emplace_back(prefix + std::to_string(i));
+    }
+    return vars;
+}
+
+inline std::vector<Expr> approximation_component_exprs(const std::vector<Var> &vars) {
+    return std::vector<Expr>(vars.begin(), vars.end());
+}
+
+}  // namespace Internal
+
+/** Losslessly reshape a flat row into fixed-size records. In block-indexed
+ * mode the flat side is `(within, record)` rather than a single flat index. */
+class BlockReshape : public Approximation {
+public:
+    explicit BlockReshape(int block_size, bool block_indexed = false)
+        : extents_{block_size}, block_indexed_(block_indexed) {
+    }
+    explicit BlockReshape(std::vector<int> extents, bool block_indexed = false)
+        : extents_(std::move(extents)), block_indexed_(block_indexed) {
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == 1) << "BlockReshape::encode expects one input\n";
+        Func flat = inputs[0];
+        std::vector<Var> dims = block_vars();
+        Var blk("blk");
+        Expr within = cast<int>(0);
+        int stride = 1;
+        for (size_t i = 0; i < dims.size(); ++i) {
+            within += dims[i] * stride;
+            stride *= extents_[i];
+        }
+        std::vector<Var> args = dims;
+        args.push_back(blk);
+        Func packed("block_reshape_packed");
+        packed(args) = block_indexed_ ? flat(within, blk) : flat(blk * block_size() + within);
+        return {{packed}, {}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 1) << "BlockReshape::decode expects one input\n";
+        Func packed = encoded[0];
+        Var k("k"), kk("kk"), blk("blk");
+        Expr within = block_indexed_ ? Expr(kk) : k % block_size();
+        Expr block = block_indexed_ ? Expr(blk) : k / block_size();
+        std::vector<Expr> args;
+        Expr rem = within;
+        for (int extent : extents_) {
+            args.push_back(rem % extent);
+            rem /= extent;
+        }
+        args.push_back(block);
+        Func out("block_reshape_unpacked");
+        if (block_indexed_) {
+            out(kk, blk) = packed(args);
+        } else {
+            out(k) = packed(args);
+        }
+        return {{out}, {}};
+    }
+
+private:
+    std::vector<int> extents_;
+    bool block_indexed_;
+
+    int block_size() const {
+        int size = 1;
+        for (int extent : extents_) {
+            size *= extent;
+        }
+        return size;
+    }
+
+    std::vector<Var> block_vars() const {
+        std::vector<Var> vars;
+        for (size_t i = 0; i < extents_.size(); ++i) {
+            vars.emplace_back(extents_.size() == 1 ? "kk" : "d" + std::to_string(i));
+        }
+        return vars;
+    }
+};
+
+/** Map consecutive logical Func slots to named fields of an exact struct
+ * type. Scalar fields have `record_dimensions` dimensions; array fields have
+ * an additional leading element dimension. */
+class StructLayout : public Approximation {
+public:
+    StructLayout(Type record_type, std::vector<std::string> logical_fields,
+                 int record_dimensions = 1)
+        : record_type_(record_type), logical_fields_(std::move(logical_fields)),
+          record_dimensions_(record_dimensions) {
+        user_assert(record_type_.is_struct()) << "StructLayout requires a struct Type\n";
+        user_assert(record_dimensions_ > 0) << "StructLayout record dimensionality must be positive\n";
+        const StructTypeInfo *info = record_type_.struct_type();
+        user_assert(logical_fields_.size() == info->fields.size())
+            << "StructLayout requires exactly one logical slot per physical field\n";
+        for (const std::string &name : logical_fields_) {
+            int matches = 0;
+            for (const StructField &field : info->fields) {
+                matches += field.name == name;
+            }
+            user_assert(matches == 1) << "StructLayout: no unique field named '" << name << "'\n";
+            int logical_matches = 0;
+            for (const std::string &logical_name : logical_fields_) {
+                logical_matches += logical_name == name;
+            }
+            user_assert(logical_matches == 1) << "StructLayout: duplicate logical field '" << name << "'\n";
+        }
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == logical_fields_.size())
+            << "StructLayout::encode input count does not match logical field count\n";
+        const StructTypeInfo *info = record_type_.struct_type();
+        std::vector<Var> records = Internal::approximation_component_vars(record_dimensions_, "record");
+        std::vector<Expr> record_args = Internal::approximation_component_exprs(records);
+        std::vector<Expr> values;
+        for (const StructField &field : info->fields) {
+            size_t slot = logical_slot(field.name);
+            Func input = inputs[slot];
+            user_assert(input.outputs() == 1 && input.types()[0] == field.type)
+                << "StructLayout field '" << field.name << "' requires exact type " << field.type
+                << " but slot " << slot << " has " << input.types()[0] << "\n";
+            int extent = field.array_extent.value_or(1);
+            user_assert(input.dimensions() == record_dimensions_ + (field.array_extent ? 1 : 0))
+                << "StructLayout field '" << field.name << "' has the wrong dimensionality\n";
+            for (int element = 0; element < extent; ++element) {
+                std::vector<Expr> args = record_args;
+                if (field.array_extent) {
+                    args.insert(args.begin(), element);
+                }
+                values.push_back(input(args));
+            }
+        }
+        Func packed("struct_layout_packed");
+        packed(records) = pack_struct(record_type_, values);
+        return {{packed}, {}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 1 && encoded[0].outputs() == 1 &&
+                    encoded[0].types()[0] == record_type_)
+            << "StructLayout::decode requires one Func of the exact record type\n";
+        Func packed = encoded[0];
+        std::vector<Var> records = Internal::approximation_component_vars(record_dimensions_, "record");
+        std::vector<Expr> record_args = Internal::approximation_component_exprs(records);
+        Expr record = packed(record_args);
+        std::vector<Func> outputs;
+        outputs.reserve(logical_fields_.size());
+        for (const std::string &name : logical_fields_) {
+            const StructField &physical = physical_field(name);
+            Func output("struct_layout_" + name);
+            if (physical.array_extent) {
+                Var element("element");
+                std::vector<Var> args = records;
+                args.insert(args.begin(), element);
+                output(args) = field(record, name)[element];
+            } else {
+                output(records) = field(record, name);
+            }
+            outputs.push_back(output);
+        }
+        return {outputs, {}};
+    }
+
+private:
+    Type record_type_;
+    std::vector<std::string> logical_fields_;
+    int record_dimensions_;
+
+    size_t logical_slot(const std::string &name) const {
+        for (size_t i = 0; i < logical_fields_.size(); ++i) {
+            if (logical_fields_[i] == name) {
+                return i;
+            }
+        }
+        user_error << "StructLayout internal field mapping failure\n";
+        return 0;
+    }
+
+    const StructField &physical_field(const std::string &name) const {
+        for (const StructField &field : record_type_.struct_type()->fields) {
+            if (field.name == name) {
+                return field;
+            }
+        }
+        user_error << "StructLayout internal physical field failure\n";
+        return record_type_.struct_type()->fields[0];
+    }
+};
+
+/** Explicit numeric conversion between a decoded computation type and the
+ * exact type stored in a representation. */
+template<typename Decoded, typename Storage>
+class StorageCast : public Approximation {
+public:
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == 1 && inputs[0].types() == std::vector<Type>{type_of<Decoded>()})
+            << "StorageCast::encode input type mismatch\n";
+        Func input = inputs[0];
+        std::vector<Var> args = Internal::approximation_component_vars(input.dimensions(), "cast");
+        Func stored("storage_cast_stored");
+        stored(args) = strict_float(cast<Storage>(input(Internal::approximation_component_exprs(args))));
+        return {{stored}, {}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 1 && encoded[0].types() == std::vector<Type>{type_of<Storage>()})
+            << "StorageCast::decode storage type mismatch\n";
+        Func input = encoded[0];
+        std::vector<Var> args = Internal::approximation_component_vars(input.dimensions(), "cast");
+        Func decoded("storage_cast_decoded");
+        decoded(args) = strict_float(cast<Decoded>(input(Internal::approximation_component_exprs(args))));
+        return {{decoded}, {}};
+    }
+};
+
+/** Losslessly translate an integral decoded alphabet by a fixed offset into
+ * its stored integral alphabet. This is representation policy, not bit
+ * packing: e.g. signed q4 codes [-8, 7] become stored nibbles [0, 15] before
+ * PlanarFieldPack handles their physical layout. */
+template<typename Decoded, typename Storage>
+class AdditiveOffset : public Approximation {
+    static_assert(std::is_integral_v<Decoded> && std::is_integral_v<Storage>,
+                  "AdditiveOffset requires integral types");
+
+public:
+    explicit AdditiveOffset(int64_t offset)
+        : offset_(offset) {
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == 1 && inputs[0].types() == std::vector<Type>{type_of<Decoded>()})
+            << "AdditiveOffset::encode input type mismatch\n";
+        Func input = inputs[0];
+        std::vector<Var> args = Internal::approximation_component_vars(input.dimensions(), "offset");
+        std::vector<Expr> call_args = Internal::approximation_component_exprs(args);
+        Func stored("additive_offset_stored");
+        Expr offset = Internal::make_const(Int(64), offset_);
+        stored(args) = cast<Storage>(cast<int64_t>(input(call_args)) + offset);
+        return {{stored}, {}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 1 && encoded[0].types() == std::vector<Type>{type_of<Storage>()})
+            << "AdditiveOffset::decode storage type mismatch\n";
+        Func input = encoded[0];
+        std::vector<Var> args = Internal::approximation_component_vars(input.dimensions(), "offset");
+        std::vector<Expr> call_args = Internal::approximation_component_exprs(args);
+        Func decoded("additive_offset_decoded");
+        Expr offset = Internal::make_const(Int(64), offset_);
+        decoded(args) = cast<Decoded>(cast<int64_t>(input(call_args)) - offset);
+        return {{decoded}, {}};
+    }
+
+private:
+    int64_t offset_;
+};
+
+/** Convert a scalar integral word per record to/from a leading little-endian
+ * byte dimension. Decode deliberately uses concat_bits so struct lowering and
+ * ordinary byte buffers share the same wide-load optimization path. */
+template<typename Word>
+class LittleEndianScalarPack : public Approximation {
+public:
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == 1 && inputs[0].types() == std::vector<Type>{type_of<Word>()})
+            << "LittleEndianScalarPack::encode word type mismatch\n";
+        Func word = inputs[0];
+        std::vector<Var> records = Internal::approximation_component_vars(word.dimensions(), "record");
+        std::vector<Expr> record_args = Internal::approximation_component_exprs(records);
+        Var byte("byte");
+        std::vector<Var> args = records;
+        args.insert(args.begin(), byte);
+        Func bytes("little_endian_scalar_bytes");
+        Expr bits = cast(type_of<Word>(), word(record_args));
+        bytes(args) = cast<uint8_t>(bits >> (byte * 8));
+        return {{bytes}, {}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 1 && encoded[0].types() == std::vector<Type>{UInt(8)} &&
+                    encoded[0].dimensions() >= 2)
+            << "LittleEndianScalarPack::decode requires byte arrays per record\n";
+        Func bytes = encoded[0];
+        std::vector<Var> records = Internal::approximation_component_vars(bytes.dimensions() - 1, "record");
+        std::vector<Expr> record_args = Internal::approximation_component_exprs(records);
+        std::vector<Expr> pieces;
+        for (size_t i = 0; i < sizeof(Word); ++i) {
+            std::vector<Expr> args = record_args;
+            args.insert(args.begin(), (int)i);
+            pieces.push_back(bytes(args));
+        }
+        Func word("little_endian_scalar_word");
+        word(records) = cast<Word>(concat_bits(pieces));
+        return {{word}, {}};
+    }
+};
+
+/** Pack a fixed vector containing exactly two values into an integer word.
+ * Decode uses an embedded 8x256 byte-expansion LUT, allowing each source byte
+ * to expand through one contiguous eight-byte load. */
+template<typename Value>
+class BinaryAlphabetPack : public Approximation {
+public:
+    BinaryAlphabetPack(int vector_size, Type word_type, Value zero_value, Value one_value)
+        : vector_size_(vector_size), word_type_(word_type), zero_value_(zero_value), one_value_(one_value),
+          expansion_(8, 256) {
+        user_assert(word_type_.is_uint() && word_type_.bits() >= vector_size_)
+            << "BinaryAlphabetPack word is too small for its vector\n";
+        user_assert(vector_size_ > 0 && vector_size_ % 8 == 0)
+            << "BinaryAlphabetPack vector size must be a positive multiple of eight\n";
+        for (int byte = 0; byte < 256; ++byte) {
+            for (int bit = 0; bit < 8; ++bit) {
+                expansion_(bit, byte) = (byte & (1 << bit)) ? one_value_ : zero_value_;
+            }
+        }
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == 1 && inputs[0].dimensions() >= 2)
+            << "BinaryAlphabetPack::encode requires (element, record...)\n";
+        Func values = inputs[0];
+        int records_n = values.dimensions() - 1;
+        std::vector<Var> records = Internal::approximation_component_vars(records_n, "record");
+        std::vector<Expr> record_args = Internal::approximation_component_exprs(records);
+        RDom bit(0, vector_size_, "binary_bit");
+        std::vector<Expr> value_args = record_args;
+        value_args.insert(value_args.begin(), bit);
+        Expr value = values(value_args);
+        Func word("binary_alphabet_word");
+        word(records) = cast(word_type_, 0);
+        word(records) = word(record_args) |
+                        select(value == cast<Value>(one_value_),
+                               cast(word_type_, 1) << bit, cast(word_type_, 0));
+        return {{word}, {word}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 1 && encoded[0].types() == std::vector<Type>{word_type_})
+            << "BinaryAlphabetPack::decode word type mismatch\n";
+        Func word = encoded[0];
+        std::vector<Var> records = Internal::approximation_component_vars(word.dimensions(), "record");
+        std::vector<Expr> record_args = Internal::approximation_component_exprs(records);
+        Var element("element");
+        Expr bits = word(record_args);
+        Expr byte = cast<int32_t>((bits >> ((element / 8) * 8)) & 0xff);
+        std::vector<Var> args = records;
+        args.insert(args.begin(), element);
+        Func values("binary_alphabet_values");
+        values(args) = expansion_(element % 8, byte);
+        return {{values}, {}};
+    }
+
+private:
+    int vector_size_;
+    Type word_type_;
+    Value zero_value_, one_value_;
+    Buffer<Value> expansion_;
+};
+
+/** Split a signed code into an unsigned low digit and an additive weighted
+ * high contribution. The second parameter recenters the signed code before
+ * taking the low digit; decode is simply `code = low + high`. */
+class AdditiveRadixSplit : public Approximation {
+public:
+    AdditiveRadixSplit(int radix, int offset)
+        : radix_(radix), offset_(offset) {
+        user_assert(radix_ > 1 && offset_ >= 0) << "Invalid AdditiveRadixSplit parameters\n";
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == 1) << "AdditiveRadixSplit::encode expects one code Func\n";
+        Func code = inputs[0];
+        std::vector<Var> args = Internal::approximation_component_vars(code.dimensions(), "code");
+        std::vector<Expr> call_args = Internal::approximation_component_exprs(args);
+        Expr value = cast<int32_t>(code(call_args));
+        Expr low_value = (value + offset_) % radix_;
+        Func low("additive_radix_low"), high("additive_radix_high");
+        low(args) = cast<uint8_t>(low_value);
+        high(args) = cast<int8_t>(value - low_value);
+        return {{low, high}, {}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 2 && encoded[0].dimensions() == encoded[1].dimensions())
+            << "AdditiveRadixSplit::decode expects low and high contributions\n";
+        std::vector<Var> args = Internal::approximation_component_vars(encoded[0].dimensions(), "code");
+        std::vector<Expr> call_args = Internal::approximation_component_exprs(args);
+        Func code("additive_radix_code");
+        code(args) = cast<int8_t>(cast<int32_t>(encoded[0](call_args)) +
+                                  cast<int32_t>(encoded[1](call_args)));
+        return {{code}, {}};
+    }
+
+private:
+    int radix_, offset_;
+};
+
+/** Exact fixed-width planar packing. For `(field_bits, positions)`, one byte
+ * contains `8/field_bits` planes, each plane spanning `positions` consecutive
+ * elements. This component applies no recentering and no lookup policy. */
+class PlanarFieldPack : public Approximation {
+public:
+    PlanarFieldPack(int field_bits, int positions)
+        : field_bits_(field_bits), positions_(positions), planes_(8 / field_bits) {
+        user_assert(field_bits_ > 0 && 8 % field_bits_ == 0 && positions_ > 0)
+            << "Invalid PlanarFieldPack shape\n";
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == 1 && inputs[0].dimensions() == 2)
+            << "PlanarFieldPack::encode currently requires (element, record)\n";
+        Func fields = inputs[0];
+        Var position("position"), record("record");
+        RDom plane(0, planes_, "plane");
+        Expr element = plane * positions_ + position;
+        Expr value = cast<uint8_t>(fields(element, record)) & ((1 << field_bits_) - 1);
+        Func bytes("planar_field_bytes");
+        bytes(position, record) = cast<uint8_t>(0);
+        bytes(position, record) = bytes(position, record) |
+                                  cast<uint8_t>(value << (plane * field_bits_));
+        return {{bytes}, {bytes}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 1 && encoded[0].types() == std::vector<Type>{UInt(8)} &&
+                    encoded[0].dimensions() == 2)
+            << "PlanarFieldPack::decode currently requires (position, record) bytes\n";
+        Func bytes = encoded[0];
+        Var element("element"), record("record");
+        Expr plane = element / positions_;
+        Expr position = element % positions_;
+        Func fields("planar_field_values");
+        fields(element, record) = cast<uint8_t>((bytes(position, record) >> (plane * field_bits_)) &
+                                                ((1 << field_bits_) - 1));
+        return {{fields}, {}};
+    }
+
+private:
+    int field_bits_, positions_, planes_;
+};
+
+enum class BlockRoundingMode {
+    Nearest,
+    TruncateHalfUpWithOffset,
+    SignOnly,
+    NearestEvenClampedHigh
+};
+
+enum class BlockScaleAnchor {
+    AbsMax,
+    ExtremeSignedValue,
+    MeanAbs,
+    ExtremeSignedValueTwoStep
+};
+
+inline Expr approximation_nearest_int(Expr value) {
+    Expr rounded = value + 12582912.0f;
+    Expr bits = reinterpret<int32_t>(rounded);
+    return (bits & 0x007fffff) - 0x00400000;
+}
+
+/** Symmetric per-block int8 quantization with explicit rounding and scale
+ * selection policies. */
+class SymmetricBlockQuantize : public Approximation {
+public:
+    SymmetricBlockQuantize(int block_size, int qmax, BlockRoundingMode rounding, BlockScaleAnchor anchor)
+        : block_size_(block_size), qmax_(qmax), rounding_(rounding), anchor_(anchor) {
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        user_assert(inputs.size() == 1) << "SymmetricBlockQuantize::encode expects one block Func\n";
+        Func block = inputs[0];
+        Var kk("kk"), blk("blk");
+        RDom r(0, block_size_, "r");
+        Func stat("symmetric_quantize_stat"), scale("symmetric_quantize_scale"), reciprocal("symmetric_quantize_reciprocal");
+        auto define_extreme = [&]() {
+            stat(blk) = Tuple(0.0f, 0.0f);
+            Expr value = block(r, blk);
+            Expr take = abs(value) > stat(blk)[0];
+            stat(blk) = Tuple(select(take, abs(value), stat(blk)[0]),
+                              select(take, value, stat(blk)[1]));
+        };
+        if (anchor_ == BlockScaleAnchor::AbsMax) {
+            stat(blk) = 0.0f;
+            stat(blk) = max(stat(blk), abs(block(r, blk)));
+            scale(blk) = stat(blk) / (float)qmax_;
+            reciprocal(blk) = select(scale(blk) != 0.0f, 1.0f / scale(blk), 0.0f);
+        } else if (anchor_ == BlockScaleAnchor::ExtremeSignedValue) {
+            define_extreme();
+            scale(blk) = stat(blk)[1] * (-1.0f / (float)qmax_);
+            reciprocal(blk) = select(scale(blk) != 0.0f, 1.0f / scale(blk), 0.0f);
+        } else if (anchor_ == BlockScaleAnchor::MeanAbs) {
+            stat(blk) = 0.0f;
+            stat(blk) += abs(block(r, blk));
+            scale(blk) = stat(blk) / (float)block_size_;
+            reciprocal(blk) = select(scale(blk) != 0.0f, 1.0f / scale(blk), 0.0f);
+        } else {
+            define_extreme();
+            reciprocal(blk) = select(stat(blk)[0] == 0.0f, 0.0f,
+                                     (-1.0f * (float)qmax_) / stat(blk)[1]);
+            scale(blk) = select(reciprocal(blk) != 0.0f, 1.0f / reciprocal(blk), 0.0f);
+        }
+        Expr scaled = block(kk, blk) * reciprocal(blk);
+        Func codes("symmetric_quantize_codes");
+        if (rounding_ == BlockRoundingMode::Nearest) {
+            codes(kk, blk) = cast<int8_t>(round(scaled));
+        } else if (rounding_ == BlockRoundingMode::TruncateHalfUpWithOffset) {
+            Expr raw = cast<int32_t>(cast<int8_t>(scaled + (float)qmax_ + 0.5f));
+            codes(kk, blk) = cast<int8_t>(min(raw, 2 * qmax_ - 1) - qmax_);
+        } else if (rounding_ == BlockRoundingMode::SignOnly) {
+            codes(kk, blk) = cast<int8_t>(select(block(kk, blk) >= 0.0f, 1, -1));
+        } else {
+            codes(kk, blk) = cast<int8_t>(min(qmax_, approximation_nearest_int(scaled)));
+        }
+        return {{codes, scale}, {stat}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        user_assert(encoded.size() == 2) << "SymmetricBlockQuantize::decode expects codes and scale\n";
+        Var kk("kk"), blk("blk");
+        Func dequantized("symmetric_dequantized");
+        dequantized(kk, blk) = cast<float>(encoded[0](kk, blk)) * encoded[1](blk);
+        return {{dequantized}, {}};
+    }
+
+private:
+    int block_size_, qmax_;
+    BlockRoundingMode rounding_;
+    BlockScaleAnchor anchor_;
+};
+
+}  // namespace Halide
+
+#endif
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 511917c9f6b7..f139c32eba10 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -61,6 +61,8 @@ target_sources(
     AlignLoads.h
     AllocationBoundsInference.h
     ApplySplit.h
+    Approximation.h
+    ApproximationComponents.h
     Argument.h
     AssociativeOpsTable.h
     Associativity.h
@@ -246,6 +248,7 @@ target_sources(
     AlignLoads.cpp
     AllocationBoundsInference.cpp
     ApplySplit.cpp
+    Approximation.cpp
     Argument.cpp
     AssociativeOpsTable.cpp
     Associativity.cpp
diff --git a/src/Func.cpp b/src/Func.cpp
index c99ca7ea6f8d..c403cb8488fc 100644
--- a/src/Func.cpp
+++ b/src/Func.cpp
@@ -13,6 +13,7 @@
 #endif
 
 #include "ApplySplit.h"
+#include "Approximation.h"
 #include "Argument.h"
 #include "Associativity.h"
 #include "Bounds.h"
@@ -1266,6 +1267,94 @@ Func Stage::rfactor(const vector<pair<RVar, Var>> &preserved) {
     return intm;
 }
 
+// How many nodes distribute_products() may rewrite before giving up. Multiplying
+// out is exponential in the nesting depth of sums, and a reduction body deep
+// enough to hit this has no business being split into that many accumulators.
+constexpr int distribute_budget = 1024;
+
+// Multiply out products over sums, so that a sum reduction's increment becomes
+// a flat sum of products: (a + b) * c -> a * c + b * c. This exposes terms with
+// *different* invariant factors, which a single flattening of the multiply chain
+// cannot see -- in (a + b) * c the sum is one opaque leaf.
+//
+// Subtraction is deliberately left alone (treated as a leaf): distributing it
+// would have to push the sign into one side, which does not group and is not
+// meaningful for unsigned wraparound.
+Expr distribute_products(const Expr &e, int &budget) {
+    if (--budget < 0) {
+        return e;
+    }
+    if (e.node_type() == IRNodeType::Add) {
+        auto [a, b] = *as_binary_operands(e);
+        return Add::make(distribute_products(a, budget), distribute_products(b, budget));
+    }
+    if (const Mul *mul = e.as<Mul>()) {
+        Expr a = distribute_products(mul->a, budget);
+        Expr b = distribute_products(mul->b, budget);
+        // Recurse on the rewritten form: either side may itself be a product of
+        // sums that only became visible after this step.
+        if (a.node_type() == IRNodeType::Add) {
+            auto [a0, a1] = *as_binary_operands(a);
+            return distribute_products(Add::make(Mul::make(a0, b), Mul::make(a1, b)), budget);
+        }
+        if (b.node_type() == IRNodeType::Add) {
+            auto [b0, b1] = *as_binary_operands(b);
+            return distribute_products(Add::make(Mul::make(a, b0), Mul::make(a, b1)), budget);
+        }
+        return Mul::make(a, b);
+    }
+    return e;
+}
+
+Stage &Stage::distribute() {
+    user_assert(!definition.is_init()) << "distribute() must be called on an update definition\n";
+
+    definition.schedule().touched() = true;
+
+    const auto &prover_result = prove_associativity(function.name(), definition.args(), definition.values());
+    user_assert(prover_result.associative())
+        << "distribute() requires an associative update definition, but the update "
+        << "definition of " << function.name() << " is not associative.\n";
+
+    auto is_self_ref = [&](const Expr &e) {
+        const Call *c = e.as<Call>();
+        return c && c->name == function.name() && c->call_type == Call::Halide;
+    };
+
+    vector<Expr> values = definition.values();
+    bool changed = false;
+    for (size_t i = 0; i < values.size(); ++i) {
+        optional<DistributiveLaw> law = distributive_law_for(prover_result.pattern.ops[i]);
+        if (!law || law->outer_op != IRNodeType::Add || law->inner_op != IRNodeType::Mul) {
+            continue;
+        }
+        // Lets may hide the combiner (e.g. an rfactor of this same update
+        // introduced promise_clamped bindings), so inline them first.
+        Expr value = substitute_in_all_lets(values[i]);
+        optional<pair<Expr, Expr>> split = select_binary_operand(value, law->outer_op, is_self_ref);
+        if (!split) {
+            continue;
+        }
+        int budget = distribute_budget;
+        Expr expanded = distribute_products(split->second, budget);
+        user_assert(budget >= 0)
+            << "distribute() gave up multiplying out the update definition of "
+            << function.name() << ": it expands to more than " << distribute_budget
+            << " nodes.\n";
+        if (!equal(expanded, split->second)) {
+            values[i] = Add::make(split->first, expanded);
+            changed = true;
+        }
+    }
+
+    user_assert(changed)
+        << "distribute() found no product over a sum to multiply out in the update "
+        << "definition of " << function.name() << ".\n";
+
+    definition.values() = values;
+    return *this;
+}
+
 FuncVec Stage::hoist_invariants() {
     user_assert(!definition.is_init()) << "hoist_invariants() must be called on an update definition\n";
 
@@ -2764,6 +2853,43 @@ Func Func::clone_in(const vector<Func> &fs) {
     return get_wrapper(func, name() + "_clone", fs, true);
 }
 
+ApproximationResult Func::approximate_by(Approximation &p, const vector<Func> &consumers) {
+    EncodeResult enc = p.encode({*this});
+    user_assert(!enc.encoded.empty())
+        << "approximate_by: Approximation::encode(" << name() << ") returned no Funcs\n";
+
+    DecodeResult dec = p.decode(enc.encoded);
+    user_assert(dec.decoded.size() == 1)
+        << "approximate_by: Approximation::decode() must return exactly one Func (the "
+        << "round-trip replacement), but returned " << dec.decoded.size() << "\n";
+
+    Func round_trip = dec.decoded[0];
+    user_assert(round_trip.dimensions() == dimensions())
+        << "approximate_by: decode(encode(" << name() << "))'s result (" << round_trip.name()
+        << ") has " << round_trip.dimensions() << " dimensions, but " << name() << " has "
+        << dimensions() << " -- Approximation implementations must reproduce the original "
+        << "Func's signature exactly\n";
+    user_assert(round_trip.types() == types())
+        << "approximate_by: decode(encode(" << name() << "))'s result (" << round_trip.name()
+        << ") has a different type than " << name() << " -- Approximation implementations "
+        << "must reproduce the original Func's signature exactly\n";
+
+    for (const Func &g : consumers) {
+        user_assert(g.name() != name())
+            << "approximate_by: " << name() << " cannot be its own consumer\n";
+        // Eager and destructive, like Func::rfactor() -- not deferred to
+        // lowering the way Func::in() is. See Approximation.h for why.
+        g.function().substitute_calls(func, round_trip.function());
+    }
+
+    vector<Func> handles = enc.encoded;
+    handles.insert(handles.end(), enc.handles.begin(), enc.handles.end());
+    handles.insert(handles.end(), dec.handles.begin(), dec.handles.end());
+    enc.stage_outputs.push_back({p.stage_key(), enc.encoded});
+    dec.stage_outputs.push_back({p.stage_key(), dec.decoded});
+    return {round_trip, enc.encoded, handles, enc.stage_outputs, dec.stage_outputs};
+}
+
 Func Func::copy_to_device(DeviceAPI d) {
     user_assert(defined())
         << "copy_to_device on Func " << name() << " with no definition\n";
diff --git a/src/Func.h b/src/Func.h
index f33d3f2d4af4..56d746d3f6cd 100644
--- a/src/Func.h
+++ b/src/Func.h
@@ -60,6 +60,8 @@ struct VarOrRVar {
 
 class ImageParam;
 class FuncVec;
+class Approximation;
+struct ApproximationResult;
 
 namespace Internal {
 struct AssociativeOp;
@@ -258,6 +260,32 @@ class Stage {
      */
     FuncVec hoist_invariants();
 
+    /** Multiply out products over sums in this update definition's increment,
+     * so that hoist_invariants() sees a flat sum of terms. Like rfactor(), this
+     * must be called on an update definition, and it rewrites that definition in
+     * place. Returns this Stage, so it can be chained.
+     *
+     * (a + b) * c becomes a * c + b * c, recursively. Subtraction is left alone.
+     *
+     * This is a schedule decision, not a normalization: whether it pays depends
+     * on what the terms turn out to contain. It pays when the sum hides operands
+     * with different loop-invariant factors, since each then reduces to a
+     * factor-free body of its own:
+     * \code
+     * f() += (d*q(r) + m) * (e*p(r));
+     * \endcode
+     * multiplies out to d*e * q(r)*p(r) + m*e * p(r), which hoist_invariants()
+     * turns into two factor-free accumulators. Left alone, the best it could do
+     * is hoist e and reduce over the sum.
+     *
+     * It costs an accumulator per distinct factor, so it is a pessimization when
+     * the terms share a factor that was already hoistable -- s * (r + 1) is
+     * better left as one accumulator with factor s than split into two.
+     *
+     * It is an error if there is no product over a sum to multiply out.
+     */
+    Stage &distribute();
+
     /** Schedule the iteration over this stage to be fused with another
      * stage 's' from outermost loop to a given LoopLevel. 'this' stage will
      * be computed AFTER 's' in the innermost fused dimension. There should not
@@ -1531,6 +1559,18 @@ class Func {
     Func clone_in(const std::vector<Func> &fs);
     //@}
 
+    /** Eagerly and destructively replace every call to this Func inside
+     * each Func in 'consumers' with a call to the round trip
+     * p.decode(p.encode(*this)) instead. The substitution happens
+     * immediately, the same way Func::rfactor() immediately rewrites a
+     * Func's definition, rather than being deferred to lowering the way
+     * Func::in() is -- see Approximation.h and doc/ApproximationDesign.md
+     * for the rationale. Because the substitution is eager, it can only
+     * rewrite Funcs that are already fully defined at the point of the
+     * call -- there is no "global" mode that also covers Funcs written
+     * later, unlike Func::in(). */
+    ApproximationResult approximate_by(Approximation &p, const std::vector<Func> &consumers);
+
     /** Declare that this function should be implemented by a call to
      * halide_buffer_copy with the given target device API. Asserts
      * that the Func has a pure definition which is a simple call to a
diff --git a/src/Generator.cpp b/src/Generator.cpp
index 24e0226cb1ec..0ec7988c923a 100644
--- a/src/Generator.cpp
+++ b/src/Generator.cpp
@@ -1665,6 +1665,28 @@ void GeneratorBase::pre_schedule() {
 void GeneratorBase::post_schedule() {
 }
 
+GeneratorInput<Buffer<>> *GeneratorBase::add_input(const ImageParam &existing) {
+    check_exact_phase(GeneratorBase::ConfigureCalled);
+    claim_name(existing.name(), "input");
+    auto *p = new GeneratorInput<Buffer<>>(existing.name(), existing.type(), existing.dimensions());
+    p->generator = this;
+    p->adopt(existing.parameter());
+    param_info_ptr->owned_extras.push_back(std::unique_ptr<Internal::GIOBase>(p));
+    param_info_ptr->filter_inputs.push_back(p);
+    return p;
+}
+
+GeneratorOutput<Buffer<>> *GeneratorBase::add_output(const Func &existing) {
+    check_exact_phase(GeneratorBase::ConfigureCalled);
+    claim_name(existing.name(), "output");
+    auto *p = new GeneratorOutput<Buffer<>>(existing.name(), existing.types(), existing.dimensions());
+    p->generator = this;
+    p->adopt(existing);
+    param_info_ptr->owned_extras.push_back(std::unique_ptr<Internal::GIOBase>(p));
+    param_info_ptr->filter_outputs.push_back(p);
+    return p;
+}
+
 void GeneratorBase::add_requirement(const Expr &condition, const std::vector<Expr> &error_args) {
     internal_assert(!pipeline.defined());
     requirements.push_back({condition, error_args});
@@ -2218,6 +2240,23 @@ void GeneratorInputBase::set_estimates_impl(const Region &estimates) {
     }
 }
 
+void GeneratorInputBase::adopt(const Parameter &p) {
+    user_assert(p.defined()) << "GeneratorBase::add_input(const ImageParam &): "
+                             << name() << " is not defined.\n";
+    user_assert(p.is_buffer()) << "GeneratorBase::add_input(const ImageParam &): "
+                               << name() << " is not a Buffer Parameter.\n";
+    check_matching_types({p.type()});
+    check_matching_dims(p.dimensions());
+    parameters_.clear();
+    exprs_.clear();
+    funcs_.clear();
+    funcs_.push_back(make_param_func(p, name()));
+    parameters_.push_back(p);
+    set_def_min_max();
+    verify_internals();
+    inputs_set = true;
+}
+
 GeneratorOutputBase::GeneratorOutputBase(size_t array_size, const std::string &name, ArgInfoKind kind, const std::vector<Type> &t, int d)
     : GIOBase(array_size, name, kind, t, d) {
     internal_assert(kind != ArgInfoKind::Scalar);
@@ -2239,6 +2278,9 @@ void GeneratorOutputBase::check_value_writable() const {
 }
 
 void GeneratorOutputBase::init_internals() {
+    if (adopted_) {
+        return;
+    }
     exprs_.clear();
     funcs_.clear();
     if (array_size_defined()) {
@@ -2250,6 +2292,17 @@ void GeneratorOutputBase::init_internals() {
     }
 }
 
+void GeneratorOutputBase::adopt(const Func &f) {
+    user_assert(f.defined()) << "GeneratorBase::add_output(const Func &): "
+                             << name() << " is not defined.\n";
+    check_matching_types(f.types());
+    check_matching_dims(f.dimensions());
+    exprs_.clear();
+    funcs_.clear();
+    funcs_.push_back(f);
+    adopted_ = true;
+}
+
 void GeneratorOutputBase::resize(size_t size) {
     internal_assert(is_array());
     internal_assert(!array_size_defined()) << "You may only call " << name()
diff --git a/src/Generator.h b/src/Generator.h
index d88bff8bea3f..bd39c7d1a640 100644
--- a/src/Generator.h
+++ b/src/Generator.h
@@ -1578,6 +1578,14 @@ class GeneratorInputBase : public GIOBase {
     void set_inputs(const std::vector<StubInput> &inputs);
     bool inputs_set = false;
 
+    // Adopts `p` (which must be a defined Buffer Parameter) as this Input's
+    // backing Parameter directly, immediately (not waiting for the
+    // ordinary InputsSet phase set_inputs() above is used for) -- sets
+    // inputs_set so init_internals() leaves it alone. Only used by
+    // GeneratorBase::add_input(const ImageParam &); see there for why this
+    // exists.
+    void adopt(const Parameter &p);
+
     virtual void set_def_min_max();
 
     void verify_internals() override;
@@ -2396,6 +2404,15 @@ class GeneratorOutputBase : public GIOBase {
         return "Output";
     }
 
+    // Adopts `f` (which must already be defined) as this Output's value
+    // directly, bypassing the usual "generate() assigns via operator()="
+    // path -- so init_internals() must leave `funcs_` alone once this has
+    // been called, the same way GeneratorInputBase::init_internals() skips
+    // its own rebuild when inputs_set is true. Only used by
+    // GeneratorBase::add_output(Func); see there for why this exists.
+    void adopt(const Func &f);
+    bool adopted_ = false;
+
 public:
     ~GeneratorOutputBase() override;
 };
@@ -3433,6 +3450,27 @@ class GeneratorBase : public NamesInterface, public AbstractGenerator {
         return p;
     }
 
+    /** Declares an Input<Buffer<>> backed directly by `existing` -- e.g. an
+     * ImageParam a Pipeline::compute_offline() call minted, or a
+     * Func::approximate_by() round trip severed into -- instead of a fresh
+     * one for generate() to leave for the caller to Buffer::set(). Useful
+     * when a Generator's whole pipeline is assembled in configure() (see
+     * doc/ApproximationDesign.md's "Usage shape" section for the motivating
+     * case: quantize/dequantize sharing a single approximate_by()/
+     * compute_offline() call, each adopting whichever half applies to it),
+     * leaving generate() an empty stub. `existing` must already be defined
+     * (have a concrete type/dimensionality); its own name is used as the
+     * port's name. */
+    GeneratorInput<Buffer<>> *add_input(const ImageParam &existing);
+
+    /** Declares an Output<Buffer<>> whose value is `existing` directly --
+     * e.g. the encoded/decoded Func half of an approximate_by()/
+     * compute_offline() split -- instead of a fresh, undefined Func for
+     * generate() to assign via operator(). See add_input(const ImageParam&)
+     * above for the motivating use. `existing` must already be defined;
+     * its own name is used as the port's name. */
+    GeneratorOutput<Buffer<>> *add_output(const Func &existing);
+
     void add_requirement(const Expr &condition, const std::vector<Expr> &error_args);
 
     template<typename... Args,
diff --git a/src/Pipeline.cpp b/src/Pipeline.cpp
index e8faf1561e3c..08c0708f639e 100644
--- a/src/Pipeline.cpp
+++ b/src/Pipeline.cpp
@@ -8,6 +8,7 @@
 #include "FindCalls.h"
 #include "Func.h"
 #include "IRVisitor.h"
+#include "ImageParam.h"
 #include "InferArguments.h"
 #include "LLVM_Output.h"
 #include "Lower.h"
@@ -231,6 +232,81 @@ std::vector<Internal::Stmt> Pipeline::requirements() const {
     return contents->requirements;
 }
 
+namespace {
+
+ComputeOfflineResult compute_offline_impl(const Pipeline &pipeline, const vector<Func> &to_sever,
+                                          const vector<ImageParam> *bind_to) {
+    vector<Function> output_funcs;
+    for (const Func &f : pipeline.outputs()) {
+        output_funcs.push_back(f.function());
+    }
+    std::map<string, Function> env = build_environment(output_funcs);
+
+    // Everything transitively reachable from to_sever (to_sever itself, plus
+    // anything only *it* depends on, e.g. a per-block reduction Func or a
+    // sibling encode() output one of to_sever's own definitions reads) is
+    // part of the offline half and must keep computing its true values --
+    // none of it should have its calls redirected to an online stand-in,
+    // even if it's also present in `env` because it used to be reachable
+    // from *this's outputs before severing.
+    vector<Function> to_sever_funcs;
+    to_sever_funcs.reserve(to_sever.size());
+    for (const Func &f : to_sever) {
+        to_sever_funcs.push_back(f.function());
+    }
+    std::map<string, Function> offline_env = build_environment(to_sever_funcs);
+
+    if (bind_to) {
+        user_assert(bind_to->size() == to_sever.size())
+            << "Pipeline::compute_offline(): bind_to has " << bind_to->size()
+            << " ImageParams, but to_sever has " << to_sever.size() << " Funcs\n";
+    }
+
+    std::map<FunctionPtr, FunctionPtr> substitutions;
+    vector<ImageParam> online_inputs;
+    online_inputs.reserve(to_sever.size());
+    for (size_t i = 0; i < to_sever.size(); i++) {
+        const Func &f = to_sever[i];
+        user_assert(f.types().size() == 1)
+            << "Pipeline::compute_offline() requires single-valued Funcs, but "
+            << f.name() << " has " << f.types().size() << " values\n";
+
+        ImageParam im = bind_to ? (*bind_to)[i] : ImageParam(f.types()[0], f.dimensions(), f.name() + "_im");
+        if (bind_to) {
+            user_assert(im.type() == f.types()[0] && im.dimensions() == f.dimensions())
+                << "Pipeline::compute_offline(): bind_to[" << i << "] (" << im.name()
+                << ") has type/dimensionality mismatched with " << f.name() << "\n";
+        }
+
+        Func stand_in(f.name() + "_offline_input");
+        vector<Var> args = f.args();
+        vector<Expr> args_as_exprs(args.begin(), args.end());
+        stand_in(args) = im(args_as_exprs);
+
+        substitutions[f.function().get_contents()] = stand_in.function().get_contents();
+        online_inputs.push_back(im);
+    }
+
+    for (auto &entry : env) {
+        if (offline_env.count(entry.first)) {
+            continue;
+        }
+        entry.second.substitute_calls(substitutions);
+    }
+
+    return {Pipeline(to_sever), std::move(online_inputs)};
+}
+
+}  // namespace
+
+ComputeOfflineResult Pipeline::compute_offline(const vector<Func> &to_sever) {
+    return compute_offline_impl(*this, to_sever, nullptr);
+}
+
+ComputeOfflineResult Pipeline::compute_offline(const vector<Func> &to_sever, const vector<ImageParam> &bind_to) {
+    return compute_offline_impl(*this, to_sever, &bind_to);
+}
+
 /* static */
 std::map<std::string, AutoSchedulerFn> &Pipeline::get_autoscheduler_map() {
     static std::map<std::string, AutoSchedulerFn> autoschedulers = {};
diff --git a/src/Pipeline.h b/src/Pipeline.h
index fac90b99a8dd..981e81b754e9 100644
--- a/src/Pipeline.h
+++ b/src/Pipeline.h
@@ -26,7 +26,9 @@ struct Argument;
 class Callable;
 class Func;
 class FuncVec;
+class ImageParam;
 struct PipelineContents;
+struct ComputeOfflineResult;
 
 /** Special the Autoscheduler to be used (if any), along with arbitrary
  * additional arguments specific to the given Autoscheduler.
@@ -211,6 +213,41 @@ class Pipeline {
     /** Get the requirements of this pipeline. */
     std::vector<Internal::Stmt> requirements() const;
 
+    /** Rewrite every call to each Func in `to_sever`, anywhere in this
+     * Pipeline's transitive call graph, to instead call a fresh ImageParam
+     * of matching type and dimensionality -- severing this Pipeline's
+     * *computation* of `to_sever` (and anything only `to_sever` depended on)
+     * while preserving the shape contract those Funcs stood in for. This is
+     * eager and destructive, like rfactor()/approximate_by(): by the time
+     * this call returns, none of this Pipeline's outputs depend on
+     * `to_sever` for their computation, only for the shape the returned
+     * ImageParams must satisfy.
+     *
+     * This is the seam doc/ApproximationDesign.md calls "compute_offline":
+     * splicing it after a static weight quantizer's encode() step turns
+     * that quantization from something recomputed on every realize() into
+     * ordinary input data, supplied once from wherever `offline` was
+     * realized or compiled.
+     *
+     * v1 restriction: each Func in `to_sever` must be single-valued (no
+     * Tuples). */
+    ComputeOfflineResult compute_offline(const std::vector<Func> &to_sever);
+
+    /** Like compute_offline(const std::vector<Func> &) above, but binds each
+     * severed Func to a caller-supplied ImageParam instead of minting a
+     * fresh one -- e.g. a Generator's own Input<Buffer<>> (which converts to
+     * ImageParam implicitly), so the online half's severed input is exactly
+     * the port `configure()`/`add_input()` already declared, rather than an
+     * unrelated ImageParam nothing else in the compiled artifact knows
+     * about. `bind_to` must have the same length as `to_sever`, and each
+     * `bind_to[i]` must have the same type and dimensionality as
+     * `to_sever[i]`. `online_inputs` in the result is `bind_to` itself,
+     * returned for symmetry with the other overload -- callers that already
+     * have `bind_to` don't need it, but code generic over both overloads
+     * still gets a uniform result shape. */
+    ComputeOfflineResult compute_offline(const std::vector<Func> &to_sever,
+                                         const std::vector<ImageParam> &bind_to);
+
     /** Generate a schedule for the pipeline using the specified autoscheduler. */
     AutoSchedulerResults apply_autoscheduler(const Target &target,
                                              const AutoschedulerParams &autoscheduler_params) const;
@@ -519,6 +556,17 @@ class Pipeline {
     std::string generate_function_name() const;
 };
 
+/** The result of Pipeline::compute_offline(): `offline` computes exactly
+ * `to_sever`'s original values, unmodified -- realize it once (JIT) or
+ * compile it as its own artifact (AOT), then feed the result to
+ * `online_inputs` before realizing the pipeline compute_offline() was called
+ * on again. `online_inputs` has one ImageParam per element of `to_sever`, in
+ * the same order, with matching type and dimensionality. */
+struct ComputeOfflineResult {
+    Pipeline offline;
+    std::vector<ImageParam> online_inputs;
+};
+
 struct ExternSignature {
 private:
     Type ret_type_;  // Only meaningful if is_void_return is false; must be default value otherwise
diff --git a/test/correctness/CMakeLists.txt b/test/correctness/CMakeLists.txt
index 6794e958e5e1..0a92b81c8739 100644
--- a/test/correctness/CMakeLists.txt
+++ b/test/correctness/CMakeLists.txt
@@ -5,6 +5,8 @@ tests(
     SOURCES
     # keep-sorted start case=no
     align_bounds.cpp
+    approximate_by.cpp
+    approximation_components.cpp
     argmax.cpp
     arm_cpu_detect.cpp
     associativity.cpp
@@ -60,6 +62,7 @@ tests(
     compute_at_reordered_update_stage.cpp
     compute_at_split_rvar.cpp
     compute_inside_guard.cpp
+    compute_offline.cpp
     compute_with_in.cpp
     compute_with_inlined.cpp
     computed_index.cpp
diff --git a/test/correctness/approximate_by.cpp b/test/correctness/approximate_by.cpp
new file mode 100644
index 000000000000..e36192b08c84
--- /dev/null
+++ b/test/correctness/approximate_by.cpp
@@ -0,0 +1,166 @@
+#include "Halide.h"
+#include <cmath>
+#include <cstdio>
+
+using namespace Halide;
+
+namespace {
+
+constexpr int kBlockSize = 8;
+
+// A minimal symmetric integer quantizer -- self-contained (no relation to
+// any specific real-world format), just enough to exercise: encode()
+// returning multiple Funcs plus a genuine scheduling-only handle (the
+// per-block amax reduction), decode() combining them back into a single
+// Func matching the original's signature, and approximate_by()'s eager
+// substitution.
+class SymmetricQuantizer : public Approximation {
+public:
+    EncodeResult encode(std::vector<Func> inputs) override {
+        Func f = inputs[0];
+        Var x("x"), i("i");
+        RDom r(0, kBlockSize, "r");
+
+        Func amax("amax");
+        amax(i) = 0.0f;
+        amax(i) = max(amax(i), abs(f(i * kBlockSize + r)));
+
+        Func d("d");
+        d(i) = amax(i) / 127.0f;
+
+        Func q("q");
+        Expr id = select(d(x / kBlockSize) != 0.0f, 1.0f / d(x / kBlockSize), 0.0f);
+        q(x) = cast<int8_t>(clamp(round(f(x) * id), -127, 127));
+
+        return {{q, d}, {amax}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        Func q = encoded[0], d = encoded[1];
+        Var x("x");
+        Func dequantized("dequantized");
+        dequantized(x) = cast<float>(q(x)) * d(x / kBlockSize);
+        return {{dequantized}, {}};
+    }
+};
+
+}  // namespace
+
+int main(int argc, char **argv) {
+    Var x("x");
+
+    Func f("f");
+    f(x) = sin(cast<float>(x) * 0.1f) * 100.0f;
+
+    // g is rewired by approximate_by() below; h is not, and must keep
+    // seeing the exact, unquantized f.
+    Func g("g");
+    g(x) = f(x) * 2.0f + 1.0f;
+
+    Func h("h");
+    h(x) = f(x) * 3.0f;
+
+    SymmetricQuantizer quant;
+    ApproximationResult result = f.approximate_by(quant, {g});
+
+    // Stage tracing preserves opaque identities through nested combinators,
+    // including repeated component types and Apply/TrustedInverse ownership.
+    Identity first_identity, second_identity, trusted_encoder, trusted_decoder;
+    const ApproximationStageKey first_key = first_identity.stage_key();
+    const ApproximationStageKey second_key = second_identity.stage_key();
+    const ApproximationStageKey encoder_key = trusted_encoder.stage_key();
+    const ApproximationStageKey decoder_key = trusted_decoder.stage_key();
+    Apply applied(0, second_identity);
+    const ApproximationStageKey apply_key = applied.stage_key();
+    TrustedInverse trusted(trusted_encoder, trusted_decoder);
+    const ApproximationStageKey trusted_key = trusted.stage_key();
+    Compose nested(first_identity, std::move(applied), std::move(trusted));
+    const ApproximationStageKey nested_key = nested.stage_key();
+
+    Func traced_source("traced_source"), traced_consumer("traced_consumer");
+    traced_source(x) = cast<float>(x);
+    traced_consumer(x) = traced_source(x);
+    ApproximationResult traced = traced_source.approximate_by(nested, {traced_consumer});
+
+    auto require_port = [&](const ApproximationStageKey &key, const char *label) {
+        if (!traced.encoded_by(key).defined() || !traced.decoded_by(key).defined()) {
+            printf("Missing encode/decode stage trace for %s\n", label);
+            return false;
+        }
+        return true;
+    };
+    if (!require_port(first_key, "first repeated Identity") ||
+        !require_port(second_key, "second repeated Identity") ||
+        !require_port(apply_key, "Apply") ||
+        !require_port(trusted_key, "TrustedInverse") ||
+        !require_port(nested_key, "outer Compose")) {
+        return 1;
+    }
+    if (!traced.encoded_by(encoder_key).defined() || traced.decoded_by(encoder_key).defined() ||
+        traced.encoded_by(decoder_key).defined() || !traced.decoded_by(decoder_key).defined()) {
+        printf("TrustedInverse did not preserve direction-specific child traces\n");
+        return 1;
+    }
+    if (first_key == second_key) {
+        printf("Distinct Approximation instances unexpectedly share a stage key\n");
+        return 1;
+    }
+    Identity unused;
+    if (traced.encoded_by(unused.stage_key()).defined() ||
+        traced.decoded_by(unused.stage_key()).defined() ||
+        traced.encoded_by(first_key, 1).defined() ||
+        traced.decoded_by(first_key, 1).defined()) {
+        printf("Invalid stage key or port unexpectedly resolved\n");
+        return 1;
+    }
+
+    if (result.handles.empty()) {
+        printf("Expected approximate_by() to return scheduling handles\n");
+        return 1;
+    }
+    result.replacement.compute_root();
+    for (Func handle : result.handles) {
+        handle.compute_root();
+    }
+
+    const int kSize = 64;
+    Buffer<float> g_out = g.realize({kSize});
+    Buffer<float> h_out = h.realize({kSize});
+
+    for (int i = 0; i < kSize; i++) {
+        const float fx = sinf(i * 0.1f) * 100.0f;
+
+        // Independently recompute the same per-block quantization encode()
+        // performs, to build a bit-exact reference for what g should see.
+        const int block = i / kBlockSize;
+        float amax = 0.0f;
+        for (int j = 0; j < kBlockSize; j++) {
+            const float v = sinf((block * kBlockSize + j) * 0.1f) * 100.0f;
+            amax = std::max(amax, std::fabs(v));
+        }
+        const float d = amax / 127.0f;
+        const float id = d != 0.0f ? 1.0f / d : 0.0f;
+        float q = std::round(fx * id);
+        q = std::max(-127.0f, std::min(127.0f, q));
+        const float dequantized = q * d;
+
+        const float expected_g = dequantized * 2.0f + 1.0f;
+        if (std::fabs(g_out(i) - expected_g) > 1e-4f) {
+            printf("g(%d) = %f, expected %f -- approximate_by's substitution did not take effect\n",
+                   i, g_out(i), expected_g);
+            return 1;
+        }
+
+        // h was never passed as a consumer to approximate_by(): it must
+        // see the real f, not the quantized round trip.
+        const float expected_h = fx * 3.0f;
+        if (std::fabs(h_out(i) - expected_h) > 1e-4f) {
+            printf("h(%d) = %f, expected %f -- approximate_by affected a Func not in `consumers`\n",
+                   i, h_out(i), expected_h);
+            return 1;
+        }
+    }
+
+    printf("Success!\n");
+    return 0;
+}
diff --git a/test/correctness/approximation_components.cpp b/test/correctness/approximation_components.cpp
new file mode 100644
index 000000000000..9e4d1e3627c9
--- /dev/null
+++ b/test/correctness/approximation_components.cpp
@@ -0,0 +1,297 @@
+#include "Halide.h"
+
+#include <cmath>
+#include <cstdio>
+
+using namespace Halide;
+
+namespace {
+
+int test_struct_layout_1d() {
+    Type record_type = Type::Struct({{"d", Float(16)}, {"qh", UInt(8), 4}, {"qs", UInt(8), 16}});
+    Var element("element"), record("record");
+    Func qs("qs"), qh("qh"), d("d");
+    qs(element, record) = cast<uint8_t>(element + 3 * record);
+    qh(element, record) = cast<uint8_t>(0x80 + element + record);
+    d(record) = cast<float16_t>(cast<float>(record) + 0.5f);
+
+    StructLayout layout(record_type, {"qs", "qh", "d"});
+    DecodeResult decoded = layout.decode(layout.encode({qs, qh, d}).encoded);
+    Buffer<uint8_t> out_qs = decoded.decoded[0].realize({16, 3});
+    Buffer<uint8_t> out_qh = decoded.decoded[1].realize({4, 3});
+    Buffer<float16_t> out_d = decoded.decoded[2].realize({3});
+    for (int r = 0; r < 3; ++r) {
+        if ((float)out_d(r) != r + 0.5f) {
+            return 1;
+        }
+        for (int i = 0; i < 16; ++i) {
+            if (out_qs(i, r) != (uint8_t)(i + 3 * r)) {
+                return 1;
+            }
+        }
+        for (int i = 0; i < 4; ++i) {
+            if (out_qh(i, r) != (uint8_t)(0x80 + i + r)) {
+                return 1;
+            }
+        }
+    }
+    return 0;
+}
+
+int test_struct_layout_2d() {
+    Type record_type = Type::Struct({{"tag", UInt(16)}, {"pixels", UInt(8), 3}});
+    Var element("element"), x("x"), y("y");
+    Func pixels("pixels"), tag("tag");
+    pixels(element, x, y) = cast<uint8_t>(element + 10 * x + 30 * y);
+    tag(x, y) = cast<uint16_t>(100 + x + 4 * y);
+    StructLayout layout(record_type, {"pixels", "tag"}, 2);
+    DecodeResult decoded = layout.decode(layout.encode({pixels, tag}).encoded);
+    Buffer<uint8_t> out_pixels = decoded.decoded[0].realize({3, 4, 2});
+    Buffer<uint16_t> out_tag = decoded.decoded[1].realize({4, 2});
+    for (int yy = 0; yy < 2; ++yy) {
+        for (int xx = 0; xx < 4; ++xx) {
+            if (out_tag(xx, yy) != 100 + xx + 4 * yy) {
+                return 1;
+            }
+            for (int i = 0; i < 3; ++i) {
+                if (out_pixels(i, xx, yy) != i + 10 * xx + 30 * yy) {
+                    return 1;
+                }
+            }
+        }
+    }
+    return 0;
+}
+
+int test_struct_layout_contract_errors() {
+    Type record_type = Type::Struct({{"tag", UInt(16)}, {"pixels", UInt(8), 3}});
+    Var element("element"), record("record");
+    Func pixels("pixels"), wrong_tag("wrong_tag");
+    pixels(element, record) = cast<uint8_t>(element);
+    wrong_tag(record) = cast<int16_t>(record);
+    try {
+        StructLayout layout(record_type, {"pixels", "tag"});
+        (void)layout.encode({pixels, wrong_tag});
+        return 1;
+    } catch (const CompileError &) {
+    }
+    try {
+        StructLayout duplicate(record_type, {"pixels", "pixels"});
+        return 1;
+    } catch (const CompileError &) {
+    }
+    return 0;
+}
+
+int test_scalar_components() {
+    Var record("record");
+    Func values("values");
+    values(record) = cast<float>(record) / 3.0f;
+    StorageCast<float, float16_t> storage;
+    EncodeResult stored = storage.encode({values});
+    DecodeResult cast_roundtrip = storage.decode(stored.encoded);
+    Buffer<float> out = cast_roundtrip.decoded[0].realize({8});
+    for (int i = 0; i < 8; ++i) {
+        float expected = (float)(float16_t)(i / 3.0f);
+        if (out(i) != expected) {
+            printf("StorageCast mismatch at %d: %g vs %g\n", i, out(i), expected);
+            return 1;
+        }
+    }
+
+    Func words("words");
+    words(record) = cast<uint32_t>((int32_t)0x10203040) + cast<uint32_t>(record);
+    LittleEndianScalarPack<uint32_t> little_endian;
+    EncodeResult bytes = little_endian.encode({words});
+    DecodeResult word_roundtrip = little_endian.decode(bytes.encoded);
+    Buffer<uint8_t> packed = bytes.encoded[0].realize({4, 5});
+    Buffer<uint32_t> unpacked = word_roundtrip.decoded[0].realize({5});
+    for (int r = 0; r < 5; ++r) {
+        if (unpacked(r) != 0x10203040u + r || packed(0, r) != (uint8_t)(0x40 + r) ||
+            packed(1, r) != 0x30 || packed(2, r) != 0x20 || packed(3, r) != 0x10) {
+            printf("LittleEndian mismatch at %d: %08x [%02x %02x %02x %02x]\n", r,
+                   unpacked(r), packed(0, r), packed(1, r), packed(2, r), packed(3, r));
+            return 1;
+        }
+    }
+    return 0;
+}
+
+int test_code_components() {
+    Var element("element"), record("record");
+    Func signed_nibbles("signed_nibbles");
+    signed_nibbles(element, record) = cast<int8_t>((element % 16) - 8);
+    AdditiveOffset<int8_t, uint8_t> offset(8);
+    EncodeResult offset_codes = offset.encode({signed_nibbles});
+    DecodeResult signed_roundtrip = offset.decode(offset_codes.encoded);
+    Buffer<uint8_t> stored_nibbles = offset_codes.encoded[0].realize({16, 2});
+    Buffer<int8_t> restored_nibbles = signed_roundtrip.decoded[0].realize({16, 2});
+    for (int r = 0; r < 2; ++r) {
+        for (int i = 0; i < 16; ++i) {
+            if (stored_nibbles(i, r) != i || restored_nibbles(i, r) != i - 8) {
+                return 1;
+            }
+        }
+    }
+
+    Func high("high");
+    high(element, record) = cast<int8_t>(select(((element + record) & 1) != 0, 0, -16));
+    BinaryAlphabetPack<int8_t> binary(32, UInt(32), -16, 0);
+    EncodeResult word = binary.encode({high});
+    word.encoded[0].compute_root();
+    DecodeResult expanded = binary.decode(word.encoded);
+    Buffer<uint32_t> packed_word = word.encoded[0].realize({2});
+    Buffer<int8_t> out_high = expanded.decoded[0].realize({32, 2});
+    if (packed_word(0) != 0xaaaaaaaau || packed_word(1) != 0x55555555u) {
+        return 1;
+    }
+    for (int r = 0; r < 2; ++r) {
+        for (int i = 0; i < 32; ++i) {
+            int8_t expected = ((i + r) & 1) ? 0 : -16;
+            if (out_high(i, r) != expected) {
+                return 1;
+            }
+        }
+    }
+
+    Func codes("codes");
+    codes(element, record) = cast<int8_t>((element % 32) - 16);
+    AdditiveRadixSplit split(16, 16);
+    EncodeResult parts = split.encode({codes});
+    DecodeResult combined = split.decode(parts.encoded);
+    Buffer<uint8_t> low = parts.encoded[0].realize({32, 1});
+    Buffer<int8_t> high_part = parts.encoded[1].realize({32, 1});
+    Buffer<int8_t> combined_codes = combined.decoded[0].realize({32, 1});
+    for (int i = 0; i < 32; ++i) {
+        if (low(i, 0) != (i & 15) || high_part(i, 0) != (i < 16 ? -16 : 0) ||
+            combined_codes(i, 0) != i - 16) {
+            return 1;
+        }
+    }
+
+    PlanarFieldPack planar(4, 16);
+    EncodeResult planar_bytes = planar.encode({parts.encoded[0]});
+    planar_bytes.encoded[0].compute_root();
+    DecodeResult planar_fields = planar.decode(planar_bytes.encoded);
+    Buffer<uint8_t> bytes_out = planar_bytes.encoded[0].realize({16, 1});
+    Buffer<uint8_t> fields_out = planar_fields.decoded[0].realize({32, 1});
+    for (int i = 0; i < 16; ++i) {
+        if (bytes_out(i, 0) != (uint8_t)(i | (i << 4)) ||
+            fields_out(i, 0) != i || fields_out(i + 16, 0) != i) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
+int test_block_components() {
+    Var k("k");
+    Func flat("flat");
+    flat(k) = cast<float>(k);
+    BlockReshape reshape(32);
+    DecodeResult reshaped = reshape.decode(reshape.encode({flat}).encoded);
+    Buffer<float> roundtrip = reshaped.decoded[0].realize({96});
+    for (int i = 0; i < 96; ++i) {
+        if (roundtrip(i) != i) {
+            return 1;
+        }
+    }
+
+    Var element("element"), record("record");
+    Func blocks("blocks");
+    blocks(element, record) = cast<float>(element - 16);
+    SymmetricBlockQuantize quantize(32, 16, BlockRoundingMode::TruncateHalfUpWithOffset,
+                                    BlockScaleAnchor::ExtremeSignedValue);
+    EncodeResult quantized = quantize.encode({blocks});
+    for (Func handle : quantized.handles) {
+        handle.compute_root();
+    }
+    DecodeResult dequantized = quantize.decode(quantized.encoded);
+    Buffer<int8_t> codes = quantized.encoded[0].realize({32, 1});
+    Buffer<float> scale = quantized.encoded[1].realize({1});
+    Buffer<float> values = dequantized.decoded[0].realize({32, 1});
+    if (scale(0) != 1.0f) {
+        return 1;
+    }
+    for (int i = 0; i < 32; ++i) {
+        if (codes(i, 0) != i - 16 || values(i, 0) != i - 16) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
+int test_standard_quant_compositions() {
+    Var k("k");
+
+    Type q4_type = Type::Struct({{"d", Float(16)}, {"qs", UInt(8), 16}});
+    Func q4_values("q4_values");
+    q4_values(k) = cast<float>((k % 16) - 8);
+    Compose q4(
+        StructLayout{q4_type, {"qs", "d"}},
+        Apply{1, StorageCast<float, float16_t>{}},
+        Apply{0, PlanarFieldPack{4, 16}},
+        Apply{0, AdditiveOffset<int8_t, uint8_t>{8}},
+        SymmetricBlockQuantize{32, 8, BlockRoundingMode::TruncateHalfUpWithOffset,
+                               BlockScaleAnchor::ExtremeSignedValue},
+        BlockReshape{32});
+    EncodeResult q4_encoded = q4.encode({q4_values});
+    for (Func handle : q4_encoded.handles) {
+        handle.compute_root();
+    }
+    DecodeResult q4_decoded = q4.decode(q4_encoded.encoded);
+    Buffer<float> q4_roundtrip = q4_decoded.decoded[0].realize({64});
+    for (int i = 0; i < 64; ++i) {
+        if (q4_roundtrip(i) != (i % 16) - 8) {
+            return 1;
+        }
+    }
+
+    Type q8_type = Type::Struct({{"d", Float(16)}, {"qs", Int(8), 32}});
+    Func q8_values("q8_values");
+    Expr local = k % 32;
+    q8_values(k) = cast<float>(select(local == 0, -127, local - 16));
+    Compose q8(
+        StructLayout{q8_type, {"qs", "d"}},
+        Apply{1, StorageCast<float, float16_t>{}},
+        SymmetricBlockQuantize{32, 127, BlockRoundingMode::Nearest,
+                               BlockScaleAnchor::AbsMax},
+        BlockReshape{32});
+    EncodeResult q8_encoded = q8.encode({q8_values});
+    for (Func handle : q8_encoded.handles) {
+        handle.compute_root();
+    }
+    DecodeResult q8_decoded = q8.decode(q8_encoded.encoded);
+    Buffer<float> q8_roundtrip = q8_decoded.decoded[0].realize({64});
+    for (int i = 0; i < 64; ++i) {
+        int local_i = i % 32;
+        float expected = local_i == 0 ? -127.0f : local_i - 16.0f;
+        if (q8_roundtrip(i) != expected) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+    struct Test {
+        const char *name;
+        int (*run)();
+    } tests[] = {{"StructLayout 1-D", test_struct_layout_1d},
+                 {"StructLayout 2-D", test_struct_layout_2d},
+                 {"StructLayout contract errors", test_struct_layout_contract_errors},
+                 {"scalar packs", test_scalar_components},
+                 {"code packs", test_code_components},
+                 {"block components", test_block_components},
+                 {"standard quant compositions", test_standard_quant_compositions}};
+    for (const Test &test : tests) {
+        if (test.run()) {
+            printf("Approximation component test failed: %s\n", test.name);
+            return 1;
+        }
+    }
+    printf("Success!\n");
+    return 0;
+}
diff --git a/test/correctness/compute_offline.cpp b/test/correctness/compute_offline.cpp
new file mode 100644
index 000000000000..14e81266cba4
--- /dev/null
+++ b/test/correctness/compute_offline.cpp
@@ -0,0 +1,191 @@
+#include "Halide.h"
+#include <cmath>
+#include <cstdio>
+#include <functional>
+#include <vector>
+
+using namespace Halide;
+
+// Exercises Pipeline::compute_offline: rewriting calls to a Func into calls to
+// a fresh ImageParam, severing a pipeline's *computation* of that Func while
+// preserving the shape contract it stood in for.
+
+namespace {
+
+// Symmetric int8 quantization for a 1-D Func, same shape as the one in
+// approximation_composition.cpp: encode() returns q(k) and scale(); decode()
+// dequantizes q(k) * scale().
+class SymmetricQuantize : public Approximation {
+public:
+    explicit SymmetricQuantize(int k)
+        : k_(k) {
+    }
+
+    EncodeResult encode(std::vector<Func> inputs) override {
+        Func v = inputs[0];
+        Var k("k");
+        RDom r(0, k_, "r");
+
+        Func amax("amax");
+        amax() = 0.0f;
+        amax() = max(amax(), abs(v(r)));
+
+        Func d("scale");
+        d() = amax() / 127.0f;
+
+        Func q("q");
+        Expr id = select(d() != 0.0f, 1.0f / d(), 0.0f);
+        q(k) = cast<int8_t>(clamp(round(v(k) * id), -127, 127));
+
+        return {{q, d}, {amax}};
+    }
+
+    DecodeResult decode(std::vector<Func> encoded) override {
+        Func q = encoded[0], d = encoded[1];
+        Var k("k");
+        Func dequantized("dequantized");
+        dequantized(k) = cast<float>(q(k)) * d();
+        return {{dequantized}, {}};
+    }
+
+private:
+    int k_;
+};
+
+void reference_symmetric_quantize(int k, const std::function<float(int)> &values,
+                                  std::vector<int8_t> &q, float &scale) {
+    float amax = 0.0f;
+    for (int kk = 0; kk < k; kk++) {
+        amax = std::max(amax, std::fabs(values(kk)));
+    }
+    scale = amax / 127.0f;
+    float id = scale != 0.0f ? 1.0f / scale : 0.0f;
+    q.resize(k);
+    for (int kk = 0; kk < k; kk++) {
+        int v = (int)std::round(values(kk) * id);
+        q[kk] = (int8_t)std::max(-127, std::min(127, v));
+    }
+}
+
+// A minimal, non-Approximation check that compute_offline() actually severs
+// the call graph, rather than being a no-op that happens to still produce the
+// right answer once. f(x) = x*2, g(x) = f(x) + 1: after
+// Pipeline({g}).compute_offline({f}), g must stop depending on f's own
+// computation -- setting a buffer on the returned ImageParam that disagrees
+// with f's true values must change g's output accordingly.
+int minimal_severance_test() {
+    Var x("x");
+    Func f("f"), g("g");
+    f(x) = x * 2;
+    g(x) = f(x) + 1;
+
+    ComputeOfflineResult split = Pipeline({g}).compute_offline({f});
+
+    Buffer<int> f_values = split.offline.realize({10});
+    for (int x = 0; x < 10; x++) {
+        if (f_values(x) != x * 2) {
+            printf("minimal_severance_test: offline f(%d) = %d, expected %d\n", x, f_values(x), x * 2);
+            return 1;
+        }
+    }
+
+    // Feed f's true values through the ImageParam: g should compute as if
+    // nothing changed.
+    split.online_inputs[0].set(f_values);
+    Buffer<int> g_true = g.realize({10});
+    for (int x = 0; x < 10; x++) {
+        if (g_true(x) != x * 2 + 1) {
+            printf("minimal_severance_test: g(%d) = %d with true f, expected %d\n", x, g_true(x), x * 2 + 1);
+            return 1;
+        }
+    }
+
+    // Now feed different values through the same ImageParam. If g still
+    // depended on f's own computation, this would have no effect.
+    Buffer<int> f_fake(10);
+    for (int x = 0; x < 10; x++) {
+        f_fake(x) = 1000 + x;
+    }
+    split.online_inputs[0].set(f_fake);
+    Buffer<int> g_fake = g.realize({10});
+    for (int x = 0; x < 10; x++) {
+        int expected = f_fake(x) + 1;
+        if (g_fake(x) != expected) {
+            printf("minimal_severance_test: g(%d) = %d with fake f, expected %d "
+                   "(compute_offline() did not actually sever the call graph)\n",
+                   x, g_fake(x), expected);
+            return 1;
+        }
+    }
+
+    return 0;
+}
+
+// A realistic case: sever a quantized vector's encode() output from a
+// consumer built via approximate_by(), and check the final result still
+// matches the plain-C++ reference round trip.
+int quantized_offline_test() {
+    const int K = 64;
+    Var k("k");
+
+    Func Vec("Vec");
+    Vec(k) = cos(cast<float>(k) * 0.05f) * 3.0f;
+
+    SymmetricQuantize quantize(K);
+
+    Func Result("Result");
+    Result(k) = Vec(k) * 2.0f;
+
+    ApproximationResult result = Vec.approximate_by(quantize, {Result});
+    Result.eager_inline({result.replacement});
+
+    // result.handles is [q, d, amax]: encode()'s two signature-contract
+    // outputs, then its own scheduling-only handle. q and d are the actual
+    // Funcs Result's call graph depends on (approximate_by() calls encode()
+    // internally; a separately-called quantize.encode({Vec}) here would
+    // build an unrelated, unconnected copy of the same graph shape).
+    std::vector<Func> encoded = {result.handles[0], result.handles[1]};
+    for (size_t i = 2; i < result.handles.size(); i++) {
+        result.handles[i].compute_root();
+    }
+
+    ComputeOfflineResult split = Pipeline({Result}).compute_offline(encoded);
+
+    // q(k) and scale() have different dimensionality (1-D vs scalar), so they
+    // can't share a single realize({sizes}) call -- realize into
+    // pre-allocated buffers of the right shape instead.
+    Buffer<int8_t> q_buf(K);
+    Buffer<float> scale_buf = Buffer<float>::make_scalar();
+    split.offline.realize({q_buf, scale_buf});
+    split.online_inputs[0].set(q_buf);
+    split.online_inputs[1].set(scale_buf);
+
+    Buffer<float> out = Result.realize({K});
+
+    std::vector<int8_t> ref_q;
+    float ref_scale;
+    reference_symmetric_quantize(K, [](int kk) { return cosf(kk * 0.05f) * 3.0f; }, ref_q, ref_scale);
+    for (int kk = 0; kk < K; kk++) {
+        float expected = (ref_q[kk] * ref_scale) * 2.0f;
+        if (std::fabs(out(kk) - expected) > 1e-3f * std::fabs(expected)) {
+            printf("quantized_offline_test: Result(%d) = %f, expected %f\n", kk, out(kk), expected);
+            return 1;
+        }
+    }
+
+    return 0;
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+    if (minimal_severance_test()) {
+        return 1;
+    }
+    if (quantized_offline_test()) {
+        return 1;
+    }
+
+    printf("Success!\n");
+    return 0;
+}
diff --git a/test/correctness/hoist_invariants.cpp b/test/correctness/hoist_invariants.cpp
index 7f61562ca580..46ba3552e5bb 100644
--- a/test/correctness/hoist_invariants.cpp
+++ b/test/correctness/hoist_invariants.cpp
@@ -588,6 +588,159 @@ int hoist_invariants_nothing_to_hoist_rejected_test() {
     return 0;
 }
 
+// An increment written as a sum of terms with *different* invariant factors has
+// no single factor to hoist. Each term gets its own accumulator instead, all
+// advanced by one loop, with the write-back applying each factor once.
+int hoist_invariants_terms_test() {
+    const int K = 64;
+    ImageParam G{Int(8), 1, "G"};
+    ImageParam H{Int(8), 1, "H"};
+    ImageParam A{Float(32), 1, "A"};
+    ImageParam B{Float(32), 1, "B"};
+
+    Var i{"i"};
+    RDom r(0, K, "r");
+
+    Func Acc{"Acc"};
+    Acc(i) = 0.0f;
+    Acc(i) += A(i) * cast<float>(G(r)) + B(i) * cast<float>(H(r));
+
+    std::vector<Func> Acc_intm = Acc.update().hoist_invariants();
+    internal_assert(Acc_intm.size() == 2)
+        << "hoist_invariants terms: expected one accumulator per term, got "
+        << Acc_intm.size() << "\n";
+    for (Func &f : Acc_intm) {
+        f.compute_root();
+    }
+
+    Buffer<int8_t> g_buf(K), h_buf(K);
+    Buffer<float> a_buf(1), b_buf(1);
+    for (int k = 0; k < K; k++) {
+        g_buf(k) = 3;
+        h_buf(k) = 5;
+    }
+    a_buf(0) = 2.0f;
+    b_buf(0) = 7.0f;
+    G.set(g_buf);
+    H.set(h_buf);
+    A.set(a_buf);
+    B.set(b_buf);
+
+    Buffer<float> result = Acc.realize({1});
+    const float expected = (2.0f * 3.0f + 7.0f * 5.0f) * (float)K;
+    internal_assert(result(0) == expected)
+        << "hoist_invariants terms: got " << result(0) << ", expected " << expected << "\n";
+
+    // Terms still get one accumulator each even when they share an identical
+    // factor: hoist_invariants() doesn't try to detect and merge such terms,
+    // so the accumulator count only ever depends on the number of terms.
+    Func Shared{"Shared"};
+    Shared(i) = 0.0f;
+    Shared(i) += A(i) * cast<float>(G(r)) + A(i) * cast<float>(H(r));
+    std::vector<Func> Shared_intm = Shared.update().hoist_invariants();
+    internal_assert(Shared_intm.size() == 2)
+        << "hoist_invariants terms: expected one accumulator per term even "
+        << "when terms share a factor, got " << Shared_intm.size() << "\n";
+    for (Func &f : Shared_intm) {
+        f.compute_root();
+    }
+
+    Buffer<float> shared = Shared.realize({1});
+    const float shared_expected = 2.0f * (3.0f + 5.0f) * (float)K;
+    internal_assert(shared(0) == shared_expected)
+        << "hoist_invariants shared factor: got " << shared(0) << ", expected "
+        << shared_expected << "\n";
+
+    return 0;
+}
+
+// distribute() multiplies a product of sums out so that hoist_invariants() can
+// see terms that were not written as terms. This is the affine-quantized dot
+// product: sum_k (d*q_k + m) * (e*p_k), whose expansion
+// d*e*sum_k(q_k*p_k) + m*e*sum_k(p_k) has two integer-bodied accumulators where
+// the unexpanded form has one float one.
+int hoist_invariants_distribute_test() {
+    const int K = 32;
+    ImageParam Q{Int(8), 1, "Q"};
+    ImageParam P{Int(8), 1, "P"};
+    ImageParam D{Float(32), 1, "D"};
+    ImageParam M{Float(32), 1, "M"};
+    ImageParam E{Float(32), 1, "E"};
+
+    Var i{"i"};
+    RDom r(0, K, "r");
+
+    Func Acc{"Acc"};
+    Acc(i) = 0.0f;
+    Acc(i) += (cast<float>(Q(r)) * D(i) + M(i)) * (cast<float>(P(r)) * E(i));
+
+    std::vector<Func> Acc_intm = Acc.update().distribute().hoist_invariants();
+    internal_assert(Acc_intm.size() == 2)
+        << "distribute: expected the multiplied-out increment to yield two "
+        << "accumulators, got " << Acc_intm.size() << "\n";
+
+    // Both bodies are integer sums of int8 products, so both retype -- and each
+    // accumulator is its own Func, so each may take its own target type.
+    Func qp = Acc_intm[0].change_type(Int(32));
+    Func p_sum = Acc_intm[1].change_type(Int(16));
+    internal_assert(qp.types()[0] == Int(32) && p_sum.types()[0] == Int(16))
+        << "distribute: retyping the accumulators separately gave "
+        << qp.types()[0] << " and " << p_sum.types()[0] << "\n";
+    qp.compute_root();
+    p_sum.compute_root();
+
+    Buffer<int8_t> q_buf(K), p_buf(K);
+    Buffer<float> d_buf(1), m_buf(1), e_buf(1);
+    int64_t sum_qp = 0, sum_p = 0;
+    for (int k = 0; k < K; k++) {
+        q_buf(k) = (int8_t)(k % 15);
+        p_buf(k) = (int8_t)((k * 5) % 127 - 63);
+        sum_qp += (int64_t)q_buf(k) * p_buf(k);
+        sum_p += p_buf(k);
+    }
+    d_buf(0) = 0.25f;
+    m_buf(0) = -0.5f;
+    e_buf(0) = 2.0f;
+    Q.set(q_buf);
+    P.set(p_buf);
+    D.set(d_buf);
+    M.set(m_buf);
+    E.set(e_buf);
+
+    Buffer<float> result = Acc.realize({1});
+    const float expected = 0.25f * 2.0f * (float)sum_qp + -0.5f * 2.0f * (float)sum_p;
+    internal_assert(std::abs(result(0) - expected) < 1e-3f)
+        << "distribute: got " << result(0) << ", expected " << expected << "\n";
+
+    return 0;
+}
+
+// distribute() is a schedule decision, so it says so when there is nothing to
+// multiply out rather than quietly leaving the reduction alone.
+int distribute_nothing_to_do_rejected_test() {
+    if (!Halide::exceptions_enabled()) {
+        return 0;
+    }
+    ImageParam A{Float(32), 1, "A"};
+    Var i{"i"};
+    RDom r(0, 8, "r");
+
+    Func f{"f"};
+    f(i) = 0.0f;
+    f(i) += A(i) * cast<float>(r);
+
+    try {
+        f.update().distribute();
+    } catch (const Halide::CompileError &e) {
+        const std::string msg = e.what();
+        internal_assert(msg.find("no product over a sum") != std::string::npos)
+            << "distribute() rejected the update for the wrong reason: " << msg << "\n";
+        return 0;
+    }
+    internal_assert(false) << "distribute() accepted an update with nothing to multiply out\n";
+    return 0;
+}
+
 }  // namespace
 
 int main(int argc, char **argv) {
@@ -610,6 +763,9 @@ int main(int argc, char **argv) {
         {"hoist_invariants test (after rfactor)", hoist_invariants_after_rfactor_test},
         {"hoist_invariants test (invalid law rejected)", hoist_invariants_invalid_law_rejected_test},
         {"hoist_invariants test (nothing to hoist rejected)", hoist_invariants_nothing_to_hoist_rejected_test},
+        {"hoist_invariants test (one accumulator per term)", hoist_invariants_terms_test},
+        {"distribute test (affine dot product)", hoist_invariants_distribute_test},
+        {"distribute test (nothing to distribute rejected)", distribute_nothing_to_do_rejected_test},
     };
 
     using Sharder = Halide::Internal::Test::Sharder;
diff --git a/test/correctness/struct_type_dot_product.cpp b/test/correctness/struct_type_dot_product.cpp
index 63067a80bb3d..186a74a5e9ab 100644
--- a/test/correctness/struct_type_dot_product.cpp
+++ b/test/correctness/struct_type_dot_product.cpp
@@ -107,7 +107,7 @@ struct Pipeline {
         RVar ko("ko"), ki("ki");
         result.update().split(r[1], ko, ki, 16);
         Func partial = result.update().rfactor(r[0], u);
-        Func dot = partial.update().eager_inline(x_wt, y_wt).hoist_invariants();
+        Func dot = partial.update().eager_inline(x_wt, y_wt).hoist_invariants()[0];
         Func dot_i32 = dot.change_type(Int(32));
 
         // One block's scaled contribution per outer step; its quants reduce
diff --git a/test/performance/tiled_matmul_arm_neon.cpp b/test/performance/tiled_matmul_arm_neon.cpp
index 807049e1e7b3..975b715f62fd 100644
--- a/test/performance/tiled_matmul_arm_neon.cpp
+++ b/test/performance/tiled_matmul_arm_neon.cpp
@@ -107,7 +107,7 @@ int main(int argc, char **argv) {
             // and Acc's own update collapses to one multiply per row,
             // Acc(i) = WtScale(i) * VecScale * Acc_wb(i). Acc_wb still accumulates
             // at Float(32) at this point.
-            Func Acc_wb = Acc.update().eager_inline(Wt, VecDq).hoist_invariants();
+            Func Acc_wb = Acc.update().eager_inline(Wt, VecDq).hoist_invariants()[0];
 
             // Second, factor Acc_wb's own (now scale-free) reduction by block.
             // Preserving ro turns it into a new dimension u of Acc_dot, so