Add LFM2.5 2.6B support and speed up decode

ARCHITECTURE.md

@@ -56,23 +56,26 @@ The parser normalizes integer metadata arrays to `u32` and string arrays to
 borrowable slices because those are the two array forms used by the model
 configuration and tokenizer.
 
-## Supported model layout
-
-The implementation targets one architecture and validates it at startup:
-
-- hidden width: 2048
-- layers: 24
-- attention layers: 2, 6, 10, 14, 18, 21
-- query heads: 32
-- KV heads: 8
-- head width: 64
-- dense FF width: 7168 for layers 0 and 1
-- MoE FF width: 1792
-- experts: 32, top 4 selected
-- short-convolution cache: 3 taps
-- vocabulary: 128,000
-
-All 256 required tensor names and shapes are checked before inference.
+## Supported model layouts
+
+The loader detects and validates two LFM2.5 configurations at startup:
+
+| Property | LFM2.5-2.6B | LFM2.5-8B-A1B |
+|---|---:|---:|
+| GGUF architecture | `lfm2` | `lfm2moe` |
+| Layers | 30 | 24 |
+| Attention layers | 2, 5, 9, 13, 17, 21, 24, 27 | 2, 6, 10, 14, 18, 21 |
+| Dense FF width | 10,752 in every layer | 7,168 in layers 0 and 1 |
+| MoE FF width | — | 1,792 |
+| Experts | — | 32, top 4 selected |
+| RoPE base | 10,000,000 | 5,000,000 |
+| Required tensors | 266 | 256 |
+
+Both models use a hidden width of 2,048, 32 query heads, 8 KV heads, 64-wide
+heads, a three-tap short-convolution cache, and a 128,000-token vocabulary. All
+required tensor names and shapes are checked before inference. Cache storage is
+sized from the detected runtime layer count so agents can use either model
+without a separate API.
 
 ## Forward pass
 
@@ -84,7 +87,7 @@ For a token at absolute position `p`:
    - execute grouped-query attention or gated short convolution;
    - add the operator residual;
    - RMSNorm again;
-   - execute dense SwiGLU or sparse top-4 MoE;
+   - execute dense SwiGLU or, for the 8B-A1B model, sparse top-4 MoE;
    - add the FFN residual;
 3. apply final RMSNorm;
 4. project through the tied token embedding to 128,000 logits.
@@ -150,9 +153,15 @@ layer because no other CPU is supported.
 Output rows are independent. A process-lifetime `std.Thread` pool assigns
 contiguous row chunks to workers from a shared counter, allowing faster cores
 to claim more work while preserving mmap locality within each chunk and
-avoiding per-operation fork/join overhead. macOS dispatch semaphores coordinate
-jobs; `dispatch_apply_f` remains a fallback if the pool cannot be initialized.
-The limit defaults to the logical CPU count and can be changed before inference.
+avoiding per-operation fork/join overhead. Helpers briefly spin on a generation
+counter between decode projections, then park on macOS dispatch semaphores when
+inference goes idle. The submitting thread also spins on completion, removing
+the per-projection wake/park round trip from sustained generation.
+`dispatch_apply_f` remains a fallback if the pool cannot be initialized.
+
+Prompt batches default to the logical CPU count. Memory-bound single-token
+matvecs retain scheduler headroom on machines with more than 12 logical CPUs;
+an explicit thread limit overrides both defaults.
 
 ### Accelerate
 
@@ -228,7 +237,7 @@ The fast suite covers:
 - sampling;
 - tool schema/call parsing;
 - tokenizer byte mapping and scanner behavior;
-- the derived 256-tensor schedule.
+- the derived 256- and 266-tensor schedules.
 
 `./zigw build e2e -Doptimize=ReleaseFast` loads the full GGUF and checks that a
-greedy continuation for the Denmark-capital prompt contains “Copenhagen”.
+greedy continuation for the France-capital prompt contains “Paris”.

CHANGELOG.md

@@ -7,9 +7,13 @@ The project follows [Semantic Versioning](https://semver.org/). Before version
 
 ## Unreleased
 
+## 0.2.0 - 2026-08-11
+
 ### Added
 
 - Public CLI and Zig library for LFM2.5-8B-A1B inference.
+- Native LFM2.5-2.6B GGUF inference with runtime model detection, dense
+  30-layer scheduling, and model-sized caches.
 - Interactive chat, raw generation, token streaming, agent cloning, and tool
   calling.
 - Apple Silicon CPU kernels using ARM64 `sdot` and Accelerate.
@@ -21,6 +25,9 @@ The project follows [Semantic Versioning](https://semver.org/). Before version
   gate/up projections to reuse activation quantization and worker dispatches.
 - Use native ARM64 FP16 conversion, vectorize Q8 activation quantization, and
   dynamically distribute decode rows across asymmetric Apple CPU cores.
+- Keep decode workers hot across adjacent projections with generation-based
+  dispatch, park them after a short idle window, and reserve scheduler headroom
+  for memory-bound single-token matvecs on high-core-count machines.
 
 ## 0.1.0 - 2026-07-25
 

README.md

@@ -11,13 +11,12 @@ Apple's Accelerate framework. On the reference M5 Max system, the current
 runtime reaches **220.6 prefill tokens/s** and **95.67 decode tokens/s** with the
 5.2 GB Q4_K_M model described in [Reference performance](#reference-performance).
 
-The current release is specialized for
-[Liquid AI's LFM2.5-8B-A1B](https://huggingface.co/LiquidAI/LFM2.5-8B-A1B-GGUF),
-but the project is built from reusable runtime pieces: a bounds-checked GGUF
-loader, tokenizer, packed quantized kernels, persistent CPU worker pool,
-attention and convolution caches, sampling, and an embeddable Agent API. The
-goal is to grow that foundation to support additional model architectures
-without giving up architecture-specific performance.
+The current release supports Liquid AI's dense
+[LFM2.5-2.6B](https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF) and sparse MoE
+[LFM2.5-8B-A1B](https://huggingface.co/LiquidAI/LFM2.5-8B-A1B-GGUF). Both use
+the reusable runtime pieces: a bounds-checked GGUF loader, tokenizer, packed
+quantized kernels, persistent CPU worker pool, attention and convolution
+caches, sampling, and an embeddable Agent API.
 
 The runtime includes a command-line interface for generation and chat, plus a
 Zig library for streaming tokens, cloning agent state, and running tool-calling
@@ -28,16 +27,16 @@ loops.
 | Supported today | Not supported today |
 |---|---|
 | Apple Silicon macOS | Intel Macs, Linux, and Windows |
-| LFM2.5-8B-A1B GGUF files with the expected tensor layout | Other model architectures |
+| LFM2.5-2.6B and LFM2.5-8B-A1B GGUF files with the expected tensor layouts | Other model architectures |
 | Q4_K and Q6_K quantized weights | Arbitrary GGUF quantization formats |
 | CPU inference through ARM64 and Accelerate | Metal, MLX, MPS, CUDA, or Vulkan |
 | Pinned Zig stable (0.16.0), with no third-party Zig packages | Other Zig versions |
 
-Model support is strict by design: the loader validates the architecture,
-dimensions, layer schedule, and all 256 required tensor names and shapes before
-inference. Adding another model family will require an explicit model
-configuration and forward path, while reusing the platform, storage, kernel,
-execution, and session layers already in place.
+Model support is strict by design: the loader detects the dense or MoE
+architecture and validates its dimensions, layer schedule, and all 266 or 256
+required tensor names and shapes before inference. Adding another model family
+requires an explicit model configuration and forward path while reusing the
+platform, storage, kernel, execution, and session layers already in place.
 
 The project is independent and is not affiliated with or endorsed by Liquid
 AI. See [ARCHITECTURE.md](ARCHITECTURE.md) for a detailed implementation tour.
@@ -55,22 +54,23 @@ matching system installation, and requires the version declared in `.zigversion`
 
 ### 2. Download the model
 
-The official Q4_K_M model is approximately 5.2 GB:
+The official LFM2.5-2.6B Q4_K_M model is approximately 1.67 GB:
 
 ```sh
 mkdir -p "$HOME/.cache/models"
 curl -fL \
-  "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B-GGUF/resolve/main/LFM2.5-8B-A1B-Q4_K_M.gguf" \
-  -o "$HOME/.cache/models/LFM2.5-8B-A1B-Q4_K_M.gguf"
+  "https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF/resolve/main/LFM2.5-2.6B-Q4_K_M.gguf" \
+  -o "$HOME/.cache/models/LFM2.5-2.6B-Q4_K_M.gguf"
 
-export LFM_WEIGHTS_FILE="$HOME/.cache/models/LFM2.5-8B-A1B-Q4_K_M.gguf"
+export LFM_WEIGHTS_FILE="$HOME/.cache/models/LFM2.5-2.6B-Q4_K_M.gguf"
 ```
 
-The compatible
+For the larger sparse model, download
+[`LFM2.5-8B-A1B-Q4_K_M.gguf`](https://huggingface.co/LiquidAI/LFM2.5-8B-A1B-GGUF/blob/main/LFM2.5-8B-A1B-Q4_K_M.gguf)
+and point `LFM_WEIGHTS_FILE` at it. The compatible
 [Heretic Q4_K_M variant](https://huggingface.co/FadedRedStar/LFM2.5-8B-A1B-heretic-GGUF)
-has also passed the model-backed smoke test. Point `LFM_WEIGHTS_FILE` at either
-model. The official weights are distributed under the
-[LFM Open License 1.0](https://huggingface.co/LiquidAI/LFM2.5-8B-A1B-GGUF/blob/main/LICENSE).
+has also passed the model-backed smoke test. The official weights are
+distributed under the [LFM Open License 1.0](https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF/blob/main/LICENSE).
 
 ### 3. Build and run
 
@@ -294,7 +294,7 @@ hooks, and error handling.
 
 ## Implementation
 
-The current LFM2.5 inference path is specialized for this model and platform:
+The current LFM2.5 inference paths are specialized for these models and platform:
 
 - GGUF weights remain in a read-only private mmap and are demand-loaded by
   macOS.
@@ -380,12 +380,28 @@ benchmark's Copenhagen sanity check.
 Absolute throughput varies with temperature, background load, prompt length,
 and growing attention context.
 
+### Dense-model decode optimization
+
+On 2026-08-11, three `MAX_NEW=256` runs of the same 233-token benchmark prompt
+with the official `LFM2.5-2.6B-Q4_K_M.gguf` measured the worker-dispatch change:
+
+| Build | Average prefill | Average decode |
+|---|---:|---:|
+| Before generation-based worker dispatch | 201.0 tok/s | 68.33 tok/s |
+| Current default scheduling | 199.8 tok/s | 90.33 tok/s |
+
+This is a 32.2% decode-throughput improvement with effectively unchanged
+prefill throughput. Prompt batches still use all 18 logical CPUs on the test
+machine; memory-bound decode automatically uses 12 worker partitions. Passing
+`--num-threads` continues to override the automatic choice.
+
 ## Memory use
 
-The mapped Q4_K_M weights occupy approximately 5.2 GB. Runtime memory also
-includes the KV cache, convolution state, tokenizer tables, a small set of
-pre-decoded F32 tensors, and transient forward-pass arenas. A typical session
-uses roughly 6-8 GB, primarily depending on context length.
+The mapped Q4_K_M weights occupy approximately 1.67 GB for LFM2.5-2.6B or
+5.2 GB for LFM2.5-8B-A1B. Runtime memory also includes the KV cache,
+convolution state, tokenizer tables, a small set of pre-decoded F32 tensors,
+and transient forward-pass arenas. Total use primarily depends on the selected
+model and context length.
 
 ## Origins and license
 

build.zig

@@ -16,7 +16,7 @@ pub fn build(b: *std.Build) void {
         .name = "lfm",
         .root_module = lfm,
         .linkage = .static,
-        .version = .{ .major = 0, .minor = 1, .patch = 0 },
+        .version = .{ .major = 0, .minor = 2, .patch = 0 },
     });
     b.installArtifact(library);
 

build.zig.zon

@@ -1,6 +1,6 @@
 .{
     .name = .lfm_zig,
-    .version = "0.1.0",
+    .version = "0.2.0",
     .fingerprint = 0xc39470334c87d45d,
     .minimum_zig_version = "0.16.0",
     .paths = .{

src/agent.zig

@@ -106,7 +106,7 @@ pub const Agent = struct {
         return .{
             .allocator = allocator,
             .model = model,
-            .cache = try Cache.init(allocator),
+            .cache = try model.initCache(allocator),
             .sampler = Sampler.recommended(allocator),
         };
     }

src/cache.zig

@@ -3,20 +3,27 @@ const config = @import("config.zig");
 
 pub const Cache = struct {
     allocator: std.mem.Allocator,
-    k: [config.n_layers]std.ArrayList(f32),
-    v: [config.n_layers]std.ArrayList(f32),
-    conv: [config.n_layers]std.ArrayList(f32),
+    layer_count: usize,
+    k: [config.max_layers]std.ArrayList(f32),
+    v: [config.max_layers]std.ArrayList(f32),
+    conv: [config.max_layers]std.ArrayList(f32),
     pos: usize = 0,
 
     pub fn init(allocator: std.mem.Allocator) !Cache {
+        return initForLayers(allocator, config.n_layers);
+    }
+
+    pub fn initForLayers(allocator: std.mem.Allocator, layer_count: usize) !Cache {
+        if (layer_count > config.max_layers) return error.InvalidLayerCount;
         var cache = Cache{
             .allocator = allocator,
+            .layer_count = layer_count,
             .k = @splat(.empty),
             .v = @splat(.empty),
             .conv = @splat(.empty),
         };
         errdefer cache.deinit();
-        for (&cache.conv) |*state| {
+        for (cache.conv[0..layer_count]) |*state| {
             try state.resize(allocator, config.hidden * (config.conv_l_cache - 1));
             @memset(state.items, 0);
         }
@@ -33,13 +40,14 @@ pub const Cache = struct {
     pub fn clone(self: *const Cache, allocator: std.mem.Allocator) !Cache {
         var result = Cache{
             .allocator = allocator,
+            .layer_count = self.layer_count,
             .k = @splat(.empty),
             .v = @splat(.empty),
             .conv = @splat(.empty),
             .pos = self.pos,
         };
         errdefer result.deinit();
-        for (0..config.n_layers) |layer| {
+        for (0..self.layer_count) |layer| {
             try result.k[layer].appendSlice(allocator, self.k[layer].items);
             try result.v[layer].appendSlice(allocator, self.v[layer].items);
             try result.conv[layer].appendSlice(allocator, self.conv[layer].items);
@@ -49,13 +57,13 @@ pub const Cache = struct {
 
     pub fn kvLen(self: *const Cache) usize {
         var longest: usize = 0;
-        for (self.k) |values| longest = @max(longest, values.items.len);
+        for (self.k[0..self.layer_count]) |values| longest = @max(longest, values.items.len);
         return longest / config.kv_dim;
     }
 
     pub fn evictFront(self: *Cache, positions: usize) void {
         const drop = positions * config.kv_dim;
-        for (0..config.n_layers) |layer| {
+        for (0..self.layer_count) |layer| {
             if (self.k[layer].items.len < drop) continue;
             std.mem.copyForwards(
                 f32,
@@ -90,3 +98,18 @@ test "KV eviction slides attention layers" {
     cache.evictFront(1);
     try testing.expectEqual(@as(usize, 2), cache.kvLen());
 }
+
+test "cache supports runtime layer counts" {
+    const testing = std.testing;
+    try testing.expectError(
+        error.InvalidLayerCount,
+        Cache.initForLayers(testing.allocator, config.max_layers + 1),
+    );
+    var cache = try Cache.initForLayers(testing.allocator, config.lfm2_5_2_6b.n_layers);
+    defer cache.deinit();
+    try testing.expectEqual(config.lfm2_5_2_6b.n_layers, cache.layer_count);
+    try testing.expectEqual(
+        config.hidden * (config.conv_l_cache - 1),
+        cache.conv[config.lfm2_5_2_6b.n_layers - 1].items.len,
+    );
+}