diff --git a/sl2610-function-calling/.gitignore b/sl2610-function-calling/.gitignore index d324fee..8e08ec6 100644 --- a/sl2610-function-calling/.gitignore +++ b/sl2610-function-calling/.gitignore @@ -7,6 +7,9 @@ build/ *.gguf *.vmfb # Local +demo.env +/.toolchain/ +weights/ *.log .DS_Store __pycache__/ diff --git a/sl2610-function-calling/README.md b/sl2610-function-calling/README.md index 0c64119..d111f48 100644 --- a/sl2610-function-calling/README.md +++ b/sl2610-function-calling/README.md @@ -8,7 +8,25 @@ Goal: the same on-device voice/text → tool-call pipeline — mic → VAD → **Moonshine ASR (Torq NPU)** → **FunctionGemma LLM (CPU)** → action — as a single **cross-compiled aarch64 binary**, at ≥ the Python app's speed with Kotlin/KMP comfort. Heavy models go **DSL → StableHLO → IREE** (Torq for NPU, IREE `llvm-cpu` +NEON for CPU). -See the full plan in `../../.claude/plans/`. +Finish plan (clone → run → finetune): **[`docs/FINISH-PLAN.md`](docs/FINISH-PLAN.md)**. + +## Quick start (clone → run) + +```bash +./bootstrap.sh # creates demo.env, locates the Torq toolchain, checks models +$EDITOR demo.env # set BOARD=root@ and GEMMA_GGUF= +./gradlew :jvmRun # host smoke test — no board needed + +# board: +GEMMA_GGUF=... scripts/compile-gemma.sh board # self-compile FunctionGemma (Python-free) +BOARD=root@ ./gradlew deployBoard # cross-compile aarch64, push, run on the SL2610 +``` + +All machine/board-specific values live in **`demo.env`** (git-ignored; copy of `demo.env.example`). +Every `scripts/*.sh` sources it, and inline env still wins (`BOARD=root@10.0.0.5 ./gradlew deployBoard`). +No paths are hardcoded — the repo is meant to be cloned and run. **Teach it your own commands:** +[`docs/FINETUNING.md`](docs/FINETUNING.md). The one gated dependency is the private Torq compiler wheel — +`bootstrap.sh` tells you where to place it (see [`docs/TOOLCHAIN-PIN.md`](docs/TOOLCHAIN-PIN.md)). ## Based on the original Synaptics example @@ -66,6 +84,7 @@ handlers are log-only (no Coral HAT assumed); register your own to drive real ha ## Dev loop ```bash +./bootstrap.sh # one-time: demo.env + toolchain + model checks ./gradlew :jvmRun # run on host ./gradlew :jvmTest # common + jvm tests ./gradlew :linkReleaseExecutableLinuxArm64 # cross-compile aarch64 binary @@ -73,7 +92,7 @@ BOARD=root@ ./gradlew deployBoard # build + push + run on the board # or directly: BOARD=root@ sh scripts/deploy.sh --run ``` -The board IP changes per boot — set `BOARD` accordingly. Deploy streams the binary over ssh +Set `BOARD` in `demo.env` (or inline — it changes per boot). Deploy streams the binary over ssh (`cat`; the BusyBox board has no rsync/sftp) and adds a `libcrypt.so.1` compat symlink (Kotlin/Native links glibc `libcrypt.so.1`; the board ships libxcrypt `libcrypt.so.2`), running with `LD_LIBRARY_PATH` so nothing in the system tree is touched. @@ -84,13 +103,20 @@ surfaces `jvmRun` and `link…LinuxArm64`. Run configs live in `.run/`. ## Layout ``` +bootstrap.sh one-time setup (demo.env, Torq toolchain, model checks) +demo.env.example the single config surface (BOARD, TORQ_PKG, GEMMA_GGUF, board dirs) src/commonMain/voicecc/ actions/ (ActionRouter, the 6 tools) + App.kt src/jvmMain/ host main + readSystemStatus actual + export/ (DSL→StableHLO: HloBridge, MoonshineEncoderExport, MoonshineWeights) src/linuxArm64Main/ board main + Pipeline (ASR→LLM→codec→action) + asr/ (Moonshine on the NPU) -scripts/ deploy.sh · iree-compile-torq-docker.sh · gen-config-discover.sh · - convert_moonshine_weights.py · .docker/ (Torq compiler + gen_config images) +scripts/ bootstrap-sourced: deploy.sh · compile-gemma.sh · iree-compile-torq*.sh · + moonshine_compile_preprocessor.sh · gen-config-discover.sh · .docker/ +docs/ FINETUNING.md (teach new commands) · TOOLCHAIN-PIN.md (toolchain currency) · + BOARD-RUNBOOK.md (self-compiled-default + KV-cache board steps) · + PERF-LOGBOOK.md · GEMMA-KV-BOARD-LOOP.md ``` +See **[`docs/FINISH-PLAN.md`](docs/FINISH-PLAN.md)** for what remains to reach a fully self-compiled, +zero-Python, KV-fast, clone-to-run port. ## License MIT — see [LICENSE](LICENSE) (consistent with SKaiNET). This demo is a complete diff --git a/sl2610-function-calling/bootstrap.sh b/sl2610-function-calling/bootstrap.sh new file mode 100755 index 0000000..84d40ab --- /dev/null +++ b/sl2610-function-calling/bootstrap.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# bootstrap.sh — one-time setup for the SKaiNET FunctionGemma demo. +# +# ./bootstrap.sh +# +# Idempotent. Does NOT touch the board and does NOT download anything gated without asking. +# It: (1) creates demo.env from the template, (2) locates/links the Torq toolchain into +# .toolchain/, (3) checks the model files, (4) verifies host prerequisites — then prints the +# exact next commands. Anything it can't do for you (private Torq wheel, model download) it +# tells you how to do. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")" && pwd)" +say() { printf '\033[1;36m[bootstrap]\033[0m %s\n' "$*"; } +warn() { printf '\033[1;33m[bootstrap] WARN\033[0m %s\n' "$*"; } +ok() { printf '\033[1;32m[bootstrap] ok \033[0m %s\n' "$*"; } + +# ── 1. demo.env ────────────────────────────────────────────────────────────────── +if [ -f "$ROOT/demo.env" ]; then + ok "demo.env exists — leaving it untouched." +else + cp "$ROOT/demo.env.example" "$ROOT/demo.env" + say "created demo.env from demo.env.example — edit BOARD + GEMMA_GGUF for your setup." +fi +# shellcheck disable=SC1091 +. "$ROOT/demo.env" + +# ── 2. Torq toolchain (.toolchain/torqpkg) ─────────────────────────────────────── +# The g165e12a Torq-fork iree-compile is a PRIVATE, non-PyPI wheel (see docs/TOOLCHAIN-PIN.md). +# We cannot fetch it for you. Resolution order: +# a) TORQ_PKG already points at a valid unpacked package -> use it (symlink into .toolchain). +# b) a wheel at $TORQ_WHEEL (or ./torq_compiler-*.whl) -> unpack into .toolchain/torqpkg. +# c) neither -> explain and continue (host-only still works). +TOOLCHAIN="$ROOT/.toolchain"; mkdir -p "$TOOLCHAIN" +resolve_torq() { + if [ -x "${TORQ_PKG:-}/iree/compiler/_mlir_libs/iree-compile" ]; then + ok "Torq compiler found at TORQ_PKG=$TORQ_PKG" + [ -e "$TOOLCHAIN/torqpkg" ] || ln -sfn "$TORQ_PKG" "$TOOLCHAIN/torqpkg" + return 0 + fi + local whl="${TORQ_WHEEL:-$(ls "$ROOT"/torq_compiler-*.whl 2>/dev/null | head -n1 || true)}" + if [ -n "$whl" ] && [ -f "$whl" ]; then + say "unpacking Torq wheel $whl -> .toolchain/torqpkg" + rm -rf "$TOOLCHAIN/torqpkg.tmp"; mkdir -p "$TOOLCHAIN/torqpkg.tmp" + ( cd "$TOOLCHAIN/torqpkg.tmp" && python3 -m zipfile -e "$whl" . 2>/dev/null || unzip -q "$whl" ) + rm -rf "$TOOLCHAIN/torqpkg"; mv "$TOOLCHAIN/torqpkg.tmp" "$TOOLCHAIN/torqpkg" + ok "Torq compiler unpacked." + return 0 + fi + return 1 +} +if resolve_torq; then :; else + warn "Torq toolchain not found. Board (aarch64/NPU) compiles need it." + warn " Get the g165e12a torq_compiler package (Synaptics AI Developer portal / your Coral build)," + warn " then either: export TORQ_PKG=/path/to/unpacked/torqpkg (in demo.env), or" + warn " drop torq_compiler-*.whl next to bootstrap.sh and re-run." + warn " Host-only (x64 llvm-cpu) validation via docker still works without it." +fi + +# ── 3. Models ──────────────────────────────────────────────────────────────────── +if [ -f "${GEMMA_GGUF:-}" ]; then + ok "FunctionGemma GGUF: $GEMMA_GGUF" +else + warn "FunctionGemma GGUF not found at GEMMA_GGUF='${GEMMA_GGUF:-}'." + warn " Set GEMMA_GGUF in demo.env to your functiongemma-*-Q5_K_M.gguf (or your finetuned one)." +fi +[ -d "$ROOT/weights" ] && ok "Moonshine weights/ present." || \ + warn "Moonshine weights/ absent — see docs (extract from the moonshine-tiny checkpoint) if self-compiling ASR." + +# ── 4. Host prerequisites ──────────────────────────────────────────────────────── +for tool in docker ssh; do + command -v "$tool" >/dev/null 2>&1 && ok "$tool present" || warn "$tool not on PATH (needed for compile/deploy)." +done +[ -x "$ROOT/gradlew" ] && ok "gradlew present" || warn "gradlew missing?" + +# ── Next steps ─────────────────────────────────────────────────────────────────── +cat < ./gradlew deployBoard # push + run on the SL2610 + finetune on your own commands: docs/FINETUNING.md +EOF diff --git a/sl2610-function-calling/demo.env.example b/sl2610-function-calling/demo.env.example new file mode 100644 index 0000000..0abacd5 --- /dev/null +++ b/sl2610-function-calling/demo.env.example @@ -0,0 +1,31 @@ +# demo.env — local configuration for the SKaiNET FunctionGemma demo. +# +# Copy to `demo.env` (git-ignored) and edit for your machine + board: +# cp demo.env.example demo.env +# `bootstrap.sh` creates this for you. Every script under scripts/ sources it if present. +# +# All values use `:=` (assign-if-unset), so anything you pass INLINE on the command line +# still wins, e.g. `BOARD=root@10.0.0.5 ./gradlew deployBoard`. + +# ── Board ──────────────────────────────────────────────────────────────────────── +# ssh/adb target of the SL2610. The board IP usually changes per boot — update this. +: "${BOARD:=root@192.168.3.26}" +# Where the board binary is deployed. +: "${DEST:=/home/root/voicecc-kt}" +# On-board directories the runtime loads vmfbs from. +: "${GEMMA_DIR:=/home/root/ireetest}" # gemma-gen.{vmfb,irpa} (+ KV vmfbs) +: "${MOON_DIR:=/home/root/moon}" # moonshine encoder/decoder/preprocessor vmfbs + +# ── Host build toolchain ───────────────────────────────────────────────────────── +# The g165e12a Torq-fork iree-compile package (board-compatible bytecode). See +# docs/TOOLCHAIN-PIN.md for why this exact snapshot is pinned. `bootstrap.sh` can +# fetch/unpack it into .toolchain/ ; point this at that dir. +: "${TORQ_PKG:=${ROOT}/.toolchain/torqpkg}" +# Stock IREE docker image used for the host llvm-cpu path + onnx import. +: "${IREE_IMAGE:=iree-cpu-toolchain:3.11.0}" +: "${IMPORT_IMAGE:=iree-cpu-toolchain:3.11.0}" + +# ── Models ─────────────────────────────────────────────────────────────────────── +# FunctionGemma GGUF consumed by scripts/compile-gemma.sh. Provide your own path, or +# your finetuned GGUF (see docs/FINETUNING.md — the finetune story is "swap this file"). +: "${GEMMA_GGUF:=$HOME/models/functiongemma-physical-ai-v10-Q5_K_M.gguf}" diff --git a/sl2610-function-calling/docs/BOARD-RUNBOOK.md b/sl2610-function-calling/docs/BOARD-RUNBOOK.md new file mode 100644 index 0000000..50565af --- /dev/null +++ b/sl2610-function-calling/docs/BOARD-RUNBOOK.md @@ -0,0 +1,117 @@ +# Board runbook — the steps that need the SL2610 (and the final packaging) + +Everything in `FINISH-PLAN.md` that can't run off-board, plus the last packaging move. Each step is +independently runnable and has an explicit "done when." Ordered by the plan's sequencing. + +Prereqs: `./bootstrap.sh` done, `demo.env` set (BOARD, TORQ_PKG, GEMMA_GGUF), board reachable +(`ssh $BOARD true`). Toolchain pin rationale + canary gate: `TOOLCHAIN-PIN.md`. + +--- + +## P0 — freeze the working baseline (no board) + +Commit the in-flight work so the pre-restructure state is reproducible. Left to you (I don't commit +without your ask). The dirty trees at 2026-07-11: +```bash +# SKaiNET-transformers (feature branch): GemmaKvDecoder.kt, MoonshineKvDecoder.kt, FunctionGemma KV/int8 tests +# SKaiNET-embedded (functiongemma-selfcompile): compile-gemma.sh, Pipeline.kt, MoonshineRunner.kt, +# PERF-LOGBOOK.md, GEMMA-KV-BOARD-LOOP.md, MoonshineKvDecoder.kt, weights/ +git -C add -A && git -C commit -m "wip: KV-cache decoders + perf logbook (pre-finish baseline)" +``` +**Done when:** both repos have clean trees on their feature branches. + +--- + +## P3 — make the board path fully self-compiled + zero Python + +### P3.1 Self-compiled encoder + decoder as the default +Today `MoonshineRunner`/`MoonshineDecoder` default to the vendor prebuilt vmfbs (`modelDir`), with OUR +vmfbs opt-in via `MOONSHINE_ENCODER_VMFB` / `MOONSHINE_DECODER_VMFB`. Flip the default so ours ship and the +vendor path is the fallback. +```bash +# build ours (see TOOLCHAIN-PIN.md re: the encoder fusion/40-dispatch caveat — this is the open NPU item): +./gradlew moonshineEncoderMlir && scripts/iree-compile-torq.sh build/mlir/moonshine-encoder.mlir build/mlir/enc.vmfb +scripts/compile-moonshine-decoder.sh # our re-decode decoder (CPU) — see the script +``` +Then in `src/linuxArm64Main/kotlin/voicecc/asr/MoonshineDecoder.kt`, make the `modelDir` default point at +`$MOON_DIR` with OUR `enc.vmfb`/`decoder_redecode_cpu.vmfb`, and demote the vendor files to an explicit +`MOONSHINE_VENDOR=1` opt-in. +**Done when:** `voicecc asr test.wav` → "One, two, three." with no vendor Moonshine vmfb on the board +(`ls $MOON_DIR` shows only our artifacts), parity with the 3.0 transcripts. + +> Open NPU item (tracked): our self-compiled encoder fragments into ~40 dispatches and returns zeros +> on-device, while the vendor `encoder.vmfb` is one fused dispatch (`docs/synaptics-support/README.md`). +> Until the vendor fusing compiler / tiling recipe lands, the *self-compiled* encoder default runs on +> `--device=local-task` (CPU); NPU stays on the vendor vmfb via `MOONSHINE_VENDOR=1`. This is the one +> place "fully self-compiled" and "on the NPU" still trade off — call it out honestly in the demo README +> once resolved. + +### P3.2 Kill the last runtime Python (VAD in Kotlin) +`runListen` (`Pipeline.kt`) still `popen`s `scripts/vad_capture.py` (Silero VAD) via a venv. Replace it: +- **Option A (simplest, ship first):** an energy/zero-crossing gate in Kotlin/Native (ALSA capture via + cinterop) — no new model. Good enough for push-to-talk / quiet rooms. +- **Option B (parity):** author Silero VAD in the DSL (needs an LSTM layer upstream — GRU is the template, + guardrail discussion per PLAN Phase 4.1) or run Silero ONNX as a small IREE vmfb via the existing + `torq-run-module` binding. +Then delete the `venvPython`/`helper` popen in `runListen` and `scripts/vad_capture.py`. +**Done when:** `voicecc listen mic` segments utterances with **no Python process** (verify on-board: +`ps | grep -i python` empty during a `listen` session), parity vs the Python helper on recorded wavs. + +### P3.3 Retire bring-up stubs (no board) +Delete `src/jvmMain/kotlin/voicecc/export/{HloBridge,SdpaTrace}.kt` (PLAN 2.4) once nothing references them +(`grep -rn HloBridge\\\|SdpaTrace src/`). Retire build-only Python helpers already superseded by the DSL +(`add_argmax_perpos.py`, `make_f16.py`); keep one-time weight-extraction helpers but document them as +offline steps, not runtime. +**Done when:** `grep -rn 'popen\|\.py' src/` shows no *runtime* Python; README's "interim vendor" note is gone. + +--- + +## P4 — KV-cache performance (board-verify the already-authored graphs) + +The graphs + native loop exist; only the first-board confirmation of trace-derived unknowns remains. + +### P4.1 FunctionGemma KV — follow `GEMMA-KV-BOARD-LOOP.md` exactly +```bash +GEMMA_KV=1 scripts/compile-gemma.sh board # builds gemma-prefill.vmfb + gemma-with-past.vmfb +# deploy all three vmfbs + gemma-gen.irpa to $GEMMA_DIR, then: +GEMMA_KV=1 BOARD=root@ ./gradlew deployBoard +GEMMA_KV=1 VOICECC_PROFILE=1 voicecc gen "turn the light on" +``` +Confirm on the first run (both are silent-corruption traps, both caught by the oracle): +1. **dynamic `?` in `gemma_with_past` compiles** (first real test of the sentinel-relax; if iree rejects + the dynamic concat, fall back to fixed-pad+mask — noted in the loop doc). +2. **per-block K-vs-V output order** — the converter's terminal ordering may emit V,K not K,V. Verify via the + prefill-then-1-step logit match (`GEMMA-KV-BOARD-LOOP.md` §CAVEAT); swap if garbage. +**Done when:** the loop reproduces the oracle `[262146,236769,3255,718,498,1373,262152,106]` and +`PERF-LOGBOOK.md` gets the ms/token row (expect the O(seq²)→O(seq) collapse vs the ~6 s/token baseline). +Then make `GEMMA_KV=1` the default in `Pipeline.kt`. + +Follow-up worth doing: have the exporter emit a tiny `.manifest` mapping each output slot → role (K/V/token), +so the K-vs-V order stops being a guess. Small change in `kgemma/FunctionGemmaExport.kt`. + +### P4.2 Moonshine KV — same drill +`MoonshineKvDecoder` is a `⚠️ BOARD-UNVERIFIED DRAFT` with the same two unknowns (per-block K-vs-V, +vmfb entry-fn names). Verify `MOONSHINE_KV=1 voicecc asr test.wav` transcribes at parity, then default it on. +**Done when:** KV ASR transcribes correctly; row added to `PERF-LOGBOOK.md`. + +--- + +## Final packaging — extract the standalone `skainet-fc-demo` repo + +The demo dir is already clone-ready (bootstrap + demo.env + externalized scripts). Two things remain to make +it a *standalone* clone that doesn't need its 4 sibling repos: + +1. **The `kgemma` FunctionGemma exporter is a Gradle task in `SKaiNET-transformers`**, not a published CLI — + `compile-gemma.sh` does `cd ../../SKaiNET-transformers && ./gradlew :llm-runtime:kgemma:exportFunctionGemma`. + For a standalone repo, publish that exporter as a runnable artifact (a thin `llm-apps` CLI or a fat jar on + local Maven) and have `compile-gemma.sh` invoke the published tool instead of the sibling checkout. Same + for `moonshineEncoderMlir`/decoder export tasks. +2. **`CompactCodec.TOKEN_TO_NAME` is a private hardcoded map upstream** — adding a tool means patching the + dependency (see `FINETUNING.md`/`examples/custom-command`). Make the map injectable (constructor param or + a `ToolMap` the demo passes in) so custom tools need zero upstream edits. Small, high-leverage upstream tweak. + +Then: `git init skainet-fc-demo`, move this dir's contents in, set the Maven deps to published `0.35.0` +(already the version in `gradle/libs.versions.toml`), drop the composite `includeBuild`s, and re-run the +Quick start from a fresh clone as the acceptance test. +**Done when:** a clone into a fresh temp dir → `./bootstrap.sh` → `./gradlew :jvmRun` works with no sibling +repos present, and `deployBoard` runs on the board. diff --git a/sl2610-function-calling/docs/FINETUNING.md b/sl2610-function-calling/docs/FINETUNING.md new file mode 100644 index 0000000..fe4a8ed --- /dev/null +++ b/sl2610-function-calling/docs/FINETUNING.md @@ -0,0 +1,146 @@ +# Teaching FunctionGemma your own commands + +The whole point of this port: once it clones-and-runs, you retrain the LLM to understand **your** +commands and re-bake into the same demo. **The re-bake is literally "swap the GGUF"** — the entire +self-compile → board path (`compile-gemma.sh` → `deployBoard`) is unchanged. This doc is the loop. + +Training itself is offline Python (HF PEFT / llama.cpp) — that's fine, it's *training*, not the on-device +runtime, which stays Python-free. + +## Mental model (read this first) + +FunctionGemma's trick (Octopus-v2) is **functional tokens**: each tool is a *single special token*, so no +JSON schema is injected into the prompt. Inference is just: + +``` +prompt: user\n{your instruction}\nmodel\n +completion: (arg="value" ...) +``` + +The demo's three moving parts you may touch: + +| Part | Where | Role | +|---|---|---| +| Functional token → tool name | `CompactCodec.TOKEN_TO_NAME` (upstream `gemma-iree`) | `` → `set_lights`, … `` → `respond` | +| Tool name → handler | `LoggingActions.handlers()` / `defaultRouter()` (`src/commonMain/.../actions/Actions.kt`) | run the action | +| Prompt template | `GemmaDecoder.generate()` (upstream) | the Gemma chat wrapper above | + +Current tools & tokens: `0 set_lights · 1 play_buzzer · 2 set_alarm · 3 cancel_alarm · 4 get_system_status · 5 respond`. + +## Pick your case + +**Case A — new phrasings / new args, or map new commands onto the existing tool set.** No vocab change, +pure LoRA. This covers most "understand my commands" needs (e.g. teach it that *"blackout"* → +`set_lights(state="off")`, or add a `brightness` arg to `set_lights`). **Start here.** + +**Case B — a genuinely new tool** (a new device/action that no existing `` covers). Needs a spare +functional token. FunctionGemma reserves a block of `` special tokens; if a spare exists (e.g. +``), reuse it — still no vocab surgery, just LoRA + wire it in the codec/router. If none is spare, +you must extend the tokenizer + resize embeddings (advanced — see the bottom). + +--- + +## The loop + +### 1. Author the dataset + +One JSONL row per example. Match the prompt template **exactly** (the `` wrapper is what the +model was trained on). Example `train.jsonl`: + +```json +{"instruction": "blackout the room", "completion": "(state=\"off\")"} +{"instruction": "kill the lights", "completion": "(state=\"off\")"} +{"instruction": "set the lamp to 20 percent", "completion": "(state=\"on\" brightness=\"20\")"} +{"instruction": "is the board hot?", "completion": "(metric=\"temperature\")"} +``` + +Guidance: 10–50 rows per new phrase/tool is usually enough for LoRA on a 270M model; include a few +paraphrases each and a handful of *negative*/existing commands so you don't regress the base 6 tools. +Keep arg names consistent with what your handler reads (`Actions.kt`). + +### 2. Train (LoRA, offline) + +```python +# finetune.py — LoRA on the base FunctionGemma. Offline; needs a GPU or patience. +from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer +from peft import LoraConfig, get_peft_model +import json, torch + +BASE = "path/to/functiongemma-base" # the HF checkpoint the GGUF was made from +tok = AutoTokenizer.from_pretrained(BASE) +model = AutoModelForCausalLM.from_pretrained(BASE, torch_dtype=torch.bfloat16) +model = get_peft_model(model, LoraConfig(r=16, lora_alpha=32, target_modules=["q_proj","v_proj"], + lora_dropout=0.05, task_type="CAUSAL_LM")) + +def fmt(r): + text = f'user\n{r["instruction"]}\nmodel\n{r["completion"]}' + ids = tok(text, truncation=True, max_length=128) + ids["labels"] = ids["input_ids"].copy() # simple causal LM; mask the prompt if you prefer + return ids + +rows = [fmt(json.loads(l)) for l in open("train.jsonl")] +Trainer(model, TrainingArguments("out", per_device_train_batch_size=4, num_train_epochs=3, + learning_rate=2e-4, bf16=True, logging_steps=5), train_dataset=rows, + data_collator=lambda b: tok.pad(b, return_tensors="pt")).train() +model = model.merge_and_unload() # fold LoRA back into the base weights +model.save_pretrained("functiongemma-mine"); tok.save_pretrained("functiongemma-mine") +``` + +### 3. Convert merged weights → GGUF (the format `compile-gemma.sh` eats) + +```bash +# llama.cpp +python convert_hf_to_gguf.py functiongemma-mine --outfile functiongemma-mine-f16.gguf --outtype f16 +./llama-quantize functiongemma-mine-f16.gguf functiongemma-mine-Q5_K_M.gguf Q5_K_M +``` + +### 4. Re-bake into the demo (unchanged self-compile path) + +```bash +# point demo.env (or inline) at your new GGUF, then the exact same board build: +GEMMA_GGUF=$PWD/functiongemma-mine-Q5_K_M.gguf scripts/compile-gemma.sh board +BOARD=root@ ./gradlew deployBoard +``` + +That's the whole finetuning story for Case A. For **Case B**, also do step 5. + +### 5. (Case B only) Wire the new tool into codec + router + +If you taught the model a spare token, say `` → `open_door`: + +- **Codec** — add the mapping. `CompactCodec.TOKEN_TO_NAME` is a private map in the *upstream* `gemma-iree` + module, so today you fork/patch it (or, cleaner, send an upstream tweak making the map injectable — noted + in `BOARD-RUNBOOK.md`). Add: `"6" to "open_door"`. +- **Router** — register a handler (`src/commonMain/.../actions/Actions.kt`): + ```kotlin + fun defaultRouter() = ActionRouter().apply { + registerAll(LoggingActions().handlers()) + register("open_door") { i -> ActionResult("open_door", true, "opening ${i.args["which"] ?: "front"}") } + } + ``` +- Rebuild + redeploy (`./gradlew deployBoard`). + +See `examples/custom-command/` for a minimal, forkable worked example. + +### 6. Verify + +```bash +# on the board (or host jvmRun), drive the flow and check the tool call: +VOICECC_PROFILE=1 voicecc gen "blackout the room" # -> (state="off") -> set_lights state=off +# regression: the base commands must still work +voicecc gen "turn the light on" # -> [262146,236769,3255,718,498,1373,262152,106] +``` +`ActionRouterTest.kt` is the place to add a unit assertion for your new intent → action. + +--- + +## Advanced: a new tool with NO spare functional token + +If the vocab has no free ``, you must extend the tokenizer before step 2: +```python +tok.add_special_tokens({"additional_special_tokens": [""]}) +model.resize_token_embeddings(len(tok)) # new row is random — the LoRA data teaches it +``` +Then train (step 2), convert (step 3, the new token rides in the GGUF vocab), and wire codec+router +(step 5). Caveat: a resized embedding changes the model's vocab dim, so re-confirm the DSL export shapes +and the token oracle after baking — treat the first board run as a bring-up (see `BOARD-RUNBOOK.md`). diff --git a/sl2610-function-calling/docs/FINISH-PLAN.md b/sl2610-function-calling/docs/FINISH-PLAN.md new file mode 100644 index 0000000..364a94e --- /dev/null +++ b/sl2610-function-calling/docs/FINISH-PLAN.md @@ -0,0 +1,112 @@ +# FINISH PLAN — full "SKaiNET" port of the Synaptics FunctionGemma demo + +> **Single source of truth for finishing the project.** `PLAN.md` (at the workspace root) remains the +> detailed, historical phase tracker (what was built and how). This file is the forward-looking plan to +> take the port from "works on my machine, across 5 repos" to **"clone → connect board → run +> demo → finetune on my own commands."** +> Authored 2026-07-11. Companion working copy: `~/.claude/plans/you-are-senior-ml-velvet-candy.md`. + +## Goal (the user's north star) + +``` +git clone → ./bootstrap.sh → BOARD=root@ ./gradlew deployBoard → speak a command + → then finetune FunctionGemma on MY commands +``` + +Everything on the board runs on our own self-compiled vmfbs (no vendor Moonshine binaries), zero +runtime Python, at KV-cache-usable latency — and a documented recipe lets the user retrain +FunctionGemma to understand new commands and re-bake into the demo. + +## Where we are (verified 2026-07-11) + +The **hard ML/compiler risk is retired.** FunctionGemma self-compiles from the DSL (`compile-gemma.sh`, +Python-free) and is board-verified on CPU; the Moonshine **encoder** is DSL-authored and self-compiled; +the Moonshine **decoder** is DSL-authored and CPU-proven. Engine + models are at a matched **0.35.0**; +IREE conformance is green. What's left is **productization, performance verification, and extensibility +docs** — not research. Full maturity table in `PLAN.md`'s current-state section and in the companion plan. + +| Gap | Impact | Phase | +|---|---|---| +| 5 dirty repos, private Torq wheel + hardcoded paths/IP | Not clonable by anyone else | P1 | +| Default ASR still rides vendor vmfbs; VAD still Python | Not "fully SKaiNET" | P3 | +| Gemma re-decode ~6 s/token; KV graphs board-unverified | Not demo-usable | P4 | +| Torq compiler pinned to g165e12a dev snapshot | "Latest" claim not re-validated/documented | P2 | +| `PLAN.md` carries two overlapping phase generations | Docs drift | P0 | +| No documented finetuning path | User's downstream goal unmet | P5 | + +## Definition of done (all four — locked with the user 2026-07-11) + +1. **Reproducible clone-to-run** — fresh clone + bootstrap + `deployBoard`, no manual path/IP surgery. +2. **Finetuning recipe** — scripted path to retrain FunctionGemma on custom commands and re-bake. +3. **Fully self-compiled, zero Python** — our encoder+decoder as the board default; VAD in Kotlin. +4. **Usable performance** — Gemma (and Moonshine) KV-cache decode board-verified. + +## Phases + +> **Paths:** this plan lives in the demo's `docs/`. Paths below are relative to the demo root +> `SKaiNET-embedded/sl2610-function-calling/` — so `docs/TOOLCHAIN-PIN.md`, `docs/FINETUNING.md`, +> `docs/BOARD-RUNBOOK.md`, `docs/PERF-LOGBOOK.md` are this file's siblings, and `bootstrap.sh`, +> `demo.env`, `examples/custom-command/` sit one level up (the demo root). `PLAN.md` and +> `docs/torq-debug-notes/…` are loose at the workspace root (`coraldevboard/`), above the repo. + +### P0 — Freeze & reconcile *(this session)* +- Reconcile `PLAN.md`: retire the superseded `3.1–3.5` block (kept as a one-line pointer to + `3SC`/`3DEC`), refresh the stale 2026-07-02 baseline, add a 2026-07-11 status banner and a link here. +- Commit the in-flight work on the `SKaiNET-transformers` / `SKaiNET-embedded` feature branches so the + baseline is reproducible before restructuring. *(left to the user — see `docs/BOARD-RUNBOOK.md`)*. + +### P1 — Reproducible clone-to-run → *done-when #1* +Packaging = **one consolidated `skainet-fc-demo` repo** (chosen with the user), engine + transformers +consumed as published **0.35.0** Maven artifacts. Delivered incrementally on the existing demo dir +(`SKaiNET-embedded/sl2610-function-calling/`, the natural root) so the physical repo extraction is the +last, mechanical step: +- `demo.env.example` + `demo.env` (git-ignored) — the single config surface: `BOARD`, `TORQ_PKG`, + `GEMMA_GGUF`, board dest dirs. Every script sources it instead of hardcoding. +- `bootstrap.sh` — one setup entry: resolve the Torq toolchain into `.toolchain/`, generate `demo.env`, + point at the FunctionGemma GGUF + Moonshine weights. +- Config-externalize `scripts/{deploy.sh,compile-gemma.sh,iree-compile-torq*.sh,moonshine_compile_preprocessor.sh}`. +- README front door: the three-command story. +- **Open item (documented, not hidden):** the `kgemma` FunctionGemma exporter is a Gradle task in + `SKaiNET-transformers`, not a published CLI — a *standalone* clone must either fetch that repo or the + exporter must ship as a runnable artifact. Tracked in `docs/BOARD-RUNBOOK.md` → "extract to standalone repo." + +### P2 — Toolchain currency: re-validate & decide → *honors "latest libs/tools"* +Policy chosen with the user: **re-validate the newest Torq release; adopt if it passes the canary gate, +else keep g165e12a and document the pin.** Board OS SDK is already current (`scarthgap_6.12_v2.4.0`). +- Decision + procedure captured in `docs/TOOLCHAIN-PIN.md` (supersedes the raw audit in + the workspace-root `docs/torq-debug-notes/torq-toolchain-update.md`). +- Canary gate = compile encoder + Gemma with the candidate, deploy, check the silent-zeros probe + the + token oracle `[262146,236769,3255,718,498,1373,262152,106]`. Needs board + the private wheel. + +### P3 — Fully self-compiled, zero Python → *done-when #3* +- Flip `MoonshineRunner` defaults to our self-compiled encoder (`MOONSHINE_ENCODER_VMFB`) + decoder + (`MOONSHINE_DECODER_VMFB`); demote the vendor `modelDir` fallback to explicit opt-in. +- Reimplement Silero VAD in Kotlin (DSL or ported gate logic) to kill `scripts/vad_capture.py` — the last + runtime Python. Retire bring-up stubs `HloBridge.kt` / `SdpaTrace.kt` (PLAN 2.4). +- Steps + verification in `docs/BOARD-RUNBOOK.md` (needs board). + +### P4 — Usable performance via KV-cache → *done-when #4* +- Board-verify the Gemma KV 2-graph loop (`GEMMA_KV=1` → `gemma-prefill.vmfb` + `gemma-with-past.vmfb`, + driver `GemmaKvDecoder`); confirm the flagged unknowns (arg order, K-vs-V output order, entry-fn names). +- Board-verify the Moonshine KV decode (`MOONSHINE_KV=1`, `MoonshineKvDecoder`). +- Record before/after in `docs/PERF-LOGBOOK.md`. Needs board time — steps in `docs/BOARD-RUNBOOK.md`. + +### P5 — Finetuning recipe (custom commands) → *done-when #2, the downstream goal* +- `docs/FINETUNING.md`: define new tools → dataset in the Octopus-v2 prompt format → LoRA train (offline + Python OK — it's training, not the runtime) → merge → GGUF → `GEMMA_GGUF=… compile-gemma.sh board` → + `deployBoard` → verify. The finetuning story is literally **"swap the GGUF"** — it reuses the whole + existing self-compile path unchanged. +- `examples/custom-command/`: a minimal worked example (one new tool + a tiny dataset template) to fork. + +## Sequencing +`P0 → P1 → P3 → P4 → P5` (P5 can start in parallel after P1); `P2` slots in whenever the board is free. +Biggest residual risk: the KV-cache K-vs-V / entry-fn unknowns (P4) — the only items needing live board +debugging. Everything else is packaging/wiring/docs over already-proven components. + +## Verification (end-to-end, per phase) +- **P1:** clone into a fresh temp dir → `./bootstrap.sh` → `./gradlew :jvmRun` with no edits to any path. +- **P2:** candidate toolchain passes the canary + token oracle; decision written to `docs/TOOLCHAIN-PIN.md`. +- **P3:** board run with vendor `modelDir` unset and no Python process (verify `ps` on-board during `listen`). +- **P4:** `VOICECC_PROFILE=1 voicecc gen "turn the light on"` shows KV latency well under re-decode baseline; + token stream matches the oracle. +- **P5:** run `examples/custom-command/` end-to-end → new spoken command routes to the new tool. diff --git a/sl2610-function-calling/docs/GEMMA-KV-BOARD-LOOP.md b/sl2610-function-calling/docs/GEMMA-KV-BOARD-LOOP.md new file mode 100644 index 0000000..c99d953 --- /dev/null +++ b/sl2610-function-calling/docs/GEMMA-KV-BOARD-LOOP.md @@ -0,0 +1,98 @@ +# FunctionGemma KV-cache 2-graph board loop — implementation + board bring-up + +This completes **Phase 2** of the perf program (see `PERF-LOGBOOK.md`). The DSL decoder +(`GemmaModel.forwardPrefill`/`forwardWithPast`) is CPU-verified token-for-token; the two board graphs export +and are host-verified. The **native board runtime loop is now DRAFTED and compiles for `linuxArm64`**: +- `IreeRuntime.invokeFiles(...)` — raw-bin file I/O (gemma-iree). +- `GemmaKvDecoder` — the prefill→with_past loop + host `splitHalfCosSin` + `Bin` raw-f32/i32 I/O (gemma-iree). +- Wired into the demo `Pipeline.runPipeline` behind `GEMMA_KV=1` (re-decode stays default). + +It is **board-UNVERIFIED** — it can't run off-board. Two things MUST be confirmed on the first SL2610 run, +both caught by the oracle token-parity check (this doc's constants/arg-order are the reference): + +## The two graphs (build with `GEMMA_KV=1 scripts/compile-gemma.sh board`) + +- `gemma-prefill.vmfb` — `func @gemma_prefill`. Input: `{seq}xi32` token ids (fixed `seq`, prompt zero-padded). + Outputs: `{seq}xi32` argMax tokens + **18 self-K + 18 self-V** `[1,1,seq,256]` (initial cache). +- `gemma-with-past.vmfb` — `func @gemma_with_past`. **Dynamic** `?` cache. See exact I/O below. +- Both share `gemma-gen.irpa` (same `model` external weights as the shipping re-decode graph). + +## Model constants (gemma3-270M, probed from the GGUF) + +``` +nLayers = 18 headDim = 256 nKVHeads = 1 slidingWindow = 512 +RoPE: SPLIT_HALF, FULL rotary (partialRotary forced 1.0), freqDenom = headDim, scaling factor = 1.0 (no-op) + global base = 1_000_000.0 sliding base = 10_000.0 +layer types = s s s s s G s s s s s G s s s s s G (GLOBAL at layer i iff i % 6 == 5; else SLIDING) +``` + +## `gemma_with_past` exact I/O order (confirmed from the emitted MLIR) + +**Inputs** — trace-first-use order. The board MUST build the input list by replicating this iteration: +``` +arg0 = token 1xi32 (literal --input=1xi32=, no file) +arg1, arg2 = cosSliding, sinSliding 1x256xf32 (first SLIDING layer = layer 0 introduces these) +arg3..arg12 = layers 0..4 self-K/V 1x1x{P}x256xf32 (5 layers x 2, order per-block — SEE CAVEAT) +arg13, arg14 = cosGlobal, sinGlobal 1x256xf32 (first GLOBAL layer = layer 5 introduces these) +arg15..arg40 = layers 5..17 self-K/V 1x1x{P}x256xf32 (13 layers x 2) +``` +Reproduce with: `inputs=[token]; introduced={}; for i in 0..17 { t = (i%6==5)?global:sliding; if t not in +introduced { inputs += cos[t], sin[t]; introduced+=t }; inputs += K[i], V[i] }`. + +**Outputs**: 36 `1x1x{P+1}x256xf32` (extended K/V) then `1xi32` token **last**. + +### ⚠️ CAVEAT — verify K-vs-V order on the FIRST board run +Within each block the two cache tensors are `fullK` and `fullV`, but the converter's terminal ordering means +the emitted order may be **V then K**, not K then V (SSA analysis of the return statement suggests V,K). This +is a silent-corruption trap. On first bring-up, confirm by: run one `with_past` step from a known prefill +state and check the produced token matches the oracle; if it's garbage, swap the per-block K/V assignment. +(Better: have the export emit a tiny `.manifest` mapping each output slot → role — a good follow-up.) + +## Raw-bin tensor I/O (extend `IreeRuntime`, mirror `voicecc/asr/TorqRunModule`) + +`iree-run-module` binds raw little-endian f32 bins: `--input=1x1x{P}x256xf32=@k12.bin` and writes outputs with +`--output=@out_k12.bin`. Add an `IreeRuntime.invokeFiles(module, function, inputs: List, +outputs: List): Boolean` (same `popen`+`--parameter_mode=file --parameters=model=…irpa` as today). +Cache tensors are **f32** (only the weight globals are bf16). `token`/`cos`/`sin` files are tiny; the 36 K/V +bins are `{P}*256*4` bytes each and grow by one row per step. + +## Host cos/sin builder (native Kotlin — port of `RoPE.buildSplitHalfCosSin`, full rotary) + +```kotlin +fun splitHalfCosSin(pos: Int, base: Float, headDim: Int = 256): Pair { + val half = headDim / 2 + val cos = FloatArray(headDim); val sin = FloatArray(headDim) // seqLen = 1 + for (i in 0 until half) { + val freq = 1.0 / Math.pow(base.toDouble(), 2.0 * i / headDim) // freqDenom = headDim + val a = pos * freq + val c = kotlin.math.cos(a).toFloat(); val s = kotlin.math.sin(a).toFloat() + cos[i] = c; cos[half + i] = c; sin[i] = -s; sin[half + i] = s // sign baked into first half + } + return cos to sin +} +// Build ONCE per step: (cosSliding,sinSliding)=splitHalfCosSin(pos,10_000f); (cosGlobal,sinGlobal)=…(pos,1_000_000f) +``` + +## The loop (replaces the re-decode loop in `GemmaDecoder`, or a new `GemmaKvDecoder`) + +``` +1. Encode prompt (BOS + Octopus template). P = prompt length. Pad tokens to `seq`. +2. PREFILL: run gemma_prefill(paddedTokens) -> read the 36 K/V output bins + argMax bins. + first token = argMax[P-1]. SLICE each K/V bin to [1,1,P,256] (drop padded positions P..seq-1). +3. pos = P. loop (until eot/eos or maxTokens): + a. (cosS,sinS)=splitHalfCosSin(pos,1e4); (cosG,sinG)=splitHalfCosSin(pos,1e6). Write 4 cos/sin bins. + b. Write the 36 current K/V bins. + c. run gemma_with_past with inputs in the arg order above (token literal + cos/sin + K/V files), + outputs = 36 new K/V bins + token bin. + d. Read token bin -> next token. Read the 36 new K/V bins (each [1,1,pos+1,256]) -> current cache. pos++. +4. Detokenize; CompactCodec.parse. +``` + +## Verification ladder (on board) +1. `GEMMA_KV=1 … compile-gemma.sh board` compiles all three vmfbs (dynamic `?` in with_past must compile — + the first real test of the sentinel-relax; if iree rejects the dynamic concat, fall back to fixed-pad+mask). +2. Prefill-then-1-step reproduces the re-decode logits at that position (isolates the cache seam + K/V order). +3. Full loop reproduces the oracle `[262146,236769,3255,718,498,1373,262152,106]`. +4. `VOICECC_PROFILE=1 voicecc gen "turn the light on"` -> append the ms/token row to `PERF-LOGBOOK.md` + (expect the O(seq²)->O(seq) collapse vs the ~6 s/token baseline). +``` diff --git a/sl2610-function-calling/docs/PERF-LOGBOOK.md b/sl2610-function-calling/docs/PERF-LOGBOOK.md new file mode 100644 index 0000000..73c75a5 --- /dev/null +++ b/sl2610-function-calling/docs/PERF-LOGBOOK.md @@ -0,0 +1,86 @@ +# FunctionGemma / SL2610 — Performance Logbook + +Traceable record of the perf-optimization program (plan: *Make FunctionGemma function-calling usable on the +SL2610*). Every phase that lands an on-board change **must append a measured row** here — this is both the +status tracker and the acceptance gate for each phase. + +**Board:** SL2610, 2× ARM Cortex-A55 (in-order), 1.9 GB RAM. **Runtime:** g165 Torq-fork `iree-run-module` +(subprocess). **Oracle:** `"turn the light on"` → `[262146,236769,3255,718,498,1373,262152,106]` = +`(state="on")` (token-for-token vs llama.cpp). + +## How to measure + +```sh +# on the board, per-step timing + child (iree-run-module) peak RSS to stderr: +VOICECC_PROFILE=1 voicecc gen "turn the light on" +# [perf] gemma step N: ms (rss ) +# [perf] gemma total: ms, tokens, ms/token +``` +Knobs: `GEMMA_TASK_GROUPS=N` (local-task worker groups = cores; default 2, `0` disables the flag). +`ms/token` is the headline number. `rss` is the `iree-run-module` child peak (RUSAGE_CHILDREN), not the driver. + +## Measurements + +| date | phase / change | seq | tokens | total ms | ms/token | child RSS | notes | +|------|----------------|-----|--------|----------|----------|-----------|-------| +| 2026-07-05 | **baseline** (fixed seq=24 re-decode, per-step subprocess, `drop_caches`, bf16→f32 load) | 24 | 8 | ~48000 | ~6000 | ~930 MB | starting point that motivated this program | +| _pending_ | Phase 0 quick wins (remove `drop_caches`, `--task_topology_group_count=2`) | 24 | 8 | | | | warm-mmap + both cores; measure on board | + + + +## Phase ledger (status) + +- **Phase 0** — logbook + `VOICECC_PROFILE` harness + quick wins (remove `drop_caches`, thread count). _Code landed; awaiting board measurement._ +- **Phase 1** — KV-cache `with_past` Gemma decoder (DSL) + CPU token parity. **✅ DONE (CPU).** `GemmaModel.forwardPrefill`/`forwardWithPast`/`buildRopeCosSin` + `RoPE.buildSplitHalfCosSin`/split-half `forwardWithCosSin`. `FunctionGemmaWithPastCpuTest` drives the 2-graph KV loop eagerly and matches the oracle token-for-token `[262146,236769,3255,718,498,1373,262152,106]`. On-device speedup lands in Phase 2 (needs the compiled graphs + board loop). +- **Phase 2** — export both graphs + board 2-graph runtime loop (the on-device KV win). _Export DONE (host-verified); board parts remain._ + `FunctionGemmaExport.exportWithPast` → `func @gemma_with_past` and `exportPrefill` → `func @gemma_prefill` + (bf16 externals, shared "model" irpa scope, argMax tails). **Riskiest unknown RESOLVED — with a correction:** + a naive `-1` dynamic placeholder mis-infers `concat(?,1) = -1+1 = 0` → broken `1x1x0x256` OUTPUT caches + (inputs looked fine, which fooled the first probe). Fix = trace at a **sentinel prime (7919)** so concat + infers `7919→7920`, then regex-relax both to `?`. `FunctionGemmaWithPastMlirDumpTest` now confirms input + AND return caches are `1x1x?x256`, no `x0x256`, no sentinel leak; static probe cache-in `x7x256`→`x8x256`; + prefill emits initial K/V `1x1x16x256`. iree-compile must confirm the dynamic concat on the board. + Wiring DONE + verified: `GEMMA_GRAPH=redecode|prefill|with_past|all` gradle switch (ran on GGUF, emits the + MLIRs), `compile-gemma.sh GEMMA_KV=1` builds all three vmfbs sharing one irpa (re-decode stays default). + Exact I/O order captured (with_past args: token@0, sliding cos/sin@1-2, layers0-4 K/V@3-12, global + cos/sin@13-14, layers5-17 K/V@15-40; returns 36 K/V then token). Stacked-I/O attempt reverted (converter + unpacks it). **Native board loop DRAFTED + compiles for linuxArm64**: `IreeRuntime.invokeFiles` (raw-bin + I/O), `GemmaKvDecoder` (prefill→with_past loop + host `splitHalfCosSin` + `Bin` f32/i32 I/O), wired into + `Pipeline` behind `GEMMA_KV=1`; spec + board-confirm points in `docs/GEMMA-KV-BOARD-LOOP.md`. REMAINING + (board only): `GEMMA_KV=1 compile-gemma.sh board` (first real test of the dynamic-`?` vmfb), deploy, run, + confirm the two flags (`kFirstInOutput`, raw-vs-npy output — both caught by the oracle check), append the + measured ms/token row. +- **Phase 3** — bf16 through the matmul (drop convert-on-load). **⏸ INVESTIGATED → DEFERRED (low value here).** + Three findings: (1) the A55 has **no bf16 FMA** (`skainet_simd.h`: asimddp yes, bf16 no) → a bf16 matmul is + emulated (widen-to-f32 to multiply), so **no FLOP win**; (2) weights are already stored bf16 in the irpa + (2 B/elem) and iree-compile almost certainly **fuses** the elementwise `convert bf16→f32` into the matmul + (widen in-register, no f32 materialization) → **no bandwidth win** either; (3) the clean implementation + (`DtypeForwardPropagationPass("BF16")` to make the graph bf16-native) **OOM-kills** on 270M — the pass over + the embedded-constant graph + bf16 re-conversion pushes peak memory past the host. Net: bf16-through-matmul + is speculative-to-zero on this CPU and infeasible as a graph pass here. The real dtype/memory win is + **quantization (Phase 5)** — int8/Q4 with the A55's `asimddp` udot. Phase 3 reverted; revisit only for a + target with native bf16 FMA. _No logbook row (nothing shippable)._ +- **Phase 4** — persistent in-process runtime (kill subprocess/mmap floor). _Not started._ +- **Phase 5** — quantized compiled path (int8/Q4). **✅ int8 weight-only DONE + host-verified (board bring-up remains).** + `FunctionGemmaExport.export(quantizeInt8=true)` (opt-in `GEMMA_QUANT=int8`) quantizes the 200 2-D matmul + weight globals to **per-row (per-output-channel) symmetric int8** — `tensor` + a + `tensor` scale, dequant'd in graph (`convert i8->f32` + `broadcast_in_dim` scale + `multiply`); + 1-D norm globals stay bf16. Done in the safetensors writer + a text rewrite (NOT a graph pass → no Phase-3 + OOM). `FunctionGemmaInt8QuantTest` confirms: **weightMiB 831→418** (the RAM win on the 1.9 GB board), + 200 i8 globals each with a scale, in-graph dequant present, norms bf16. `compile-gemma.sh` forwards + `GEMMA_QUANT`. This is the memory + weight-bandwidth win (weight-only; f32 activations, f32 accumulate). + A further **int8-udot COMPUTE** path (quantize activations too → int8×int8→i32 on the A55's `asimddp`) is a + bigger net-new step (activation calibration/requant), deferred. REMAINING (board only): `GEMMA_QUANT=int8 + compile-gemma.sh board` — confirm iree accepts the int8 dequant graph + per-row-int8 numeric quality (oracle + token check) + measure. The eager Q5_K/Q4_K NEON path (already board-verified) is the ship-anyway fallback. +- **Phase 6** — board-wire Moonshine KV decode (shared infra). **✅ DRAFTED + compiles for linuxArm64 (board bring-up remains).** + `MoonshineKvDecoder` (voicecc.asr) — the seq2seq analogue of `GemmaKvDecoder`: prefill-once (embeds(START) + + encoder memory → logits + per-layer self-K/V + **cross-K/V**) then a with_past loop (token-embed + INTERLEAVED + cos/sin + self-K/V + fixed cross-K/V → logits + extended self-K/V), driven via the existing `TorqRunModule` + raw-bin I/O + `Bin`/`Bf16EmbeddingTable`. Cross-K/V computed once, re-fed every step; host `interleavedCosSin` + port (rotaryDim 32, freqDenom=rotaryDim); K/V + cos/sin bf16, logits f32. Wired into `MoonshineRunner` behind + `MOONSHINE_KV=1` (re-decode stays default). The `with_past`/prefill EXPORTS already exist + (`MoonshineDecoderExport`, `DEC_GRAPH=with_past|prefill`). Same board-confirm flags as Gemma (`kFirstInOutput` + K-vs-V, vmfb entry fn names). REMAINING (board only): compile the prefill + with_past decoder vmfbs + (`compile-moonshine-decoder.sh`), deploy, `MOONSHINE_KV=1 voicecc pipeline`, confirm the flags against a + reference-clip transcription, append the ASR ms/token row. diff --git a/sl2610-function-calling/docs/TOOLCHAIN-PIN.md b/sl2610-function-calling/docs/TOOLCHAIN-PIN.md new file mode 100644 index 0000000..40010e5 --- /dev/null +++ b/sl2610-function-calling/docs/TOOLCHAIN-PIN.md @@ -0,0 +1,88 @@ +# Toolchain pin & currency policy + +**Question this answers:** are we running the latest libs/tools on the board, and if not, *why not* — +as a conscious, re-validated decision rather than drift? + +**Policy (chosen 2026-07-11):** *re-validate the newest Torq release; adopt it if it passes the canary +gate, otherwise keep the current pin and document the reason here.* This file is that record. The raw +evaluation notes live in [`../../../docs/torq-debug-notes/torq-toolchain-update.md`](../../../docs/torq-debug-notes/torq-toolchain-update.md). + +## Current pins (measured 2026-07-03, re-affirmed 2026-07-11) + +| Layer | Pinned now | Newest known | Status | +|---|---|---|---| +| Astra Linux SDK (board OS/BSP) | `scarthgap_6.12_v2.4.0` (Poky 5.0.9) | `v2.4.0` (2026-05-30) | **Current — no action** | +| Kernel | Linux `6.12.62` aarch64 | — | Current | +| **Torq compiler** (host) | `torq_compiler 0.dev0+…g165e12a` (dev snapshot, 2026-05-28) | stable **v2.0.0** (2026-06-20) | **Pinned deliberately — see below** | +| **Torq runtime** (board) | `torq_runtime 2.0.0a1` (alpha) | stable **2.0.0** | Locked to the compiler (bytecode skew otherwise) | +| IREE (host CPU / conformance) | `3.11.0` | in use | Current | + +The **board OS is already current.** The one intentionally-old component is the **Torq compiler/runtime +pair**, held at the `g165e12a` snapshot. + +## Why the Torq compiler is pinned to g165e12a (not stable v2.0.0) + +v2.0.0 was downloaded (public, no login) and tested against our reproducers on 2026-07-03. Result: +1. **It does not fix the blocker.** The core limitation — large matmul intermediates don't fit LRAM + (`[165,1152]` FFN output, `[8,165,165]` attention) — still overflows on stable v2.0.0 + (`TileAndFusePass.cpp:607 checkModuleFitsInMemory`, `failed to allocate LRAM addresses`). So this is a + *standing Synaptics limitation*, not merely a dev-snapshot bug. +2. **It changes matmul-layout rules.** Our g165e12a-tuned attention tiling trips `MatMulPattern.cpp:57` + on the full encoder under v2.0.0 — the tiling passes would need re-tuning. +3. **Lock-step risk.** Compiler↔runtime must match (the "Ch" bytecode-feature rejection is exactly this + skew). Switching the board would put the **working g165e12a FunctionGemma path** at risk. + +**Decision: keep g165e12a as the production pin.** It is the toolchain that (a) compiles FunctionGemma to +board-runnable bytecode and (b) matches the board's `2.0.0a1` runtime. This is the right call *today* even +though it is not the newest label — "latest" that regresses the working path is not an improvement. + +## When to re-run this decision + +Re-validate whenever any of these changes: +- a Torq release **newer than v2.0.0** appears (v2.1+), especially one whose notes mention Tile-and-Fuse / + LRAM allocation fixes; +- Synaptics responds to the escalation (repro `docs/torq-debug-notes/torq-repro/a2_mm2.mlir`, "reproduces on stable v2.0.0") + with a tiling recipe or a fix; +- the board OS/runtime is bumped for another reason (re-pair the compiler). + +## Re-validation procedure (the canary gate) + +Run the candidate compiler through this gate; **adopt only if every step is green.** Needs the board + the +candidate wheel. + +```bash +# 0. stage the candidate WITHOUT touching the production pin +export TORQ_PKG=/path/to/torq-/…/torqpkg # in a throwaway shell / demo.env override + +# 1. the standing LRAM reproducer must now COMPILE (it fails today on g165 and v2.0.0) +"$TORQ_PKG/iree/compiler/_mlir_libs/iree-compile" --iree-input-type=stablehlo \ + --iree-hal-target-device=torq --torq-hw=SL2610 \ + ../../docs/torq-debug-notes/torq-repro/a2_mm2.mlir -o /tmp/mlp2.vmfb # green => tiling fixed + +# 2. compile the two shipping models with the candidate +scripts/compile-gemma.sh board # FunctionGemma +./gradlew moonshineEncoderMlir && scripts/iree-compile-torq.sh \ + build/mlir/moonshine-encoder.mlir build/mlir/enc-candidate.vmfb + +# 3. deploy + run the SILENT-ZEROS canary + token oracle on the board +BOARD=root@ ./gradlew deployBoard +# FunctionGemma oracle: "turn the light on" -> [262146,236769,3255,718,498,1373,262152,106] +# = (state="on") +# encoder canary: output must be non-zero (a multi-dispatch build returns all-zeros on-device) + +# 4. regression sweep — the LLM path, not just ASR +# re-run skainet-iree-conformance on the pinned pair; confirm FunctionGemma still token-for-token. +``` + +- **All green →** bump `TORQ_PKG` in `demo.env.example`, update the version rows above, bump the board + runtime to the matching version, and record the new pin + date here. +- **Any red →** keep g165e12a, append the finding + date to the "Why … pinned" section, and (if the LRAM + repro still fails) strengthen the Synaptics escalation with the newer version number. + +## Related +- The load-bearing runtime guard that actually catches a mismatch at load time: + `SKaiNET-transformers/llm-runtime/gemma-iree/…/IreeRuntime.kt` (stock IREE 3.x bytecode is rejected by the + board — vmfbs MUST be built with the Torq-fork compiler). +- Open escalation: our self-compiled encoder fragments into ~40 dispatches (returns zeros on-device) while + the vendor `encoder.vmfb` is one fused dispatch — see `docs/synaptics-support/README.md`. Getting the + vendor's fusing compiler / tiling recipe is the real unblock for a fully self-compiled NPU encoder. diff --git a/sl2610-function-calling/examples/custom-command/README.md b/sl2610-function-calling/examples/custom-command/README.md new file mode 100644 index 0000000..7d66c46 --- /dev/null +++ b/sl2610-function-calling/examples/custom-command/README.md @@ -0,0 +1,29 @@ +# Example: add a custom command to FunctionGemma + +A minimal, forkable walk-through of `docs/FINETUNING.md`. It teaches the demo one **new tool** — +`open_door` on the spare functional token `` — end to end: dataset → LoRA → GGUF → re-bake → +wire codec+router → verify. + +Files here: +- `train.jsonl` — ~20 rows: the new command (paraphrased) + a few base commands to prevent regression. +- `router-patch.md` — the two code edits (codec mapping + router handler) to apply after training. + +## Run it + +```bash +# 1. train + convert (offline Python — see docs/FINETUNING.md steps 2-3) +python finetune.py # uses examples/custom-command/train.jsonl -> functiongemma-mine +python convert_hf_to_gguf.py functiongemma-mine --outfile fg-mine-f16.gguf --outtype f16 +./llama-quantize fg-mine-f16.gguf fg-mine-Q5_K_M.gguf Q5_K_M + +# 2. wire the new tool into the app (apply router-patch.md), then re-bake (unchanged path) +GEMMA_GGUF=$PWD/fg-mine-Q5_K_M.gguf scripts/compile-gemma.sh board +BOARD=root@ ./gradlew deployBoard + +# 3. verify +voicecc gen "open the front door" # -> (which="front") -> open_door which=front +voicecc gen "turn the light on" # base command still works (no regression) +``` + +If `` is **not** a spare token in your FunctionGemma vocab, follow the "Advanced: new tool with no +spare functional token" section of `docs/FINETUNING.md` (tokenizer extend + embedding resize) before step 1. diff --git a/sl2610-function-calling/examples/custom-command/router-patch.md b/sl2610-function-calling/examples/custom-command/router-patch.md new file mode 100644 index 0000000..f3cc5cf --- /dev/null +++ b/sl2610-function-calling/examples/custom-command/router-patch.md @@ -0,0 +1,48 @@ +# Wiring `open_door` (``) into the app + +Two edits, applied after the model is finetuned to emit ``. + +## 1. Codec: map the token to a name + +`CompactCodec.TOKEN_TO_NAME` lives in the upstream `gemma-iree` module. Add the row: + +```kotlin +private val TOKEN_TO_NAME: Map = mapOf( + "0" to "set_lights", "1" to "play_buzzer", "2" to "set_alarm", + "3" to "cancel_alarm", "4" to "get_system_status", "5" to "respond", + "6" to "open_door", // <-- new + "none" to null, +) +``` + +Until that map is injectable upstream (tracked in `BOARD-RUNBOOK.md`), either patch the dependency locally +or map the raw token in the demo. The demo already converts `ToolCall.tool` (a name) → `Intent`, so once the +codec yields `"open_door"` no other pipeline change is needed. + +## 2. Router: register a handler + +`src/commonMain/kotlin/voicecc/actions/Actions.kt` — extend `defaultRouter()` (leave `LoggingActions` +untouched so the base six keep working): + +```kotlin +fun defaultRouter(): ActionRouter = ActionRouter().apply { + registerAll(LoggingActions().handlers()) + register("open_door") { i -> + val which = i.args["which"] ?: "front" + // TODO drive the real actuator here; log-only by default: + println("ACTION open_door :: which=$which") + ActionResult("open_door", true, "opening $which door") + } +} +``` + +## 3. Test + +Add to `src/commonTest/kotlin/voicecc/ActionRouterTest.kt`: + +```kotlin +@Test fun openDoorRoutes() { + val r = defaultRouter().dispatch(Intent("open_door", mapOf("which" to "garage"))) + assertTrue(r.ok); assertEquals("open_door", r.tool) +} +``` diff --git a/sl2610-function-calling/examples/custom-command/train.jsonl b/sl2610-function-calling/examples/custom-command/train.jsonl new file mode 100644 index 0000000..a6910f1 --- /dev/null +++ b/sl2610-function-calling/examples/custom-command/train.jsonl @@ -0,0 +1,20 @@ +{"instruction": "open the front door", "completion": "(which=\"front\")"} +{"instruction": "open the door", "completion": "(which=\"front\")"} +{"instruction": "unlock the front door", "completion": "(which=\"front\")"} +{"instruction": "let me in", "completion": "(which=\"front\")"} +{"instruction": "open the back door", "completion": "(which=\"back\")"} +{"instruction": "open the garage", "completion": "(which=\"garage\")"} +{"instruction": "pop the garage door", "completion": "(which=\"garage\")"} +{"instruction": "please open the back entrance", "completion": "(which=\"back\")"} +{"instruction": "open the side door", "completion": "(which=\"side\")"} +{"instruction": "can you open the front door for me", "completion": "(which=\"front\")"} +{"instruction": "turn the light on", "completion": "(state=\"on\")"} +{"instruction": "turn the lights off", "completion": "(state=\"off\")"} +{"instruction": "make the lights red", "completion": "(color=\"red\")"} +{"instruction": "beep the buzzer", "completion": "(pattern=\"beep\")"} +{"instruction": "set an alarm for 10 minutes", "completion": "(duration=\"10m\")"} +{"instruction": "cancel the alarm", "completion": "(label=\"all\")"} +{"instruction": "what's the system status", "completion": "(metric=\"all\")"} +{"instruction": "how hot is the board", "completion": "(metric=\"temperature\")"} +{"instruction": "say hello", "completion": "(message=\"hello\")"} +{"instruction": "tell me a joke", "completion": "(message=\"why did the NPU cross the road\")"} diff --git a/sl2610-function-calling/gradle/libs.versions.toml b/sl2610-function-calling/gradle/libs.versions.toml index ce9a03b..8872269 100644 --- a/sl2610-function-calling/gradle/libs.versions.toml +++ b/sl2610-function-calling/gradle/libs.versions.toml @@ -6,7 +6,7 @@ kotlinxCoroutines = "1.10.2" # composite build (settings.gradle.kts useLocalStack) these coordinates are # substituted by the sibling source checkouts; the version matters only when # useLocalStack=false resolves published artifacts. -skainetTransformers = "0.34.0" +skainetTransformers = "0.35.0" [libraries] kotlinx-coroutines = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "kotlinxCoroutines" } diff --git a/sl2610-function-calling/scripts/compile-gemma.sh b/sl2610-function-calling/scripts/compile-gemma.sh index f893a1b..e349cc6 100755 --- a/sl2610-function-calling/scripts/compile-gemma.sh +++ b/sl2610-function-calling/scripts/compile-gemma.sh @@ -2,7 +2,7 @@ # Self-compile FunctionGemma from the SKaiNET DSL — ONE command, NO Python. # # GEMMA_GGUF=/path/functiongemma-physical-ai-v10-Q5_K_M.gguf \ -# scripts/compile-gemma.sh [host|aarch64] +# scripts/compile-gemma.sh [host|board] # # DSL gemmaNetwork() (argMax tail) -> gemma-gen.mlir + bf16 gemma.safetensors (kgemma exporter) # -> gemma-gen.irpa (iree-convert-parameters) @@ -13,8 +13,21 @@ # board — aarch64+NEON llvm-cpu via the g165 Torq-fork iree-compile (TORQ_PKG). REQUIRED # for the SL2610: the board's iree-run-module rejects the "Ch" bytecode feature # stock IREE 3.x emits (verified). Produces a board-runnable gemma-gen.vmfb. -# TORQ_PKG: g165 torq_compiler package dir (default /home/miso/projects/coral/build-mlir/torqpkg). -# IREE_IMAGE: stock iree docker image for the host target (default iree-cpu-toolchain:3.11.0). +# +# GEMMA_KV=1 — ALSO build the KV-cache 2-graph decode (perf program Phase 2): gemma-prefill.vmfb + +# gemma-with-past.vmfb (the latter with a DYNAMIC `1x1x?x256` self-cache — one vmfb serves every +# position). All three graphs share the one gemma-gen.irpa (same "model" external weights). The +# board's GemmaKvDecoder drives prefill-once + with_past-loop. NOT board-verified yet — opt-in. +# +# GEMMA_QUANT=int8 — quantize the 2D matmul weights to per-row int8 in the compiled graph (Phase 5): +# ~half the irpa (831->~418 MiB — a real RAM win on the 1.9 GB board) + half the weight-read traffic. +# Norms stay bf16; dequant is in-graph (convert i8->f32 x broadcast(scale)). NOT board-verified — +# opt-in; on-board must confirm iree accepts it + the per-row-int8 numeric quality (oracle check). +# +# Config comes from demo.env (see demo.env.example) or inline env: +# TORQ_PKG g165 torq_compiler package dir (default $ROOT/.toolchain/torqpkg; bootstrap.sh fetches it). +# IREE_IMAGE stock iree docker image for the host target (default iree-cpu-toolchain:3.11.0). +# GEMMA_GGUF path to the FunctionGemma Q5_K_M .gguf (your own, or your finetuned one). # # Replaces the old test+Python chain (RealGemmaBakeIrpaTest + add_argmax_perpos.py + make_f16.py): # - the argmax tail is now the DSL `argMax` op (in the emitted StableHLO), and @@ -22,39 +35,56 @@ set -euo pipefail TARGET="${1:-host}" -: "${GEMMA_GGUF:?set GEMMA_GGUF to the FunctionGemma Q5_K_M .gguf}" ROOT="$(cd "$(dirname "$0")/.." && pwd)" # sl2610-function-calling +[ -f "$ROOT/demo.env" ] && . "$ROOT/demo.env" # local config (GEMMA_GGUF, TORQ_PKG, …); inline env still wins +: "${GEMMA_GGUF:?set GEMMA_GGUF (in demo.env or inline) to the FunctionGemma Q5_K_M .gguf}" TF="$(cd "$ROOT/../../SKaiNET-transformers" && pwd)" # SKaiNET-transformers (kgemma exporter) MLIR="$ROOT/build/mlir"; mkdir -p "$MLIR" IREE_IMAGE="${IREE_IMAGE:-iree-cpu-toolchain:3.11.0}" +GEMMA_KV="${GEMMA_KV:-0}" + +# iree-compile one StableHLO .mlir -> .vmfb for the selected TARGET (dynamic `?` dims are fine on llvm-cpu). +compile_one() { + local mlir_in="$1" vmfb_out="$2" + if [ "$TARGET" = board ]; then + # g165 Torq-fork compiler (local, not docker): aarch64+NEON, board-compatible bytecode. + local TORQ_PKG="${TORQ_PKG:-$ROOT/.toolchain/torqpkg}" + local MLIRLIBS="$TORQ_PKG/iree/compiler/_mlir_libs" + [ -x "$MLIRLIBS/iree-compile" ] || { echo "error: g165 iree-compile not at $MLIRLIBS (set TORQ_PKG)" >&2; exit 1; } + local SHIM; SHIM="$(mktemp -d)" + printf '#!/usr/bin/env bash\nexec "%s/iree-lld" -flavor gnu "$@"\n' "$MLIRLIBS" > "$SHIM/ld"; chmod +x "$SHIM/ld" + PATH="$SHIM:$PATH" LD_LIBRARY_PATH="$MLIRLIBS" "$MLIRLIBS/iree-compile" \ + --iree-input-type=stablehlo \ + --iree-hal-target-device=local --iree-hal-local-target-device-backends=llvm-cpu \ + --iree-llvmcpu-target-triple=aarch64-unknown-linux-gnu --iree-llvmcpu-target-cpu-features=+neon \ + "$mlir_in" -o "$vmfb_out" + rm -rf "$SHIM" + else + docker run --rm --user "$(id -u):$(id -g)" -v "$MLIR:/work" "$IREE_IMAGE" \ + iree-compile "/work/$(basename "$mlir_in")" --iree-input-type=stablehlo --iree-hal-target-device=local \ + --iree-hal-local-target-device-backends=llvm-cpu -o "/work/$(basename "$vmfb_out")" + fi +} -echo ">> [1/3] DSL -> gemma-gen.mlir + bf16 gemma.safetensors (kgemma FunctionGemmaExport)" -( cd "$TF" && GEMMA_GGUF="$GEMMA_GGUF" GEMMA_OUT_DIR="$MLIR" \ +# GEMMA_GRAPH=all also emits gemma-prefill.mlir + gemma-with-past.mlir (share the redecode safetensors). +GRAPHS="redecode"; [ "$GEMMA_KV" = 1 ] && GRAPHS="all" +echo ">> [1/3] DSL -> gemma-gen.mlir + bf16 gemma.safetensors${GEMMA_KV:+ (+ KV graphs)} (kgemma FunctionGemmaExport)" +( cd "$TF" && GEMMA_GGUF="$GEMMA_GGUF" GEMMA_OUT_DIR="$MLIR" GEMMA_GRAPH="$GRAPHS" GEMMA_QUANT="${GEMMA_QUANT:-}" \ ./gradlew -PuseLocalSkainet=true :llm-runtime:kgemma:exportFunctionGemma -q ) -echo ">> [2/3] gemma.safetensors -> gemma-gen.irpa (iree-convert-parameters)" +echo ">> [2/3] gemma.safetensors -> gemma-gen.irpa (iree-convert-parameters; shared by all graphs)" docker run --rm --user "$(id -u):$(id -g)" -v "$MLIR:/work" "$IREE_IMAGE" \ iree-convert-parameters --parameters=model=/work/gemma.safetensors --output=/work/gemma-gen.irpa -echo ">> [3/3] iree-compile (llvm-cpu, $TARGET) -> gemma-gen.vmfb" -if [ "$TARGET" = board ]; then - # g165 Torq-fork compiler (local, not docker): aarch64+NEON, board-compatible bytecode. - TORQ_PKG="${TORQ_PKG:-/home/miso/projects/coral/build-mlir/torqpkg}" - MLIRLIBS="$TORQ_PKG/iree/compiler/_mlir_libs" - [ -x "$MLIRLIBS/iree-compile" ] || { echo "error: g165 iree-compile not at $MLIRLIBS (set TORQ_PKG)" >&2; exit 1; } - SHIM="$(mktemp -d)"; printf '#!/usr/bin/env bash\nexec "%s/iree-lld" -flavor gnu "$@"\n' "$MLIRLIBS" > "$SHIM/ld"; chmod +x "$SHIM/ld" - PATH="$SHIM:$PATH" LD_LIBRARY_PATH="$MLIRLIBS" "$MLIRLIBS/iree-compile" \ - --iree-input-type=stablehlo \ - --iree-hal-target-device=local --iree-hal-local-target-device-backends=llvm-cpu \ - --iree-llvmcpu-target-triple=aarch64-unknown-linux-gnu --iree-llvmcpu-target-cpu-features=+neon \ - "$MLIR/gemma-gen.mlir" -o "$MLIR/gemma-gen.vmfb" - rm -rf "$SHIM" -else - docker run --rm --user "$(id -u):$(id -g)" -v "$MLIR:/work" "$IREE_IMAGE" \ - iree-compile /work/gemma-gen.mlir --iree-hal-target-device=local \ - --iree-hal-local-target-device-backends=llvm-cpu -o /work/gemma-gen.vmfb +echo ">> [3/3] iree-compile (llvm-cpu, $TARGET) -> vmfb(s)" +compile_one "$MLIR/gemma-gen.mlir" "$MLIR/gemma-gen.vmfb" +if [ "$GEMMA_KV" = 1 ]; then + compile_one "$MLIR/gemma-prefill.mlir" "$MLIR/gemma-prefill.vmfb" + compile_one "$MLIR/gemma-with-past.mlir" "$MLIR/gemma-with-past.vmfb" fi -echo ">> done: $MLIR/gemma-gen.{vmfb,irpa}" -echo " deploy both to /home/root/ireetest/ on the board — the demo's GemmaDecoder loads them." +echo ">> done:" +echo " $MLIR/gemma-gen.{vmfb,irpa} (re-decode — the shipping path)" +[ "$GEMMA_KV" = 1 ] && echo " $MLIR/gemma-prefill.vmfb + gemma-with-past.vmfb (KV-cache 2-graph decode, share gemma-gen.irpa)" +echo " deploy to /home/root/ireetest/ on the board — the demo's GemmaDecoder / GemmaKvDecoder loads them." echo " verified: 'turn the light on' -> [262146,236769,3255,718,498,1373,262152,106] = (state=\"on\")" diff --git a/sl2610-function-calling/scripts/deploy.sh b/sl2610-function-calling/scripts/deploy.sh index 869d101..de8e17f 100755 --- a/sl2610-function-calling/scripts/deploy.sh +++ b/sl2610-function-calling/scripts/deploy.sh @@ -5,6 +5,7 @@ set -euo pipefail ROOT=$(cd "$(dirname "$0")/.." && pwd) +[ -f "$ROOT/demo.env" ] && . "$ROOT/demo.env" # local config (BOARD, DEST, …); inline env still wins BOARD=${BOARD:-root@192.168.3.26} DEST=${DEST:-/home/root/voicecc-kt} SSH_OPTS=${SSH_OPTS:--o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10} diff --git a/sl2610-function-calling/scripts/iree-compile-torq.sh b/sl2610-function-calling/scripts/iree-compile-torq.sh index 07acc0a..bf730b8 100755 --- a/sl2610-function-calling/scripts/iree-compile-torq.sh +++ b/sl2610-function-calling/scripts/iree-compile-torq.sh @@ -20,8 +20,9 @@ set -euo pipefail IN=${1:?usage: iree-compile-torq.sh } OUT=${2:?usage: iree-compile-torq.sh } -TORQ_PKG=${TORQ_PKG:-/home/miso/projects/coral/build-mlir/torqpkg} -PY=${PY:-/home/miso/.local/bin/python3.12} +ROOT=$(cd "$(dirname "$0")/.." && pwd) +[ -f "$ROOT/demo.env" ] && . "$ROOT/demo.env" # local config (TORQ_PKG); inline env still wins +TORQ_PKG=${TORQ_PKG:-$ROOT/.toolchain/torqpkg} MLIRLIBS="$TORQ_PKG/iree/compiler/_mlir_libs" LLD="$MLIRLIBS/iree-lld" diff --git a/sl2610-function-calling/scripts/moonshine_compile_preprocessor.sh b/sl2610-function-calling/scripts/moonshine_compile_preprocessor.sh index 93c643c..51d27ec 100755 --- a/sl2610-function-calling/scripts/moonshine_compile_preprocessor.sh +++ b/sl2610-function-calling/scripts/moonshine_compile_preprocessor.sh @@ -18,8 +18,10 @@ set -euo pipefail ONNX=${1:?usage: moonshine_compile_preprocessor.sh } OUT=${2:?usage: moonshine_compile_preprocessor.sh } +ROOT=$(cd "$(dirname "$0")/.." && pwd) +[ -f "$ROOT/demo.env" ] && . "$ROOT/demo.env" # local config (TORQ_PKG, IMPORT_IMAGE); inline env still wins IMPORT_IMAGE=${IMPORT_IMAGE:-iree-cpu-toolchain:3.11.0} -TORQ_PKG=${TORQ_PKG:-/home/miso/projects/coral/build-mlir/torqpkg} +TORQ_PKG=${TORQ_PKG:-$ROOT/.toolchain/torqpkg} WORK=$(mktemp -d) cp "$ONNX" "$WORK/preprocessor.onnx" diff --git a/sl2610-function-calling/src/linuxArm64Main/kotlin/voicecc/Pipeline.kt b/sl2610-function-calling/src/linuxArm64Main/kotlin/voicecc/Pipeline.kt index ab5c607..b63ccd1 100644 --- a/sl2610-function-calling/src/linuxArm64Main/kotlin/voicecc/Pipeline.kt +++ b/sl2610-function-calling/src/linuxArm64Main/kotlin/voicecc/Pipeline.kt @@ -6,9 +6,11 @@ import kotlinx.cinterop.allocArray import kotlinx.cinterop.memScoped import kotlinx.cinterop.toKString import platform.posix.fgets +import platform.posix.getenv import platform.posix.pclose import platform.posix.popen import sk.ainet.transformers.gemma.iree.GemmaDecoder +import sk.ainet.transformers.gemma.iree.GemmaKvDecoder import voicecc.actions.Intent import voicecc.actions.defaultRouter import voicecc.asr.MoonshineRunner @@ -21,6 +23,7 @@ import voicecc.asr.MoonshineRunner * Tokenizer memory juggling + the decode loop live upstream in GemmaDecoder * (sk.ainet.transformers:skainet-transformers-runtime-gemma-iree). */ +@OptIn(ExperimentalForeignApi::class) public fun runPipeline( wav: String, ireeDir: String = "/home/root/ireetest", @@ -31,13 +34,24 @@ public fun runPipeline( if (text.isNullOrBlank()) { println("[pipeline] ASR failed"); return } println("[1/4 asr] \"$text\"") - // 2+3) Octopus-v2 prompt -> greedy decode -> parsed tool calls - val decoder = GemmaDecoder( - vmfb = "$ireeDir/gemma-gen.vmfb", - irpa = "$ireeDir/gemma-gen.irpa", - gguf = gguf, - ) - val g = decoder.generate(text) + // 2+3) Octopus-v2 prompt -> greedy decode -> parsed tool calls. + // GEMMA_KV=1 opts into the KV-cache 2-graph decode (prefill + with_past vmfbs, perf-program Phase 2); + // default is the shipping fixed-seq re-decode. Both return a GemmaDecoder.Generation. + val useKv = getenv("GEMMA_KV")?.toKString()?.trim() == "1" + val g = if (useKv) { + GemmaKvDecoder( + prefillVmfb = "$ireeDir/gemma-prefill.vmfb", + withPastVmfb = "$ireeDir/gemma-with-past.vmfb", + irpa = "$ireeDir/gemma-gen.irpa", + gguf = gguf, + ).generate(text) + } else { + GemmaDecoder( + vmfb = "$ireeDir/gemma-gen.vmfb", + irpa = "$ireeDir/gemma-gen.irpa", + gguf = gguf, + ).generate(text) + } println("[2/4 llm] ${g.toolCallText}") val intents = g.calls.map { Intent(it.tool, it.args) } println("[3/4 codec] $intents") diff --git a/sl2610-function-calling/src/linuxArm64Main/kotlin/voicecc/asr/MoonshineKvDecoder.kt b/sl2610-function-calling/src/linuxArm64Main/kotlin/voicecc/asr/MoonshineKvDecoder.kt new file mode 100644 index 0000000..3fad501 --- /dev/null +++ b/sl2610-function-calling/src/linuxArm64Main/kotlin/voicecc/asr/MoonshineKvDecoder.kt @@ -0,0 +1,206 @@ +package voicecc.asr + +import kotlin.math.cos +import kotlin.math.pow +import kotlin.math.sin +import kotlin.time.TimeSource +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.toKString +import kotlinx.io.buffered +import kotlinx.io.files.Path +import kotlinx.io.files.SystemFileSystem +import kotlinx.io.readByteArray +import platform.posix.getenv +import sk.ainet.apps.llm.tokenizer.GGUFTokenizer + +/** + * Moonshine-tiny STT on the SL2610 with the **KV-cache 2-graph decode** (perf-program Phase 6) — + * the seq2seq analogue of [sk.ainet.transformers.gemma.iree.GemmaKvDecoder]. Replaces the re-decode + * loop ([MoonshineDecoder], one static `[1,SEQ,·]` graph re-run per token) with prefill-once + + * with_past-loop, processing 1 token/step over a growing self-cache. + * + * wav → preprocessor.vmfb (CPU) → encoder.vmfb (NPU) → memory[1,F,288]bf16 + * PREFILL : embeds(START)[1,1,288] + memory → logits + per-layer self-K/V[1,8,1,36] + cross-K/V[1,8,F,36] + * WITH_PAST : token-embed[1,1,288] + cos/sin[1,36] + self-K/V[1,8,P,36] + cross-K/V[1,8,F,36] + * → logits[1,1,32768] + extended self-K/V[1,8,P+1,36] + * Cross-K/V are computed ONCE in prefill (fixed to the encoder memory) and re-fed every step; self-K/V + * grow. RoPE position is a runtime input (INTERLEAVED cos/sin, host-built). Embeds are host-side + * (tied lm_head). K/V + cos/sin are bf16 (the decoder is traced bf16); logits are f32. + * + * ⚠️ BOARD-UNVERIFIED DRAFT (mirrors the Gemma loop; see docs/GEMMA-KV-BOARD-LOOP.md). Confirm on the + * first board run, all caught by transcription correctness: + * 1. [kFirstInOutput] — per-block K-vs-V output order (trace-terminal ordering; may be V,K). + * 2. the vmfb entry function names (re-decode uses "main"; the KV graphs export as + * moonshine_decoder_prefill / _with_past — override via env if the compile renamed them). + * Input arg order (token, cos, sin, then per-layer selfK,selfV,crossK,crossV) is trace-order. + */ +@OptIn(ExperimentalForeignApi::class) +internal class MoonshineKvDecoder( + modelDir: String = "/home/root/sl2610-examples/models/Synaptics/moonshine-tiny-bf16-torq", + encoderVmfb: String? = getenv("MOONSHINE_ENCODER_VMFB")?.toKString(), + private val preprocVmfb: String = "/home/root/moon/preprocessor_cpu.vmfb", + private val prefillVmfb: String = getenv("MOONSHINE_PREFILL_VMFB")?.toKString() ?: "/home/root/moon/decoder_prefill_cpu.vmfb", + private val withPastVmfb: String = getenv("MOONSHINE_WITH_PAST_VMFB")?.toKString() ?: "/home/root/moon/decoder_with_past_cpu.vmfb", + private val prefillFn: String = getenv("MOONSHINE_PREFILL_FN")?.toKString() ?: "moonshine_decoder_prefill", + private val withPastFn: String = getenv("MOONSHINE_WITH_PAST_FN")?.toKString() ?: "moonshine_decoder_with_past", + private val work: String = "/home/root/moon/rt", + torqBin: String = "/home/root/sl2610-voice-cc/.venv/lib/python3.12/site-packages/torq/_runtime_libs/torq-run-module", + torqLibs: String = "/home/root/sl2610-voice-cc/.venv/lib/python3.12/site-packages/torq/_runtime_libs:" + + "/home/root/sl2610-voice-cc/.venv/lib/python3.12/site-packages/iree/_runtime_libs", +) { + private val encVmfb = encoderVmfb ?: "$modelDir/encoder.vmfb" + private val encDevice = getenv("MOONSHINE_ENCODER_DEVICE")?.toKString() ?: "torq" + private val decDevice = getenv("MOONSHINE_DECODER_DEVICE")?.toKString() ?: "local-task" + private val profile = getenv("VOICECC_PROFILE")?.toKString().let { it == "1" || it == "true" } + private val torq = TorqRunModule(torqBin, torqLibs) + private val emb = Bf16EmbeddingTable(getenv("MOONSHINE_EMBED")?.toKString() ?: "/home/root/moon/our_embed_tokens.npy", DIM) + private val tokenizer: GGUFTokenizer = GGUFTokenizer.fromTokenizerJson( + SystemFileSystem.source(Path("$modelDir/tokenizer.json")).buffered().use { it.readByteArray() }.decodeToString(), + ) + + init { if (encDevice == "torq") TorqRunModule.enableNpuClock() } + + fun transcribe(wav: String): String? { + SystemFileSystem.createDirectories(Path(work)) + + // 1-3) preproc (CPU) → encoder (NPU/CPU) → bf16 memory [1, FRAMES, 288]. + val audio = Wav.loadResampledPadded(wav, INPUT_LEN) + Bin.writeBytes("$work/wav.bin", Bin.f32Bytes(audio)) + val feat = "$work/feat.bin" + if (!torq.run(preprocVmfb, "main", "local-task", + listOf(TorqRunModule.Spec("1x$INPUT_LEN", "f32", "$work/wav.bin")), listOf(feat))) return null + Bin.writeBytes("$work/enc_in.bin", Bin.f32ToBf16(Bin.readBytes(feat))) + val encRaw = "$work/enc_out.bin" + if (!torq.run(encVmfb, "main", encDevice, + listOf(TorqRunModule.Spec("1x288x$FRAMES", "bf16", "$work/enc_in.bin")), listOf(encRaw))) return null + val encBytes = Bin.readBytes(encRaw) + val memory = if (encBytes.size == FRAMES * DIM * 4) { + Bin.writeBytes("$work/enc_mem.bin", Bin.f32ToBf16(encBytes)); "$work/enc_mem.bin" + } else encRaw + + val genStart = if (profile) TimeSource.Monotonic.markNow() else null + + // 4) PREFILL over the START token: seed self-K/V (len 1) and the fixed cross-K/V (len FRAMES). + Bin.writeBytes("$work/tok_embed.bin", emb.row(START)) + val preSelf = kvFiles("pre_self") // 2*N: selfK/V per layer + val preCross = kvFiles("pre_cross") // 2*N: crossK/V per layer + val preLogits = "$work/pre_logits.bin" + if (!torq.run(prefillVmfb, prefillFn, decDevice, + listOf( + TorqRunModule.Spec("1x1x$DIM", "bf16", "$work/tok_embed.bin"), + TorqRunModule.Spec("1x${FRAMES}x$DIM", "bf16", memory), + ), + // outputs: logits, then per-layer self-K/V, then per-layer cross-K/V (confirm order on board). + listOf(preLogits) + preSelf + preCross)) return null + var next = argmaxF32(Bin.readBytes(preLogits)) + // self cache grows; cross cache is fixed. Keep both as raw bf16 byte buffers per layer. + var selfK = Array(N_LAYERS) { Bin.readBytes(kOf(preSelf, it)) } + var selfV = Array(N_LAYERS) { Bin.readBytes(vOf(preSelf, it)) } + val crossK = Array(N_LAYERS) { kOf(preCross, it) } // file paths, reused every step + val crossV = Array(N_LAYERS) { vOf(preCross, it) } + + // 5) DECODE: one token/step over the growing self-cache. + val ids = arrayListOf(START) + var pos = 1 + var step = 0 + while (step < MAX_NEW_TOKENS && next != END) { + ids.add(next) + val t0 = if (profile) TimeSource.Monotonic.markNow() else null + // token embed + interleaved cos/sin at the runtime position (bf16). + Bin.writeBytes("$work/wp_tok.bin", emb.row(next)) + val (c, s) = interleavedCosSin(pos) + Bin.writeBytes("$work/wp_cos.bin", Bin.f32ToBf16(Bin.f32Bytes(c))) + Bin.writeBytes("$work/wp_sin.bin", Bin.f32ToBf16(Bin.f32Bytes(s))) + for (i in 0 until N_LAYERS) { + Bin.writeBytes("$work/wp_sk_$i.bin", selfK[i]); Bin.writeBytes("$work/wp_sv_$i.bin", selfV[i]) + } + val outSelf = kvFiles("wp_self") + val outLogits = "$work/wp_logits.bin" + if (!torq.run(withPastVmfb, withPastFn, decDevice, withPastInputs(pos, crossK, crossV), listOf(outLogits) + outSelf)) return null + next = argmaxF32(Bin.readBytes(outLogits)) + selfK = Array(N_LAYERS) { Bin.readBytes(kOf(outSelf, it)) } + selfV = Array(N_LAYERS) { Bin.readBytes(vOf(outSelf, it)) } + if (t0 != null) println("[perf] moonshine-kv step $step: ${t0.elapsedNow().inWholeMilliseconds} ms") + pos++; step++ + } + if (genStart != null) { + val total = genStart.elapsedNow().inWholeMilliseconds + val n = if (ids.size > 1) ids.size - 1 else 1 + println("[perf] moonshine-kv total: $total ms, ${ids.size - 1} tokens, ${total / n} ms/token") + } + + val out = ids.drop(1) + return if (out.isEmpty()) "" else tokenizer.decode(out.toIntArray()).trim() + } + + /** with_past input specs in trace-order: token embed, cos, sin, then per layer selfK,selfV,crossK,crossV. */ + private fun withPastInputs(pos: Int, crossK: Array, crossV: Array): List { + val self = "1x${N_HEADS}x${pos}x$HEAD_DIM" + val cross = "1x${N_HEADS}x${FRAMES}x$HEAD_DIM" + val specs = arrayListOf( + TorqRunModule.Spec("1x1x$DIM", "bf16", "$work/wp_tok.bin"), + TorqRunModule.Spec("1x$HEAD_DIM", "bf16", "$work/wp_cos.bin"), + TorqRunModule.Spec("1x$HEAD_DIM", "bf16", "$work/wp_sin.bin"), + ) + for (i in 0 until N_LAYERS) { + specs += TorqRunModule.Spec(self, "bf16", "$work/wp_sk_$i.bin") + specs += TorqRunModule.Spec(self, "bf16", "$work/wp_sv_$i.bin") + specs += TorqRunModule.Spec(cross, "bf16", crossK[i]) + specs += TorqRunModule.Spec(cross, "bf16", crossV[i]) + } + return specs + } + + /** INTERLEAVED sign-baked cos/sin `[headDim]` at [position] (port of RoPE.buildInterleavedCosSin; + * Moonshine: partial rotary → rotaryDim 32, freqDenom = rotaryDim). */ + private fun interleavedCosSin(position: Int): Pair { + val half = HEAD_DIM / 2 + val c = FloatArray(HEAD_DIM); val s = FloatArray(HEAD_DIM) + for (i in 0 until half) { + val rot = i < HALF_ROTARY + val cv = if (rot) cos(position * (1.0 / ROPE_BASE.toDouble().pow(2.0 * i / ROTARY_DIM))).toFloat() else 1f + val sv = if (rot) sin(position * (1.0 / ROPE_BASE.toDouble().pow(2.0 * i / ROTARY_DIM))).toFloat() else 0f + c[2 * i] = cv; c[2 * i + 1] = cv + s[2 * i] = -sv; s[2 * i + 1] = sv + } + return c to s + } + + /** Argmax over a `[1,1,vocab]` little-endian f32 logits buffer (single decode position). */ + private fun argmaxF32(b: ByteArray): Int { + var best = 0 + var bestV = Float.NEGATIVE_INFINITY + for (v in 0 until VOCAB) { + val o = v * 4 + val bits = (b[o].toInt() and 0xFF) or ((b[o + 1].toInt() and 0xFF) shl 8) or + ((b[o + 2].toInt() and 0xFF) shl 16) or ((b[o + 3].toInt() and 0xFF) shl 24) + val f = Float.fromBits(bits) + if (f > bestV) { bestV = f; best = v } + } + return best + } + + // Output/staging file lists: 2*N per-layer K/V (block order, two per block). + private fun kvFiles(tag: String): List = (0 until 2 * N_LAYERS).map { "$work/${tag}_$it.bin" } + private fun kOf(files: List, layer: Int): String = files[2 * layer + if (kFirstInOutput) 0 else 1] + private fun vOf(files: List, layer: Int): String = files[2 * layer + if (kFirstInOutput) 1 else 0] + + private companion object { + const val DIM = 288 + const val VOCAB = 32768 + const val N_LAYERS = 6 + const val N_HEADS = 8 + const val HEAD_DIM = 36 + const val ROTARY_DIM = 32 // headDim 36 * partialRotary 0.9 -> 32 (even) + const val HALF_ROTARY = ROTARY_DIM / 2 + const val ROPE_BASE = 10000f + const val FRAMES = 207 + const val INPUT_LEN = 80000 + const val START = 1 + const val END = 2 + const val MAX_NEW_TOKENS = 30 + // Per-block K-vs-V output order; false = (V,K) per the return-SSA analysis. Flip if the first + // board run mis-transcribes (the only thing this controls). See docs/GEMMA-KV-BOARD-LOOP.md. + const val kFirstInOutput = false + } +} diff --git a/sl2610-function-calling/src/linuxArm64Main/kotlin/voicecc/asr/MoonshineRunner.kt b/sl2610-function-calling/src/linuxArm64Main/kotlin/voicecc/asr/MoonshineRunner.kt index 6ca3bd2..f50c6bc 100644 --- a/sl2610-function-calling/src/linuxArm64Main/kotlin/voicecc/asr/MoonshineRunner.kt +++ b/sl2610-function-calling/src/linuxArm64Main/kotlin/voicecc/asr/MoonshineRunner.kt @@ -1,5 +1,9 @@ package voicecc.asr +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.toKString +import platform.posix.getenv + /** * Board-side Moonshine ASR on the Torq NPU — now fully Python-free. Delegates to * [MoonshineDecoder], which drives the prebuilt Synaptics vmfbs (preprocessor on @@ -9,13 +13,19 @@ package voicecc.asr * Replaces the previous `popen` of `scripts/moonshine_npu.py` (venv + onnxruntime * + torq.runtime). No Python interpreter is involved; `torq-run-module` and its * `.so`s are native ELF artifacts that ship inside the vendor wheel. + * + * `MOONSHINE_KV=1` opts into the KV-cache 2-graph decode ([MoonshineKvDecoder], perf-program + * Phase 6) instead of the default fixed-seq re-decode. Both expose the same `transcribe`. */ +@OptIn(ExperimentalForeignApi::class) public class MoonshineRunner { - private val decoder = MoonshineDecoder() + private val useKv = getenv("MOONSHINE_KV")?.toKString()?.trim() == "1" + private val reDecoder = if (!useKv) MoonshineDecoder() else null + private val kvDecoder = if (useKv) MoonshineKvDecoder() else null /** Transcribe a wav on the NPU; returns the text (or null on failure). */ public fun transcribe(wavPath: String): String? = - runCatching { decoder.transcribe(wavPath) } + runCatching { (kvDecoder?.transcribe(wavPath)) ?: reDecoder?.transcribe(wavPath) } .onFailure { println("[asr] ${it.message}") } .getOrNull() }