Add Laya request preprocessing

.github/workflows/ci.yml

@@ -26,6 +26,18 @@ jobs:
         run: cargo clippy --workspace --locked --all-targets -- -D warnings
       - name: Test
         run: cargo test --workspace --locked
+      - name: Official Laya packing parity
+        env:
+          LAYA_TOKENIZER: ${{ runner.temp }}/laya-tokenizer.json
+          LAYA_PACKING_ORACLE: ${{ runner.temp }}/laya-packing.json
+        run: |
+          curl --fail --location --retry 3 \
+            https://huggingface.co/convaiinnovations/laya/resolve/55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851/tokenizer/tokenizer.json \
+            --output "$LAYA_TOKENIZER"
+          curl --fail --location --retry 3 \
+            https://raw.githubusercontent.com/linear3735/system1-omni/5e4dd4215c925ebd93bb9ce4097b27bd6375f7c0/recipe/laya/native/packing-golden.json \
+            --output "$LAYA_PACKING_ORACLE"
+          cargo test --locked -p omni-laya --test packing -- --ignored
       - name: Build
         run: cargo build --workspace --release --locked
 

recipe/laya/README.md

@@ -56,3 +56,47 @@ if the worker requires a bearer token.
 
 See the [frontend documentation](../../src/frontend/README.md) for configuration
 and transport behavior.
+
+## Native CPU packing check
+
+The `omni-laya` preprocessor packs English Laya 0.3.20 requests without weights
+or a GPU.
+
+```sh
+cargo test --locked -p omni-laya --test preprocess
+```
+
+Native callers use `Request::from_json(&str)` for a single top-level JSON request,
+or `Request::from_value(Value)` for an existing structured value. `Request`
+retains its public fields and `Serialize`; it does not implement generic
+`Deserialize`. The JSON entry checks the complete request against serde_json's
+default nesting limit. The value entry preserves existing nested values without
+reparsing. Both preserve literal private Number/RawValue object keys and reject
+unknown request fields.
+
+The normal tests cover validation, JSON rendering, question and option order,
+and truncation with a small tokenizer:
+Pass raw JSON directly to `from_json`.
+
+For the official 17-case comparison, use the same pinned inputs as CPU CI.
+The test checks both files by SHA-256 before comparing:
+
+```sh
+LAYA_PACKING_DIR=$(mktemp -d)
+export LAYA_TOKENIZER="$LAYA_PACKING_DIR/tokenizer.json"
+export LAYA_PACKING_ORACLE="$LAYA_PACKING_DIR/packing-golden.json"
+curl --fail --location --retry 3 \
+  https://huggingface.co/convaiinnovations/laya/resolve/55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851/tokenizer/tokenizer.json \
+  --output "$LAYA_TOKENIZER"
+curl --fail --location --retry 3 \
+  https://raw.githubusercontent.com/linear3735/system1-omni/5e4dd4215c925ebd93bb9ce4097b27bd6375f7c0/recipe/laya/native/packing-golden.json \
+  --output "$LAYA_PACKING_ORACLE"
+cargo test --locked -p omni-laya --test packing -- --ignored
+```
+
+Existing copies of these pinned files can be supplied through `LAYA_TOKENIZER`
+and `LAYA_PACKING_ORACLE` instead. The comparison covers every token, marker,
+question type, row length, question order and usage count; it excludes backend
+padding and bucket dimensions. The [reference generator and inputs](https://github.com/linear3735/system1-omni/tree/5e4dd4215c925ebd93bb9ce4097b27bd6375f7c0/recipe/laya/native)
+use `laya==0.3.20`. Packing parity does not measure model quality or execute
+native model inference.

src/models/cua_s1/native/Cargo.toml

@@ -18,7 +18,7 @@ libloading = "0.8"
 memmap2 = "0.9.9"
 safetensors = "0.8.0"
 serde = "1"
-serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] }
+serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order", "raw_value"] }
 # the onig regex backend, as in the Python tokenizers wheel
 tokenizers = { version = "=0.22.2", default-features = false, features = ["onig"] }
 tokio = { version = "1.49.0", features = ["macros", "net", "rt-multi-thread", "sync"] }

src/models/cua_s1/native/src/json.rs

@@ -10,14 +10,14 @@ use std::fmt::Write as _;
 use std::io;
 
 use serde::Serialize;
-use serde::de::{self, Deserializer, MapAccess, SeqAccess, Visitor};
+use serde::de::{self, DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor};
 use serde_json::{Map, Number, Value};
 
 /// Decode a request body into its top-level object; the error is the 400 message.
 pub fn parse(raw: &[u8]) -> Result<Map<String, Value>, String> {
     let mut de = serde_json::Deserializer::from_slice(raw);
-    let value = de
-        .deserialize_any(NoDuplicates)
+    let value = NoDuplicates(0)
+        .deserialize(&mut de)
         .and_then(|v| de.end().map(|()| v))
         .map_err(|e| format!("request body is not valid JSON: {e}"))?;
     match value {
@@ -27,16 +27,37 @@ pub fn parse(raw: &[u8]) -> Result<Map<String, Value>, String> {
 }
 
 /// Builds a `Value` like serde_json does, but fails on a repeated key.
-struct NoDuplicates;
+struct NoDuplicates(usize);
 
-impl<'de> de::Deserialize<'de> for Wrapped {
-    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
-        d.deserialize_any(NoDuplicates).map(Wrapped)
+impl<'de> DeserializeSeed<'de> for NoDuplicates {
+    type Value = Value;
+
+    fn deserialize<D: Deserializer<'de>>(self, d: D) -> Result<Value, D::Error> {
+        let raw = <&serde_json::value::RawValue as de::Deserialize>::deserialize(d)?;
+        let text = raw.get();
+        // Another workspace member may enable arbitrary_precision. Read number
+        // tokens directly so its private map encoding cannot become user data.
+        if matches!(text.as_bytes()[0], b'-' | b'0'..=b'9') {
+            if text != "-0" {
+                if let Ok(n) = text.parse::<i64>() {
+                    return self.visit_i64(n);
+                }
+                if let Ok(n) = text.parse::<u64>() {
+                    return self.visit_u64(n);
+                }
+            }
+            let n = serde_json::from_str::<f64>(text).map_err(de::Error::custom)?;
+            return self.visit_f64(n);
+        }
+        if self.0 >= 127 && matches!(text.as_bytes()[0], b'{' | b'[') {
+            return Err(de::Error::custom("recursion limit exceeded"));
+        }
+        serde_json::Deserializer::from_str(text)
+            .deserialize_any(self)
+            .map_err(de::Error::custom)
     }
 }
 
-struct Wrapped(Value);
-
 impl<'de> Visitor<'de> for NoDuplicates {
     type Value = Value;
 
@@ -68,15 +89,15 @@ impl<'de> Visitor<'de> for NoDuplicates {
     }
     fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Value, A::Error> {
         let mut items = Vec::new();
-        while let Some(Wrapped(v)) = seq.next_element()? {
+        while let Some(v) = seq.next_element_seed(NoDuplicates(self.0 + 1))? {
             items.push(v);
         }
         Ok(Value::Array(items))
     }
     fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<Value, A::Error> {
         let mut obj = Map::new();
         while let Some(key) = map.next_key::<String>()? {
-            let Wrapped(v) = map.next_value()?;
+            let v = map.next_value_seed(NoDuplicates(self.0 + 1))?;
             if obj.contains_key(&key) {
                 return Err(de::Error::custom(format_args!(
                     "duplicate key {}",
@@ -107,6 +128,14 @@ pub fn dumps(value: &Value) -> String {
 struct PyFormatter;
 
 impl serde_json::ser::Formatter for PyFormatter {
+    fn write_number_str<W: ?Sized + io::Write>(&mut self, w: &mut W, n: &str) -> io::Result<()> {
+        if n.contains(['.', 'e', 'E']) {
+            let x = serde_json::from_str::<f64>(n).map_err(io::Error::other)?;
+            self.write_f64(w, x)
+        } else {
+            w.write_all(n.as_bytes())
+        }
+    }
     fn begin_array_value<W: ?Sized + io::Write>(
         &mut self,
         w: &mut W,
@@ -243,3 +272,7 @@ mod tests {
         assert!(parse(b"{\"a\": \"\xff\"}").is_err());
     }
 }
+
+#[cfg(test)]
+#[path = "../../../../../tests/cua_s1/json.rs"]
+mod json_regression_tests;

src/models/laya/Cargo.toml

@@ -10,7 +10,8 @@ half = "2"
 memmap2 = "0.9"
 safetensors = "0.6"
 serde = { version = "1", features = ["derive"] }
-serde_json = "1"
+serde_json = { version = "1", features = ["preserve_order", "arbitrary_precision", "raw_value"] }
+tokenizers = { version = "0.23.2", default-features = false, features = ["fancy-regex"] }
 
 [dev-dependencies]
 sha2 = "0.10"
@@ -23,3 +24,11 @@ path = "../../../tests/laya/checkpoint.rs"
 [[test]]
 name = "weights"
 path = "../../../tests/laya/weights.rs"
+
+[[test]]
+name = "preprocess"
+path = "../../../tests/laya/preprocess.rs"
+
+[[test]]
+name = "packing"
+path = "../../../tests/laya/packing.rs"

src/models/laya/README.md

@@ -8,6 +8,18 @@ The `omni-laya` crate currently reads and checks the English Laya 0.3.20 checkpo
 
 Keep checkpoint files unchanged while `Weights` holds a read-only memory mapping. This crate does not yet execute inference.
 
+`Preprocessor::load` reads a tokenizer JSON file. `prepare` packs English `choice`, `score` and `noul` questions into ordered token rows, option-marker positions and type IDs. Rows follow Laya 0.3.20's 512-token limit and 192-token head budget. Conversation lists keep the newest state tokens; other state values keep the beginning. The result includes normalized criteria for later decoding and the total input-token usage. Backends own padding, batching and resource limits.
+
+Build a request with `Request::from_json(&str)` for one top-level JSON object, or
+`Request::from_value(Value)` for an already constructed value. The JSON entry
+preserves object order and arbitrary-size integers, treats serde_json's private
+Number/RawValue keys as ordinary user keys, and applies its default nesting limit
+to the complete request. The value entry moves state and questions without
+reparsing or adding a depth limit. Both reject unknown request fields and require
+state and an object of questions. Public fields and `Serialize` remain available;
+`Request` does not implement generic `Deserialize`, so use these explicit entries
+instead of `serde_json::from_str::<Request>` or `serde_json::from_value::<Request>`.
+
 ## CPU checks
 
 The normal workspace tests cover configuration errors, malformed tensors, inventory mismatches and conversion boundaries without downloading weights.
@@ -23,6 +35,8 @@ cargo test --release --locked -p omni-laya --test weights -- --ignored
 
 These two CPU tests check all 206 tensor names and shapes, 618 conversion hashes, and the legacy temperature buffer. The normal CI job skips them because it does not download the full checkpoint.
 
+The normal tests also check input validation, question and option order, truncation and JSON rendering with a small test tokenizer. The [Laya recipe](../../../recipe/laya/README.md#native-cpu-packing-check) provides the CPU packing validation commands and pinned inputs for the official 17-case comparison. No weights or GPU are needed; packing parity does not measure model quality.
+
 ## Python worker
 
 The Python worker serves LAYA through laya-serve on CPU and Apple Silicon (PyTorch MPS,

src/models/laya/src/lib.rs

@@ -1,2 +1,3 @@
 pub mod config;
+pub mod preprocess;
 pub mod weights;