From 9c5539de72389d1e48f7652df1bd91358a5e40d4 Mon Sep 17 00:00:00 2001
From: cacheline999 <326908201+cacheline999@users.noreply.github.com>
Date: Sun, 27 Sep 2026 23:08:04 +0800
Subject: [PATCH 1/5] [Deps] Require laya 0.3.9 and drop the local weight-init
 skip

laya 0.3.9 builds the encoder under transformers' no_init_weights inside
laya.load and loads the checkpoint with strict=True, so the
without_weight_init() wrapper from #22 and its test stubs are no longer
needed. The lock moves laya from 0.3.5 to 0.3.9, the first release with
the skip; later releases change MPS precision and are left for their
own bump.

The system_one config-error test pins the version lookup, so it passes
with the laya extra installed as well as on the core install.

Closes #28
---
 CHANGELOG.md                          |  4 +-
 pyproject.toml                        |  2 +-
 s1a/decision_models/laya.py           | 18 +-------
 tests/test_decision_models_factory.py |  5 +--
 tests/test_decision_models_laya.py    | 64 ++++-----------------------
 uv.lock                               |  8 ++--
 6 files changed, 17 insertions(+), 84 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e1a3c9d..9b800f8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -32,8 +32,8 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); ver
 
 ### Changed
 
-- `--model laya` loads in about 3 s instead of about 35 s: the encoder is built with transformers' weight init
-  off, since the checkpoint replaces every weight. Weights and answers are unchanged.
+- `--model laya` loads in about 3 s instead of about 35 s: the `laya` extra now needs laya 0.3.9 or later, which
+  builds the encoder with transformers' weight init off, since the checkpoint replaces every weight.
 - `--model` picks the model on every agent, on `decide` and on `probe`: `jev`, `laya`, `cua`, `llm`, `random` or
   `rule`. The results table's column, the replay page's badge data and a browser run's `answer.json` name it
   `model` as well; the replay still reads the `slot` key of records written by 0.1.0.
diff --git a/pyproject.toml b/pyproject.toml
index caf1efd..d271688 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -40,7 +40,7 @@ alfworld-visual = [
     "torchvision>=0.15",
 ]
 report = ["pillow>=10", "playwright>=1.45"]  # evals.replay: pages, GIFs; never imported by a runner
-laya = ["laya>=0.3.4"]  # the in-process decision model behind --model laya; pulls torch and transformers
+laya = ["laya>=0.3.9"]  # the in-process decision model behind --model laya; pulls torch and transformers
 cua = [  # Cua-S1 Nano behind --model cua; pinned to the Cua PR that ships the checkpoints (trycua/cua#4023), pulls torch
     "cua-s1 @ git+https://github.com/trycua/cua.git@aea61b6eb97e2d8c0f6f71eb804e5769fe910af4#subdirectory=libs/cua-s1/python",
     "huggingface-hub>=0.24",
diff --git a/s1a/decision_models/laya.py b/s1a/decision_models/laya.py
index 1337592..a6c27fa 100644
--- a/s1a/decision_models/laya.py
+++ b/s1a/decision_models/laya.py
@@ -7,10 +7,8 @@
 from __future__ import annotations
 
 import asyncio
-import importlib
 import os
 import time
-from contextlib import AbstractContextManager, nullcontext
 from importlib import metadata
 from typing import Any
 
@@ -51,19 +49,6 @@ def check_window(usage: Usage, questions: int, max_len: int, hint: str) -> None:
         )
 
 
-def without_weight_init() -> AbstractContextManager[Any]:
-    """transformers' ``no_init_weights``. ``laya.load`` builds the encoder from its config, which draws every weight
-    at random (about 30 s of the load on CPU), then loads the checkpoint over all of them with ``strict=True``, so
-    the draw is thrown away. The helper sits in ``transformers.initialization`` from 5.0 and in
-    ``transformers.modeling_utils`` before; without either the load runs as it is."""
-    for module in ("transformers.initialization", "transformers.modeling_utils"):
-        try:
-            return importlib.import_module(module).no_init_weights()
-        except (ImportError, AttributeError):
-            continue
-    return nullcontext()
-
-
 class LayaModel(DecisionModel):
     """Laya's ``Agent`` (or anything with ``system_one(state, questions)`` and a ``cfg``) behind the interface."""
 
@@ -123,8 +108,7 @@ def from_env(cls) -> "LayaModel":
             ) from exc
         model = os.getenv("LAYA_MODEL") or LAYA_DEFAULT_MODEL
         subfolder = os.getenv("LAYA_SUBFOLDER") or None
-        with without_weight_init():
-            agent = laya.load(model, device=os.getenv("LAYA_DEVICE") or None, subfolder=subfolder)
+        agent = laya.load(model, device=os.getenv("LAYA_DEVICE") or None, subfolder=subfolder)
         if not callable(getattr(agent, "system_one", None)):
             try:
                 version = metadata.version("laya")
diff --git a/tests/test_decision_models_factory.py b/tests/test_decision_models_factory.py
index 79f5162..d94a4b3 100644
--- a/tests/test_decision_models_factory.py
+++ b/tests/test_decision_models_factory.py
@@ -5,7 +5,6 @@
 
 import os
 import sys
-from contextlib import nullcontext
 from types import SimpleNamespace
 from unittest import TestCase
 from unittest.mock import patch
@@ -32,9 +31,7 @@ def test_every_name_builds_its_class(self) -> None:
         fake_laya = SimpleNamespace(
             load=lambda *a, **k: SimpleNamespace(cfg={}, system_one=lambda state, questions: {})
         )
-        # transformers' helper stubbed so torch is not imported inside patch.dict: see TestFromEnv in the laya tests.
-        modules = {"laya": fake_laya, "transformers.initialization": SimpleNamespace(no_init_weights=nullcontext)}
-        with patch.dict(sys.modules, modules), patch.dict(os.environ, {"LAYA_SUBFOLDER": ""}):
+        with patch.dict(sys.modules, {"laya": fake_laya}), patch.dict(os.environ, {"LAYA_SUBFOLDER": ""}):
             self.assertIsInstance(build_model("laya"), LayaModel)
         self.assertIsInstance(build_model("random", seed=3), RandomModel)
         rule = build_model("rule", rule=("always-inc", lambda state, options: "inc"))
diff --git a/tests/test_decision_models_laya.py b/tests/test_decision_models_laya.py
index 992924d..6c91122 100644
--- a/tests/test_decision_models_laya.py
+++ b/tests/test_decision_models_laya.py
@@ -1,14 +1,12 @@
 # coding: utf-8
 """``LayaModel`` over a fake ``laya.Agent`` (no torch): the contract, the question mapping, the error wrap,
-the filled-window error, and ``from_env`` with and without the extra and with the weight init off."""
+the filled-window error, and ``from_env`` with and without the extra."""
 
 from __future__ import annotations
 
 import os
 import sys
 import time
-from collections.abc import Iterator
-from contextlib import contextmanager, nullcontext
 from types import SimpleNamespace
 from typing import Any
 from unittest import IsolatedAsyncioTestCase, TestCase
@@ -163,14 +161,6 @@ async def test_the_window_scales_with_the_number_of_questions(self) -> None:
 
 
 class TestFromEnv(TestCase):
-    def setUp(self) -> None:
-        # A stand-in for transformers' helper, so no test imports torch: patch.dict drops a torch imported inside it
-        # from sys.modules, and importing torch a second time in one process crashes it.
-        helper = {"transformers.initialization": SimpleNamespace(no_init_weights=nullcontext)}
-        stub = patch.dict(sys.modules, helper)
-        stub.start()
-        self.addCleanup(stub.stop)
-
     def test_without_the_extra_it_is_a_config_error_naming_the_extra(self) -> None:
         with patch.dict(sys.modules, {"laya": None}):
             with self.assertRaises(BaseError) as caught:
@@ -180,12 +170,17 @@ def test_without_the_extra_it_is_a_config_error_naming_the_extra(self) -> None:
 
     def test_an_agent_without_system_one_is_a_config_error_naming_the_method(self) -> None:
         fake_laya = SimpleNamespace(load=lambda *a, **k: SimpleNamespace(cfg={}))
-        with patch.dict(sys.modules, {"laya": fake_laya}), patch.dict(os.environ, {"LAYA_SUBFOLDER": ""}):
+        # The installed version is pinned so the message reads the same with and without the extra.
+        with (
+            patch.dict(sys.modules, {"laya": fake_laya}),
+            patch.dict(os.environ, {"LAYA_SUBFOLDER": ""}),
+            patch.object(laya_module.metadata, "version", return_value="0.3.0"),
+        ):
             with self.assertRaises(BaseError) as caught:
                 LayaModel.from_env()
         self.assertEqual(caught.exception.status, StatusCode.MODEL_SERVICE_CONFIG_ERROR)
         self.assertIn("system_one", str(caught.exception))
-        self.assertIn("laya unknown", str(caught.exception))
+        self.assertIn("laya 0.3.0", str(caught.exception))
 
     def test_the_env_names_the_checkpoint_and_overrides_the_window(self) -> None:
         loads: list[tuple[Any, ...]] = []
@@ -207,49 +202,6 @@ def load(model: str, device: Any = None, token: Any = None, subfolder: Any = Non
         self.assertEqual(decision_model.model, "convaiinnovations/laya/multilingual")
         self.assertEqual(decision_model._agent.cfg, {"max_len": 1024, "head_max_len": 512})
 
-    def test_the_checkpoint_loads_with_the_weight_init_off(self) -> None:
-        events: list[str] = []
-
-        @contextmanager
-        def no_init_weights() -> Iterator[None]:
-            events.append("off")
-            yield
-            events.append("on")
-
-        def load(*args: Any, **kwargs: Any) -> FakeLayaAgent:
-            events.append("load")
-            return FakeLayaAgent()
-
-        modules = {
-            "laya": SimpleNamespace(load=load),
-            "transformers.initialization": SimpleNamespace(no_init_weights=no_init_weights),
-        }
-        with patch.dict(sys.modules, modules), patch.dict(os.environ, {"LAYA_SUBFOLDER": ""}):
-            LayaModel.from_env()
-        self.assertEqual(events, ["off", "load", "on"])
-
-    def test_the_4x_location_of_the_helper_is_used_when_the_5x_one_is_missing(self) -> None:
-        @contextmanager
-        def no_init_weights() -> Iterator[str]:
-            yield "4.x"
-
-        modules = {
-            "transformers.initialization": None,
-            "transformers.modeling_utils": SimpleNamespace(no_init_weights=no_init_weights),
-        }
-        with patch.dict(sys.modules, modules), laya_module.without_weight_init() as entered:
-            self.assertEqual(entered, "4.x")
-
-    def test_without_the_helper_the_checkpoint_still_loads(self) -> None:
-        modules = {
-            "laya": SimpleNamespace(load=lambda *a, **k: FakeLayaAgent()),
-            "transformers.initialization": None,
-            "transformers.modeling_utils": SimpleNamespace(),
-        }
-        with patch.dict(sys.modules, modules), patch.dict(os.environ, {"LAYA_SUBFOLDER": ""}):
-            decision_model = LayaModel.from_env()
-        self.assertIsInstance(decision_model._agent, FakeLayaAgent)
-
     def test_the_defaults_when_the_env_is_empty(self) -> None:
         env = {"LAYA_MODEL": "", "LAYA_SUBFOLDER": "", "LAYA_DEVICE": "", "LAYA_MAX_LEN": "", "LAYA_HEAD_MAX_LEN": ""}
         with patch.dict(sys.modules, {"laya": SimpleNamespace(load=lambda *a, **k: FakeLayaAgent())}):
diff --git a/uv.lock b/uv.lock
index cd62ef3..6fdab1f 100644
--- a/uv.lock
+++ b/uv.lock
@@ -2287,7 +2287,7 @@ wheels = [
 
 [[package]]
 name = "laya"
-version = "0.3.5"
+version = "0.3.9"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "huggingface-hub" },
@@ -2297,9 +2297,9 @@ dependencies = [
     { name = "torch" },
     { name = "transformers" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/f7/f1/12459ecda42123f93da36f9f1e0c9b7b9191648fa68e7e0b09d292cc6dce/laya-0.3.5.tar.gz", hash = "sha256:5e8a4c2b38dbddc0febe7443f26f74fd9d7571fb172648b1481830c667d59219", size = 69616, upload-time = "2026-09-21T18:20:10.029Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/4f/89/6eec1f50cbc421fa5b1733d166387eae6bc0dc222a8928ac875c271c8cf1/laya-0.3.9.tar.gz", hash = "sha256:3d255a778a1c70e2ed1fb4129147ccbb7fa96f3d0a0c910b2f41ddacb49ea5f6", size = 201267, upload-time = "2026-09-23T14:55:16.694Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/ec/1c/a9903d5c3c51579f45f9656f733c8a2d985b16a3cb7594edf45369d1f995/laya-0.3.5-py3-none-any.whl", hash = "sha256:4c57f64cbaf893bb5c7b4affddc2bf21a819f55df51941689f11868583be2903", size = 41658, upload-time = "2026-09-21T18:20:08.641Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/cc/75f6b030f26b78a0ac0165c455866ab897eee251f7edb115733d4675fcf6/laya-0.3.9-py3-none-any.whl", hash = "sha256:8080d99792867096c970b1b24464e888f3c37d902f2c947be1f8baa3677c9717", size = 93224, upload-time = "2026-09-23T14:55:15.288Z" },
 ]
 
 [[package]]
@@ -5204,7 +5204,7 @@ requires-dist = [
     { name = "httpx", specifier = ">=0.28" },
     { name = "huggingface-hub", marker = "extra == 'cua'", specifier = ">=0.24" },
     { name = "jsonschema", marker = "extra == 'dev'", specifier = ">=4.18" },
-    { name = "laya", marker = "extra == 'laya'", specifier = ">=0.3.4" },
+    { name = "laya", marker = "extra == 'laya'", specifier = ">=0.3.9" },
     { name = "mcp", specifier = ">=1.26" },
     { name = "opencv-python-headless", marker = "extra == 'alfworld-visual'", specifier = ">=4.10" },
     { name = "openjiuwen", git = "https://github.com/ThinkFlowLab/agent-core?rev=jj-0.1.0" },

From d26d762899068f03eec14bb4a52c002c9820e297 Mon Sep 17 00:00:00 2001
From: cacheline999 <326908201+cacheline999@users.noreply.github.com>
Date: Sun, 4 Oct 2026 17:18:38 +0800
Subject: [PATCH 2/5] [Docs] Say what the laya 0.3.9 load skips, without a load
 time for one machine

---
 CHANGELOG.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9b800f8..68dc642 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -32,8 +32,9 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); ver
 
 ### Changed
 
-- `--model laya` loads in about 3 s instead of about 35 s: the `laya` extra now needs laya 0.3.9 or later, which
-  builds the encoder with transformers' weight init off, since the checkpoint replaces every weight.
+- `--model laya` no longer draws the encoder's random weights before the checkpoint replaces them, which took most
+  of a load of about 40 s on CPU. The `laya` extra now needs laya 0.3.9 or later, which skips the draw itself.
+  Weights and answers are unchanged.
 - `--model` picks the model on every agent, on `decide` and on `probe`: `jev`, `laya`, `cua`, `llm`, `random` or
   `rule`. The results table's column, the replay page's badge data and a browser run's `answer.json` name it
   `model` as well; the replay still reads the `slot` key of records written by 0.1.0.

From a5e6ab3d7f90953ec8e2169954d5b938299095de Mon Sep 17 00:00:00 2001
From: cacheline999 <326908201+cacheline999@users.noreply.github.com>
Date: Mon, 5 Oct 2026 00:32:30 +0800
Subject: [PATCH 3/5] [Deps] Lock laya 0.3.20 and keep MPS requests in fp32

uv lock --upgrade-package laya, as #28 asks, pinned to 0.3.20, the release
system1-omni's worker runs. Only the laya entry of the lock changes.

From 0.3.10 laya runs a request of five or more questions in fp16 on MPS.
Against fp32 that moved probabilities by up to 0.05 on the multilingual
checkpoint and flipped 2 of 180 decisions, and on an M1 Pro it was slower for
short states. LayaModel.from_env now raises the agent's mps_amp_min_rows so
such requests stay in fp32, unless LAYA_MPS_AMP_MIN_ROWS is set. With that,
answers match main on CPU and MPS for both cached checkpoints.

Also tests the 'laya unknown' branch, which the pinned version lookup had
stopped covering.
---
 .env.example                       |  1 +
 CHANGELOG.md                       |  6 ++++--
 docs/configuration.md              |  1 +
 s1a/decision_models/laya.py        |  8 +++++++-
 tests/test_decision_models_laya.py | 30 ++++++++++++++++++++++++++++++
 uv.lock                            |  6 +++---
 6 files changed, 46 insertions(+), 6 deletions(-)

diff --git a/.env.example b/.env.example
index 4afe81f..1a9856b 100644
--- a/.env.example
+++ b/.env.example
@@ -33,6 +33,7 @@ MODEL_NAME=google/gemini-2.5-flash
 # LAYA_DEVICE=cpu                    # cuda when available
 # LAYA_MAX_LEN=1024
 # LAYA_HEAD_MAX_LEN=512              # raise for choice questions with many options
+# LAYA_MPS_AMP_MIN_ROWS=5            # on MPS, fp16 from this many questions (laya's default); unset keeps fp32
 
 # ---- Served Laya (behind --model laya-served; a system1-omni worker or laya-serve, no extra needed) ----
 # LAYA_SERVED_URL=http://127.0.0.1:8000   # the worker; :8080 for the omni-jev frontend
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 68dc642..73f8c1b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -33,8 +33,10 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); ver
 ### Changed
 
 - `--model laya` no longer draws the encoder's random weights before the checkpoint replaces them, which took most
-  of a load of about 40 s on CPU. The `laya` extra now needs laya 0.3.9 or later, which skips the draw itself.
-  Weights and answers are unchanged.
+  of a load of about 40 s on CPU. The `laya` extra now needs laya 0.3.9 or later, which skips the draw itself,
+  and the lock moves from 0.3.5 to 0.3.20. Weights and answers are unchanged: laya 0.3.10 and later run a request
+  of five or more questions in fp16 on MPS, which moves the answers, so `--model laya` keeps such requests in fp32
+  unless `LAYA_MPS_AMP_MIN_ROWS` is set.
 - `--model` picks the model on every agent, on `decide` and on `probe`: `jev`, `laya`, `cua`, `llm`, `random` or
   `rule`. The results table's column, the replay page's badge data and a browser run's `answer.json` name it
   `model` as well; the replay still reads the `slot` key of records written by 0.1.0.
diff --git a/docs/configuration.md b/docs/configuration.md
index e804062..9772f74 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -32,6 +32,7 @@ Variables can be exported in your shell or placed in a `.env` file at the root o
 | `LAYA_DEVICE` | `laya` model | `(library default)` | PyTorch device for Laya model evaluation; passes None so the library selects CUDA, MPS, or CPU. |
 | `LAYA_MAX_LEN` | `laya` model | `(checkpoint default)` | Maximum token sequence length for Laya state representation; overrides checkpoint window only when set. |
 | `LAYA_HEAD_MAX_LEN` | `laya` model | `(checkpoint default)` | Maximum token sequence length for Laya decision head options; overrides checkpoint window only when set. |
+| `LAYA_MPS_AMP_MIN_ROWS` | `laya` model | *(unset: fp32)* | Laya's own variable: on MPS, requests with at least this many questions run in fp16. Unset, `--model laya` keeps every request in fp32, as on CPU; `5` is Laya's default. fp16 moves the probabilities and can flip a close decision. |
 | `LAYA_SERVED_URL` | `laya-served` model | *(unset, required)* | Base URL of a served Laya: the system1-omni worker (`http://127.0.0.1:8000`), its `omni-jev` frontend (`:8080`) or plain laya-serve. |
 | `LAYA_SERVED_MODEL` | `laya-served` model | `english` | Name of the served checkpoint to ask: `english`, `multilingual` or `typed-decisions`. A name the server does not know raises instead of being routed by language. |
 | `LAYA_SERVED_API_KEY` | `laya-served` model | *(unset)* | Bearer token, the server's `LAYA_API_KEY` when it sets one. |
diff --git a/s1a/decision_models/laya.py b/s1a/decision_models/laya.py
index a6c27fa..b7eda60 100644
--- a/s1a/decision_models/laya.py
+++ b/s1a/decision_models/laya.py
@@ -21,6 +21,7 @@
 
 LAYA_DEFAULT_MODEL = "convaiinnovations/laya"
 LAYA_DEFAULT_MAX_LEN = 512  # the window Laya assumes when a checkpoint config names none
+LAYA_MPS_FP32_ROWS = 10**9  # no request has this many questions, so Laya never switches to fp16 on MPS
 
 
 def laya_question(question: Question) -> Json:
@@ -98,7 +99,8 @@ def _check_the_window(self, usage: Usage, questions: int) -> None:
     @classmethod
     def from_env(cls) -> "LayaModel":
         """``LAYA_MODEL`` (a hub id or a path), ``LAYA_SUBFOLDER``, ``LAYA_DEVICE``; ``LAYA_MAX_LEN`` and
-        ``LAYA_HEAD_MAX_LEN`` override the checkpoint's window."""
+        ``LAYA_HEAD_MAX_LEN`` override the checkpoint's window. On MPS the model answers in fp32 whatever the
+        number of questions, unless ``LAYA_MPS_AMP_MIN_ROWS`` (Laya's own variable) is set."""
         try:
             import laya
         except ImportError as exc:
@@ -121,6 +123,10 @@ def from_env(cls) -> "LayaModel":
                     "the s1a laya model needs that method"
                 ),
             )
+        # From 0.3.10 Laya runs a request of five or more questions in fp16 on MPS. That moves the answers, enough
+        # to flip a close decision, so one browser episode would mix both precisions.
+        if not os.getenv("LAYA_MPS_AMP_MIN_ROWS") and hasattr(agent, "mps_amp_min_rows"):
+            agent.mps_amp_min_rows = LAYA_MPS_FP32_ROWS
         for key, variable in (("max_len", "LAYA_MAX_LEN"), ("head_max_len", "LAYA_HEAD_MAX_LEN")):
             value = os.getenv(variable)
             if value:
diff --git a/tests/test_decision_models_laya.py b/tests/test_decision_models_laya.py
index 6c91122..99d6da3 100644
--- a/tests/test_decision_models_laya.py
+++ b/tests/test_decision_models_laya.py
@@ -182,6 +182,36 @@ def test_an_agent_without_system_one_is_a_config_error_naming_the_method(self) -
         self.assertIn("system_one", str(caught.exception))
         self.assertIn("laya 0.3.0", str(caught.exception))
 
+    def test_a_laya_without_package_metadata_is_named_unknown(self) -> None:
+        fake_laya = SimpleNamespace(load=lambda *a, **k: SimpleNamespace(cfg={}))
+        with (
+            patch.dict(sys.modules, {"laya": fake_laya}),
+            patch.dict(os.environ, {"LAYA_SUBFOLDER": ""}),
+            patch.object(laya_module.metadata, "version", side_effect=laya_module.metadata.PackageNotFoundError),
+        ):
+            with self.assertRaises(BaseError) as caught:
+                LayaModel.from_env()
+        self.assertIn("laya unknown", str(caught.exception))
+
+    def test_mps_stays_in_fp32_unless_the_laya_variable_is_set(self) -> None:
+        def from_env(env: dict[str, str]) -> Any:
+            agent = FakeLayaAgent()
+            agent.mps_amp_min_rows = 5  # what laya sets from 0.3.10: fp16 from five questions on MPS
+            fake_laya = SimpleNamespace(load=lambda *a, **k: agent)
+            env = {"LAYA_SUBFOLDER": "", "LAYA_MPS_AMP_MIN_ROWS": "", **env}
+            with patch.dict(sys.modules, {"laya": fake_laya}), patch.dict(os.environ, env):
+                return LayaModel.from_env()._agent
+
+        self.assertEqual(from_env({}).mps_amp_min_rows, laya_module.LAYA_MPS_FP32_ROWS)
+        self.assertEqual(from_env({"LAYA_MPS_AMP_MIN_ROWS": "5"}).mps_amp_min_rows, 5)
+
+    def test_a_laya_before_the_mps_gate_gets_no_such_attribute(self) -> None:
+        agent = FakeLayaAgent()  # laya 0.3.9 has no mps_amp_min_rows
+        with patch.dict(sys.modules, {"laya": SimpleNamespace(load=lambda *a, **k: agent)}):
+            with patch.dict(os.environ, {"LAYA_SUBFOLDER": "", "LAYA_MPS_AMP_MIN_ROWS": ""}):
+                LayaModel.from_env()
+        self.assertFalse(hasattr(agent, "mps_amp_min_rows"))
+
     def test_the_env_names_the_checkpoint_and_overrides_the_window(self) -> None:
         loads: list[tuple[Any, ...]] = []
 
diff --git a/uv.lock b/uv.lock
index 6fdab1f..0783eff 100644
--- a/uv.lock
+++ b/uv.lock
@@ -2287,7 +2287,7 @@ wheels = [
 
 [[package]]
 name = "laya"
-version = "0.3.9"
+version = "0.3.20"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "huggingface-hub" },
@@ -2297,9 +2297,9 @@ dependencies = [
     { name = "torch" },
     { name = "transformers" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/4f/89/6eec1f50cbc421fa5b1733d166387eae6bc0dc222a8928ac875c271c8cf1/laya-0.3.9.tar.gz", hash = "sha256:3d255a778a1c70e2ed1fb4129147ccbb7fa96f3d0a0c910b2f41ddacb49ea5f6", size = 201267, upload-time = "2026-09-23T14:55:16.694Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/31/82/0964e13e1a67ae4ac2824c3115470d31cd149a065d971a5dec6e4049ca23/laya-0.3.20.tar.gz", hash = "sha256:692de1346cf0239bb7bbcffdf0cfbd538c24834f8bc61e9b60f4106465c033c9", size = 275028, upload-time = "2026-09-24T05:41:07.653Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/d0/cc/75f6b030f26b78a0ac0165c455866ab897eee251f7edb115733d4675fcf6/laya-0.3.9-py3-none-any.whl", hash = "sha256:8080d99792867096c970b1b24464e888f3c37d902f2c947be1f8baa3677c9717", size = 93224, upload-time = "2026-09-23T14:55:15.288Z" },
+    { url = "https://files.pythonhosted.org/packages/34/0c/80982f9ad6f52ca9cb146cfc00fac79e7b7f4388f00f4197575a3170be48/laya-0.3.20-py3-none-any.whl", hash = "sha256:6039e802fa5effb8dd492061cd7ad39a43087beadc4a4fa4a649614e77eb83d4", size = 118520, upload-time = "2026-09-24T05:41:06.273Z" },
 ]
 
 [[package]]

From d0b3d290e7b84d334c6040b569bb4a9fbee15c96 Mon Sep 17 00:00:00 2001
From: cacheline999 <326908201+cacheline999@users.noreply.github.com>
Date: Mon, 5 Oct 2026 01:02:20 +0800
Subject: [PATCH 4/5] [Fix] Reject a LAYA_MPS_AMP_MIN_ROWS that is not a number

Laya reads a value it cannot parse as its default of 5, which runs requests of
five or more questions in fp16 on MPS. from_env saw the variable set and left
that in place, so a mistyped value meant to keep fp32 did the opposite. It is
now a configuration error, raised before the checkpoint loads.
---
 docs/configuration.md              |  2 +-
 s1a/decision_models/laya.py        | 12 +++++++++++-
 tests/test_decision_models_laya.py | 11 +++++++++++
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index 9772f74..30031af 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -32,7 +32,7 @@ Variables can be exported in your shell or placed in a `.env` file at the root o
 | `LAYA_DEVICE` | `laya` model | `(library default)` | PyTorch device for Laya model evaluation; passes None so the library selects CUDA, MPS, or CPU. |
 | `LAYA_MAX_LEN` | `laya` model | `(checkpoint default)` | Maximum token sequence length for Laya state representation; overrides checkpoint window only when set. |
 | `LAYA_HEAD_MAX_LEN` | `laya` model | `(checkpoint default)` | Maximum token sequence length for Laya decision head options; overrides checkpoint window only when set. |
-| `LAYA_MPS_AMP_MIN_ROWS` | `laya` model | *(unset: fp32)* | Laya's own variable: on MPS, requests with at least this many questions run in fp16. Unset, `--model laya` keeps every request in fp32, as on CPU; `5` is Laya's default. fp16 moves the probabilities and can flip a close decision. |
+| `LAYA_MPS_AMP_MIN_ROWS` | `laya` model | *(unset: fp32)* | Laya's own variable: on MPS, requests with at least this many questions run in fp16. Unset, `--model laya` keeps every request in fp32, as on CPU; `5` is Laya's default. fp16 moves the probabilities and can flip a close decision. A value that is not a whole number is a configuration error. |
 | `LAYA_SERVED_URL` | `laya-served` model | *(unset, required)* | Base URL of a served Laya: the system1-omni worker (`http://127.0.0.1:8000`), its `omni-jev` frontend (`:8080`) or plain laya-serve. |
 | `LAYA_SERVED_MODEL` | `laya-served` model | `english` | Name of the served checkpoint to ask: `english`, `multilingual` or `typed-decisions`. A name the server does not know raises instead of being routed by language. |
 | `LAYA_SERVED_API_KEY` | `laya-served` model | *(unset)* | Bearer token, the server's `LAYA_API_KEY` when it sets one. |
diff --git a/s1a/decision_models/laya.py b/s1a/decision_models/laya.py
index b7eda60..6bf45a1 100644
--- a/s1a/decision_models/laya.py
+++ b/s1a/decision_models/laya.py
@@ -108,6 +108,16 @@ def from_env(cls) -> "LayaModel":
                 StatusCode.MODEL_SERVICE_CONFIG_ERROR,
                 error_msg="--model laya needs the laya extra: uv sync --extra laya",
             ) from exc
+        mps_rows = os.getenv("LAYA_MPS_AMP_MIN_ROWS")
+        if mps_rows:
+            try:
+                int(mps_rows)
+            except ValueError as exc:  # Laya would fall back to its default of 5 and run those requests in fp16
+                raise build_error(
+                    StatusCode.MODEL_SERVICE_CONFIG_ERROR,
+                    cause=exc,
+                    error_msg=f"LAYA_MPS_AMP_MIN_ROWS must be a whole number, not {mps_rows!r}; unset it to keep fp32",
+                ) from exc
         model = os.getenv("LAYA_MODEL") or LAYA_DEFAULT_MODEL
         subfolder = os.getenv("LAYA_SUBFOLDER") or None
         agent = laya.load(model, device=os.getenv("LAYA_DEVICE") or None, subfolder=subfolder)
@@ -125,7 +135,7 @@ def from_env(cls) -> "LayaModel":
             )
         # From 0.3.10 Laya runs a request of five or more questions in fp16 on MPS. That moves the answers, enough
         # to flip a close decision, so one browser episode would mix both precisions.
-        if not os.getenv("LAYA_MPS_AMP_MIN_ROWS") and hasattr(agent, "mps_amp_min_rows"):
+        if not mps_rows and hasattr(agent, "mps_amp_min_rows"):
             agent.mps_amp_min_rows = LAYA_MPS_FP32_ROWS
         for key, variable in (("max_len", "LAYA_MAX_LEN"), ("head_max_len", "LAYA_HEAD_MAX_LEN")):
             value = os.getenv(variable)
diff --git a/tests/test_decision_models_laya.py b/tests/test_decision_models_laya.py
index 99d6da3..bcdabdd 100644
--- a/tests/test_decision_models_laya.py
+++ b/tests/test_decision_models_laya.py
@@ -205,6 +205,17 @@ def from_env(env: dict[str, str]) -> Any:
         self.assertEqual(from_env({}).mps_amp_min_rows, laya_module.LAYA_MPS_FP32_ROWS)
         self.assertEqual(from_env({"LAYA_MPS_AMP_MIN_ROWS": "5"}).mps_amp_min_rows, 5)
 
+    def test_a_laya_variable_that_is_not_a_number_is_a_config_error(self) -> None:
+        loads: list[int] = []
+        fake_laya = SimpleNamespace(load=lambda *a, **k: loads.append(1))
+        env = {"LAYA_SUBFOLDER": "", "LAYA_MPS_AMP_MIN_ROWS": "off"}  # laya would read this as its default, fp16
+        with patch.dict(sys.modules, {"laya": fake_laya}), patch.dict(os.environ, env):
+            with self.assertRaises(BaseError) as caught:
+                LayaModel.from_env()
+        self.assertEqual(caught.exception.status, StatusCode.MODEL_SERVICE_CONFIG_ERROR)
+        self.assertIn("LAYA_MPS_AMP_MIN_ROWS", str(caught.exception))
+        self.assertEqual(loads, [])  # before the checkpoint loads
+
     def test_a_laya_before_the_mps_gate_gets_no_such_attribute(self) -> None:
         agent = FakeLayaAgent()  # laya 0.3.9 has no mps_amp_min_rows
         with patch.dict(sys.modules, {"laya": SimpleNamespace(load=lambda *a, **k: agent)}):

From b6edc845697525e0319b1a29fdedabb3a844d604 Mon Sep 17 00:00:00 2001
From: cacheline999 <326908201+cacheline999@users.noreply.github.com>
Date: Mon, 5 Oct 2026 21:43:58 +0800
Subject: [PATCH 5/5] [Docs] Add a reproducible baseline/head comparison for
 the laya upgrade

docs/results/laya-upgrade: run.sh checks out two commits, installs each one's
lock, runs --model laya's backend over the 30 ticket-router tickets (one
choice, one noul and six questions) on both checkpoints and devices, adds the
LAYA_MPS_AMP_MIN_ROWS=5 and no-skip controls, and prints the comparison.
The README gives the steps. The recorded run's outputs are attached to #37
rather than committed, per the artifact hygiene guide.
---
 docs/results/laya-upgrade/README.md  |  88 ++++++++++++++++++
 docs/results/laya-upgrade/compare.py |  83 +++++++++++++++++
 docs/results/laya-upgrade/laya_ab.py | 132 +++++++++++++++++++++++++++
 docs/results/laya-upgrade/run.sh     |  73 +++++++++++++++
 4 files changed, 376 insertions(+)
 create mode 100644 docs/results/laya-upgrade/README.md
 create mode 100644 docs/results/laya-upgrade/compare.py
 create mode 100644 docs/results/laya-upgrade/laya_ab.py
 create mode 100755 docs/results/laya-upgrade/run.sh

diff --git a/docs/results/laya-upgrade/README.md b/docs/results/laya-upgrade/README.md
new file mode 100644
index 0000000..aca29ad
--- /dev/null
+++ b/docs/results/laya-upgrade/README.md
@@ -0,0 +1,88 @@
+# laya 0.3.5 → 0.3.20: same answers from `--model laya`
+
+Reproduces the evidence for #37: `--model laya` gives the same answers on main and on that branch, on CPU and
+on MPS, and the answers move only when Laya's MPS fp16 is turned back on.
+
+| file | does |
+|---|---|
+| `run.sh` | checks out both commits, installs each one's lock, runs every case below, prints the tables |
+| `laya_ab.py` | one fresh process: load through `LayaModel.from_env()`, answer the 30 ticket-router tickets, write a JSON |
+| `compare.py` | the tables from those JSONs |
+
+## Steps
+
+1. Use a Mac with Apple silicon for the MPS cases (on other machines, step 4 with `DEVICES=cpu`), with
+   `git` and `uv` installed and this repository cloned.
+2. Fetch both checkpoints once, about 1.4 GB (`english` 804 MB, `multilingual` 614 MB):
+
+   ```sh
+   uv run --extra laya python -c "import laya; laya.load('convaiinnovations/laya', device='cpu'); laya.load('convaiinnovations/laya', device='cpu', subfolder='multilingual')"
+   ```
+
+3. Fetch the head commit. It is on #37's branch only, which a clone of ThinkFlowLab/system1-agents does not
+   fetch (with `origin` pointing there):
+
+   ```sh
+   git fetch origin pull/37/head
+   ```
+
+4. From the repository root, with the baseline (main, laya 0.3.5) and head (#37's last code commit, laya 0.3.20):
+
+   ```sh
+   HF_HUB_OFFLINE=1 docs/results/laya-upgrade/run.sh . 3008e8f d0b3d29
+   ```
+
+   About 30 minutes on an M1 Pro. It writes to `evals/results/laya-upgrade` (ignored by git), or to the
+   folder given as a fourth argument, which must be empty or new. `RUNS` (default 3), `DEVICES` (`cpu mps`)
+   and `CHECKPOINTS` (`english multilingual`) narrow it. The checkouts go to a temporary folder and are removed
+   at the end. Each run's output is in `logs/`; a failing run stops `run.sh` and prints the end of its log.
+5. To compare with the recorded run, unpack `pr37-evidence.zip` from #37 and run
+   `python docs/results/laya-upgrade/compare.py pr37-evidence/records`.
+
+## What it runs
+
+Each run is one process. It loads the model, then for each of the 30 tickets in
+`s1a/agents/_data/ticket_router_eval.jsonl` asks one choice question, one noul question, and one request of
+six questions (the choice and five nouls). Six questions is the case that changed: from 0.3.10 Laya answers a
+request of five or more questions in fp16 on MPS, and #37 keeps it in fp32.
+
+| case | runs |
+|---|---|
+| baseline and head, each checkpoint, each device | `RUNS` each, alternating baseline and head |
+| head with `LAYA_MPS_AMP_MIN_ROWS=5` (Laya's default turned back on), MPS | 1 per checkpoint |
+| baseline without its weight-init skip, CPU | 1 per checkpoint |
+
+Every JSON records the commit, the laya, torch and transformers versions, the checkpoint revision, the
+device and the Laya variables that were set.
+
+## Reading the output
+
+```
+| checkpoint | device | load s, baseline | load s, head | ... | answers |
+| convaiinnovations/laya | mps | 9.8 / 7.4 / 7.8 | 9.6 / 7.7 / 7.9 | ... | identical (3 + 3 runs) |
+```
+
+- `answers` is `identical` when every choice, probability, confidence and noul value of every run equals the
+  first baseline run's. That is the claim; a `DIFFERENT` row disproves it.
+- The `LAYA_MPS_AMP_MIN_ROWS=5` lines show what the fp32 default prevents: the largest probability change
+  and each decision that changed, by ticket id. On an M1 Pro, `multilingual` changed two.
+- The no-skip lines show the load the weight-init skip saves, now done by laya itself.
+- Load and latency columns are for comparing the two sides of one row, run alternately on the same machine.
+  The first load after the checkout can include reading the model files from disk.
+
+## Recorded run
+
+The 28 outputs of step 4 from 2026-10-05 are `records/` in `pr37-evidence.zip`, attached to #37: Apple M1 Pro (16 GB),
+macOS 26.1, Python 3.14, torch 2.14.0, transformers 5.17.0, checkpoint revision `7b928d8`, baseline `222e656`
+(laya 0.3.5), head `c197fe6` (laya 0.3.20). Those were main and #37's head before #37 was rebased onto main;
+`c197fe6` is no longer on any branch. Against the commits in step 4, the path these runs exercise is unchanged: every
+locked package has the same version, and `s1a/decision_models/laya.py` differs only by `check_window` moving to a
+module function (#35) and the `bills_input_tokens` declaration (#17).
+The machine had other work running (1-min load average about 6 to 10). All four rows are `identical`; with
+`LAYA_MPS_AMP_MIN_ROWS=5`, `multilingual` changed `t_6cb98186dde0` (pick, logistics to returns) and
+`t_f5251390fb88` (cancel, 0.4959 to 0.5074); without the skip the baseline loaded in 36.8 s (`english`) and
+42.5 s (`multilingual`) against 6 to 10 s with it. The first `english` CPU load of each side (26.2 and 24.8 s)
+read the model files from disk.
+
+Not covered: the `typed-decisions` checkpoint, a local checkpoint path, transformers 4.x, CUDA, and Apple chips
+other than the M1 Pro.
diff --git a/docs/results/laya-upgrade/compare.py b/docs/results/laya-upgrade/compare.py
new file mode 100644
index 0000000..be7a29e
--- /dev/null
+++ b/docs/results/laya-upgrade/compare.py
@@ -0,0 +1,83 @@
+# coding: utf-8
+"""Tables from the outputs of laya_ab.py:  python compare.py records/
+
+For each checkpoint and device: load time and latency per run for baseline and head, and whether every
+answer of every run equals the first baseline run's. Then every run with LAYA_MPS_AMP_MIN_ROWS set, against
+the baseline on the same device: the largest change in any probability and each decision that changed.
+"""
+
+import json
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+ANSWERS = ("one_question", "one_noul", "six_questions")
+
+
+def answers(record: dict) -> list:
+    return [{key: ticket[key] for key in ANSWERS} for ticket in record["tickets"]]
+
+
+def changes(reference: dict, record: dict) -> tuple[float, list[str]]:
+    largest, changed = 0.0, []
+    for a, b in zip(reference["tickets"], record["tickets"], strict=True):
+        for part in ANSWERS:
+            for name, value in a[part].items():
+                other = b[part][name]
+                if name == "probabilities":
+                    largest = max([largest] + [abs(p - other[option]) for option, p in value.items()])
+                elif name == "pick":
+                    if value != other:
+                        changed.append(f"{a['id']} {part} pick: {value} -> {other}")
+                elif name != "confidence":
+                    largest = max(largest, abs(value - other))
+                    if (value >= 0.5) != (other >= 0.5):
+                        changed.append(f"{a['id']} {part} {name}: {value} -> {other}")
+    return round(largest, 4), changed
+
+
+def main(folder: str) -> None:
+    records = [json.loads(path.read_text(encoding="utf-8")) for path in sorted(Path(folder).glob("*.json"))]
+    cells: dict[tuple[str, str], dict[str, list[dict]]] = defaultdict(lambda: defaultdict(list))
+    for record in records:
+        kind = (
+            "no-skip"
+            if record["no_skip"]
+            else "override"
+            if "LAYA_MPS_AMP_MIN_ROWS" in record["environment"]
+            else record["label"]
+        )
+        cells[(record["checkpoint"], record["device"])][kind].append(record)
+    for record in records[:1]:
+        print(f"{record['chip']}, {record['os']}, torch {record['torch']}, transformers {record['transformers']}")
+    for label in ("baseline", "head"):
+        seen = {(r["s1a_commit"][:7], r["laya"], r["checkpoint_revision"][:7]) for r in records if r["label"] == label}
+        print(f"{label}: " + "; ".join(f"s1a {c}, laya {v}, checkpoint revision {rev}" for c, v, rev in sorted(seen)))
+    print()
+    print(
+        "| checkpoint | device | load s, baseline | load s, head | one question p50 ms, baseline | head | six questions p50 ms, baseline | head | answers |"
+    )
+    print("|---|---|---|---|---|---|---|---|---|")
+    for (checkpoint, device), kinds in sorted(cells.items()):
+        base, head = kinds["baseline"], kinds["head"]
+        same = all(answers(r) == answers(base[0]) for r in base + head)
+        column = lambda rows, key: " / ".join(str(r[key]) for r in rows)  # noqa: E731
+        print(
+            f"| {checkpoint} | {device} | {column(base, 'load_s')} | {column(head, 'load_s')} | "
+            f"{column(base, 'one_question_p50_ms')} | {column(head, 'one_question_p50_ms')} | "
+            f"{column(base, 'six_questions_p50_ms')} | {column(head, 'six_questions_p50_ms')} | "
+            f"{'identical' if same else 'DIFFERENT'} ({len(base)} + {len(head)} runs) |"
+        )
+    for (checkpoint, device), kinds in sorted(cells.items()):
+        for record in kinds["override"]:
+            largest, changed = changes(kinds["baseline"][0], record)
+            setting = record["environment"]["LAYA_MPS_AMP_MIN_ROWS"]
+            print(f"\n{checkpoint} on {device}, head with LAYA_MPS_AMP_MIN_ROWS={setting}, against baseline:")
+            print(f"  largest probability change {largest}; decisions changed: {len(changed)}")
+            for line in changed:
+                print("  " + line)
+        for record in kinds["no-skip"]:
+            print(f"\n{checkpoint} on {device}, baseline without its weight-init skip: load {record['load_s']} s")
+
+
+main(sys.argv[1])
diff --git a/docs/results/laya-upgrade/laya_ab.py b/docs/results/laya-upgrade/laya_ab.py
new file mode 100644
index 0000000..420476d
--- /dev/null
+++ b/docs/results/laya-upgrade/laya_ab.py
@@ -0,0 +1,132 @@
+# coding: utf-8
+"""One run of `--model laya`'s backend in a fresh process; run.sh calls it from each checkout's root.
+
+It loads the model through `LayaModel.from_env()`, then for each of the 30 tickets in the repository's
+ticket-router file asks one choice question, one noul question, and one request of six questions. The JSON
+it writes holds every answer, the timings, and what ran: commit, library versions, checkpoint revision,
+device and the Laya variables that were set. `--no-skip` loads without main's weight-init skip.
+"""
+
+import argparse
+import asyncio
+import json
+import os
+import platform
+import statistics
+import subprocess
+import time
+from importlib import metadata
+from pathlib import Path
+
+T0 = time.perf_counter()
+NOULS = {
+    "refund": "Does the customer ask for a refund?",
+    "angry": "Is the customer angry?",
+    "cancel": "Does the customer want to cancel the order?",
+    "human": "Does the customer ask for a human?",
+    "late": "Is a delivery late?",
+}
+LAYA_VARIABLES = (
+    "LAYA_MODEL",
+    "LAYA_SUBFOLDER",
+    "LAYA_DEVICE",
+    "LAYA_MAX_LEN",
+    "LAYA_HEAD_MAX_LEN",
+    "LAYA_MPS_AMP_MIN_ROWS",
+)
+
+
+def run(*command: str) -> str:
+    try:
+        return subprocess.run(command, capture_output=True, text=True, timeout=10, check=True).stdout.strip()
+    except (OSError, subprocess.SubprocessError):
+        return ""
+
+
+def checkpoint_revision(repo: str) -> str:
+    """The commit the local Hugging Face cache has for the repository's main."""
+    from huggingface_hub.constants import HF_HUB_CACHE
+
+    ref = Path(HF_HUB_CACHE) / f"models--{repo.replace('/', '--')}" / "refs" / "main"
+    return ref.read_text().strip() if ref.exists() else ""
+
+
+async def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--label", required=True, help="baseline or head")
+    parser.add_argument("--out", required=True)
+    parser.add_argument("--no-skip", action="store_true", help="baseline only: load without its weight-init skip")
+    args = parser.parse_args()
+
+    from s1a.agents.ticket_router import DEFAULT_DATASET, PUBLIC_FIELDS, QUEUES, RULES, load_tickets
+    from s1a.decision_models import ChoiceQuestion, NoulQuestion, Observation
+    from s1a.decision_models import laya as laya_module
+
+    if args.no_skip:
+        import contextlib
+
+        laya_module.without_weight_init = contextlib.nullcontext
+    model = laya_module.LayaModel.from_env()
+    load_s = time.perf_counter() - T0
+
+    pick = ChoiceQuestion(dict(QUEUES), rules=RULES)
+    tickets, one_ms, six_ms = [], [], []
+    for row in load_tickets(DEFAULT_DATASET):
+        observation = Observation({key: row[key] for key in PUBLIC_FIELDS if key in row})
+        started = time.perf_counter()
+        one = await model.decide_many(observation, {"pick": pick})
+        one_ms.append((time.perf_counter() - started) * 1000)
+        noul = await model.decide_many(observation, {"refund": NoulQuestion(NOULS["refund"])})
+        started = time.perf_counter()
+        six = await model.decide_many(observation, {"pick": pick, **{k: NoulQuestion(v) for k, v in NOULS.items()}})
+        six_ms.append((time.perf_counter() - started) * 1000)
+        choice, six_choice = one.choice("pick"), six.choice("pick")
+        tickets.append(
+            {
+                "id": row["id"],
+                "one_question": {
+                    "pick": choice.key,
+                    "probabilities": choice.probabilities,
+                    "confidence": choice.confidence,
+                },
+                "one_noul": {"refund": noul.noul("refund").p},
+                "six_questions": {
+                    "pick": six_choice.key,
+                    "probabilities": six_choice.probabilities,
+                    "confidence": six_choice.confidence,
+                    **{k: six.noul(k).p for k in NOULS},
+                },
+            }
+        )
+    agent = model._agent
+    repo = os.getenv("LAYA_MODEL") or laya_module.LAYA_DEFAULT_MODEL
+    record = {
+        "label": args.label,
+        "no_skip": args.no_skip,
+        "s1a_commit": run("git", "rev-parse", "HEAD"),
+        "laya": metadata.version("laya"),
+        "torch": metadata.version("torch"),
+        "transformers": metadata.version("transformers"),
+        "python": platform.python_version(),
+        "chip": run("sysctl", "-n", "machdep.cpu.brand_string"),
+        "os": "macOS " + platform.mac_ver()[0],
+        "checkpoint": model.model,
+        "checkpoint_revision": checkpoint_revision(repo),
+        "device": str(agent.device),
+        "mps_amp_min_rows": getattr(agent, "mps_amp_min_rows", None),
+        "environment": {name: os.environ[name] for name in LAYA_VARIABLES if os.getenv(name)},
+        "load_average_1m": round(os.getloadavg()[0], 1),
+        "load_s": round(load_s, 1),
+        "one_question_p50_ms": round(statistics.median(one_ms[1:])),
+        "six_questions_p50_ms": round(statistics.median(six_ms[1:])),
+        "tickets": tickets,
+    }
+    head = {key: value for key, value in record.items() if key != "tickets"}
+    lines = [json.dumps(head)[:-1] + ', "tickets": ['] + [
+        "  " + json.dumps(ticket) + ("," if n < len(tickets) - 1 else "") for n, ticket in enumerate(tickets)
+    ]
+    Path(args.out).write_text("\n".join(lines) + "\n]}\n", encoding="utf-8")  # one ticket per line
+    print(args.out, record["laya"], record["device"], "load", record["load_s"], "s")
+
+
+asyncio.run(main())
diff --git a/docs/results/laya-upgrade/run.sh b/docs/results/laya-upgrade/run.sh
new file mode 100755
index 0000000..23ca213
--- /dev/null
+++ b/docs/results/laya-upgrade/run.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# Every run behind the README: baseline and head, both checkpoints, each device, RUNS times, then the controls.
+# Usage: run.sh <repository> <baseline commit> <head commit> [out dir, default evals/results/laya-upgrade]
+set -euo pipefail
+
+repo=$(cd "$1" && pwd)
+base=$2
+head=$3
+out=${4:-evals/results/laya-upgrade}
+if [ -n "$(ls -A "$out" 2>/dev/null)" ]; then
+  echo "run.sh: $out is not empty; compare.py reads every JSON there, so give an empty or new folder" >&2
+  exit 1
+fi
+out=$(mkdir -p "$out/logs" && cd "$out" && pwd)
+here=$(cd "$(dirname "$0")" && pwd)
+runs=${RUNS:-3}
+devices=${DEVICES:-"cpu mps"}
+checkpoints=${CHECKPOINTS:-"english multilingual"}
+
+work=$(mktemp -d)
+cleanup() {
+  git -C "$repo" worktree remove --force "$work/baseline" 2>/dev/null || true
+  git -C "$repo" worktree remove --force "$work/head" 2>/dev/null || true
+  rm -rf "$work"
+}
+trap cleanup EXIT
+
+for side in baseline head; do
+  commit=$base
+  [ "$side" = head ] && commit=$head
+  git -C "$repo" worktree add --quiet --detach "$work/$side" "$commit"
+  echo "$side: $commit"
+  (cd "$work/$side" && uv sync --quiet --compile-bytecode --extra dev --extra laya)
+  "$work/$side/.venv/bin/python" -c "import torch, transformers, laya" >/dev/null 2>&1  # first imports off the clock
+done
+
+# one <side> <checkpoint> <device> <name> [VAR=value ...] [-- laya_ab.py flags]
+one() {
+  local side=$1 checkpoint=$2 device=$3 name=$4
+  shift 4
+  local subfolder=""
+  [ "$checkpoint" = multilingual ] && subfolder=multilingual
+  local vars=() flags=()
+  while [ $# -gt 0 ]; do
+    if [ "$1" = -- ]; then shift; flags=("$@"); break; fi
+    vars+=("$1"); shift
+  done
+  echo "  $name"
+  if ! (cd "$work/$side" && env LAYA_MODEL= LAYA_MAX_LEN= LAYA_HEAD_MAX_LEN= LAYA_MPS_AMP_MIN_ROWS= \
+    LAYA_SUBFOLDER="$subfolder" LAYA_DEVICE="$device" ${vars[@]+"${vars[@]}"} \
+    .venv/bin/python "$here/laya_ab.py" --label "$side" --out "$out/$name.json" ${flags[@]+"${flags[@]}"}) >"$out/logs/$name.log" 2>&1; then
+    echo "run.sh: $name failed; the end of $out/logs/$name.log:" >&2
+    tail -n 20 "$out/logs/$name.log" >&2
+    exit 1
+  fi
+}
+
+for i in $(seq "$runs"); do
+  for checkpoint in $checkpoints; do
+    for device in $devices; do
+      one baseline "$checkpoint" "$device" "baseline-$checkpoint-$device-$i"
+      one head "$checkpoint" "$device" "head-$checkpoint-$device-$i"
+    done
+  done
+done
+for checkpoint in $checkpoints; do
+  case " $devices " in *" mps "*)
+    one head "$checkpoint" mps "head-$checkpoint-mps-override5" LAYA_MPS_AMP_MIN_ROWS=5 ;;
+  esac
+  one baseline "$checkpoint" cpu "baseline-$checkpoint-cpu-noskip" -- --no-skip
+done
+
+"$work/head/.venv/bin/python" "$here/compare.py" "$out"