From db6a5f14b2ef4f855013fea50d653e07ed6e54c9 Mon Sep 17 00:00:00 2001 From: Eric Law <39393654+acn-ericlaw@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:55:18 -0800 Subject: [PATCH] feat(llm-helper): the loop contract - llm.chat serves one turn of the engines' agent loop RFC-0011 build plan step 1 (sections 3.2 and 3.3 of the design spec). A request that carries tools is one model turn of llm.run, the agent loop of the Java and Rust engines: the transcript stays in the helper's store and the loop holds a reference to it. A later turn sends conversation, turn and results back; the helper appends the one tool_result message in call order and the model's turn exactly as the API returned it, thinking blocks included. The reply carries status (done, tool_use, refusal, cap), calls, cap for a cut-off turn (whose calls never run), the four usage fields, conversation and turn. A stale turn count or an ended run is a 409 and each write is a compare-and-set on the count, so a duplicated turn never forks a conversation; an ended or expired one is a 404. The model is pinned at the first turn, server-side fallbacks are never sent on a loop turn, and the helper places the cache breakpoints (system, plus a top-level one). The store is llm.transcript.store: memory for one process, redis for replicas (a new extra; a hash under transcript:{conversation}, a Lua compare-and-set, llm.transcript.redis.*). Every write sets ttl; a run the model ends takes retention (0 deletes it). A new shared vector file, tests/vectors/llm-loop-vectors.json (66 cases), pins the contract for the Node twin and runs on the memory store, a Redis fake and, with LLM_TRANSCRIPT_REDIS_URL, a live Redis. A request without tools behaves as before. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 20 + extensions/llm-helper/README.md | 116 +- extensions/llm-helper/llm_helper.py | 629 +- .../llm-helper/resources/application.yml | 19 +- pyproject.toml | 13 +- tests/test_llm_helper.py | 412 +- tests/vectors/llm-loop-vectors.json | 6296 +++++++++++++++++ 7 files changed, 7474 insertions(+), 31 deletions(-) create mode 100644 tests/vectors/llm-loop-vectors.json diff --git a/CHANGELOG.md b/CHANGELOG.md index e8607a2..b352ed4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,26 @@ The routes and the contract are unchanged: the shared vector file gains five Bedrock cases (68), byte-identical with the Node pack, and the Bedrock clients, their region and credential checks are pinned by unit tests. **Upgrade note:** none - the Claude API backend is the default and behaves as before; `llm.backend` now also accepts `bedrock`. +2. **The loop contract: `llm.chat` serves one model turn of the engines' agent loop (RFC-0011).** A request that carries + `tools` is a turn of `llm.run`, the agent loop of the Java and Rust engines, which dispatches each tool call to a + composable-function route. The transcript stays in the helper's own store and the loop holds a reference to it: the + first turn returns `conversation` and `turn`, a later turn sends them back with `results` (`[{id, output}]` or + `[{id, error}]`), and the helper appends the one `tool_result` message, in call order, and the model's turn exactly as + the API returned it, thinking blocks included. The reply carries `status` (`done`, `tool_use`, `refusal` or `cap`), + `calls [{id, name, input}]`, the cache usage, and `cap` (`max_tokens` or `context`) for a cut-off turn, whose calls + never run. A stale turn count, or a turn on a run that has ended, is a 409, and each write is a compare-and-set on + the count, so a duplicated turn can never fork a conversation; an ended or expired one is a 404. The model is pinned + at the first turn, server-side refusal fallbacks are never sent on a loop turn, and the helper places the cache + breakpoints (the system prompt, and a top-level one for the growing conversation). `llm.transcript.store` names the + store: `memory` for one process, `redis` for any number of replicas (the new `redis` extra; one hash per conversation + under `transcript:{conversation}`, a Lua compare-and-set, a connection from the `llm.transcript.redis.*` keys). Every + write sets the record's time to live to the turn's `ttl` (default `7d`), and a run the model ends takes `retention` + (default `0`, which deletes the record). A new shared vector file, `tests/vectors/llm-loop-vectors.json` (66 cases), + pins the contract for the Node twin; the cases run on the memory store, a Redis fake, and a live Redis when + `LLM_TRANSCRIPT_REDIS_URL` names one. **Upgrade note:** a request without `tools` behaves as before. `tools`, + `conversation`, `turn`, `results`, `ttl` and `retention` are now fields of the loop contract: on `llm.chat` they need + `tools`, and on `llm.stream` any of them is a 400. A loop turn needs `llm.transcript.store`; the sample configuration + sets `memory`. ### Changed diff --git a/extensions/llm-helper/README.md b/extensions/llm-helper/README.md index 1450b41..52797e6 100644 --- a/extensions/llm-helper/README.md +++ b/extensions/llm-helper/README.md @@ -12,8 +12,10 @@ model advises within it. flow, or graph ``` +`llm.chat` is also the model turn of the engines' agent loop (`llm.run`): see The loop contract. + The Node.js twin ([mercury-nodejs `examples/llm-helper`](https://github.com/Accenture/mercury-nodejs)) -speaks the same contract. Both run one shared vector file, so they cannot drift apart. +speaks the same contract. Both run the same shared vector files, so they cannot drift apart. ## Run it @@ -45,6 +47,15 @@ mercury-serve extensions/llm-helper/llm_helper.py -Dllm.backend=bedrock -Dllm.aw -Dllm.model=us.anthropic.claude-haiku-4-5-20251001-v1:0 ``` +To serve an agent loop from more than one replica, keep the transcripts in Redis (see The transcript store); +the `redis` extra adds the client: + +```bash +pip install 'mercury-composable[llm,redis]' +export REDIS_PASSWORD=... # from the environment, like every credential +mercury-serve extensions/llm-helper/llm_helper.py -Dllm.transcript.store=redis +``` + The app listens on port 8086 (`rest.server.port` in `resources/application.yml`). Map its routes from the engine's `event-over-http.yaml`, as the Java and Rust MiniGraph Playgrounds do: @@ -126,6 +137,80 @@ the model refused. A stream that ends before its first token fails with a 422 in is too small for the question can end with no text at all (seen live with a 120-token budget): that is the 422 above, not an empty stream. Raise `params.max_tokens` or lower `params.effort`. +## The loop contract + +`llm.chat` also serves one model turn of an agent loop. The loop is `llm.run`, a function of the Java and Rust +engines (RFC-0011 in mercury-composable): it calls `llm.chat` once per turn, runs each tool the model asks for as a +composable-function route, and sends the results back. A request that carries `tools` is such a turn. The +transcript never leaves this app: it lives in the transcript store, and the loop holds only a reference to it. + +```text +loop → llm.chat: system, tools, prompt (the first turn) +llm.chat → loop: status=tool_use, calls=[{id, name, input}], conversation, turn=1 +loop → route: each call's input, as the event body +loop → llm.chat: system, tools, conversation, turn=1, results=[{id, output}] +llm.chat → loop: status=done, text, conversation, turn=2 +``` + +| Field | Meaning | +|---|---| +| `tools` | `[{name, description, input_schema, strict}]`, as the model sees them. `name` matches `^[a-zA-Z0-9_-]{1,128}$` and is unique; `strict: true` asks for grammar-constrained arguments. Any other key is a 400: a route or an approval mark stays in the loop's catalog, and the model never sees it. | +| `prompt` or `messages` | The first turn only. A later turn carries neither: the transcript holds the conversation. | +| `conversation` | The reference a previous turn returned. Absent on the first turn. | +| `turn` | The turn count the loop expects the record to hold: `0` or absent on the first turn, required with `conversation`. | +| `results` | The answers to the previous turn's calls, one per call: `[{id, output}]` or `[{id, error}]`. | +| `ttl` | How long the record lives while the run goes on: seconds, or `30m`, `12h`, `2d`. Default `7d` (`llm.transcript.ttl`). | +| `retention` | How long the record is kept once the model ends the run. Default `0` (`llm.transcript.retention`), which deletes it. | +| `system`, `schema`, `params` | As on any call, and sent on every turn. | + +**Reply:** `status` (`done`, `tool_use`, `refusal` or `cap`); `text`, or `data` for a `schema` request, when done; +`calls [{id, name, input}]` for `tool_use`; `refusal {category}`; `cap` (`max_tokens` or `context`); then +`conversation`, `turn` (the record's count after this turn), `model`, `stop_reason`, `usage {input_tokens, +output_tokens, cache_creation_input_tokens, cache_read_input_tokens}` and `request_id` when the API sent one. + +- **A model outcome is a status, not an error.** A refusal, and a turn cut off at `max_tokens` or at the context + window, come back with their status, so the loop can name them in its envelope. A cut-off turn returns no calls, + so a truncated call never runs. A run that ends with nothing usable (no text, or a verdict that is not JSON) is + still the 422 above, and `pause_turn` (server tools, which a loop never sends) is a 502. +- **The append check.** A request whose `turn` differs from the record's count is a 409, and so is a turn on a run + that has ended. Each write is a compare-and-set on that count, so two copies of one turn (a duplicated resume) + can never fork the conversation: the second is a 409 even when both passed the check. A conversation that ended + or expired is a 404. +- **Append-only, exactly as returned.** Each assistant turn is stored as the API returned it, thinking blocks and + their signatures included, and sent back unchanged. The helper builds the one user message of `tool_result` + blocks, in call order, with `is_error` on an `error` result; a result that is not text is written as compact JSON. + A missing `results`, an unknown or duplicate id, or a call left unanswered is a 400 that names the ids. +- **The model is pinned at the first turn.** Every later turn uses it, and a `params.model` that names another is a + 400. Server-side refusal fallbacks are never sent on a loop turn: a fallback moves the conversation to another + model, whose replay rules edit the history. The graph routes `status: refusal` instead. +- **Caching is the helper's job.** Tools and system come first and stay the same across turns, so the system prompt + carries one cache breakpoint and the request a top-level one that follows the conversation as it grows. No stored + block is ever marked. `usage.cache_read_input_tokens` shows the hits. +- **No tool runs here.** The helper carries the model's calls out and the loop's results in; the loop dispatches. + +### The transcript store + +`llm.transcript.store` names it, and a loop turn without one is a `503 transcript store not configured`. A request +without `tools` never touches it. + +| `llm.transcript.store` | For | +|---|---| +| `memory` | One process: development and tests. A turn that reaches another replica answers 404. | +| `redis` | Any number of replicas. The `redis` extra adds the client. | + +Redis keeps one hash per conversation under `transcript:{conversation}`, holding the turn count and the record. A +Lua script makes the compare-and-set atomic on that one key. A deployment may share one Redis with the graph's +suspend store (`minigraph-state-redis`, keys `graph:{graph_id}:{cid}`): give this app a Redis ACL user confined to +`~transcript:*`, so the two share a server but neither a namespace nor a credential. The client connects on the +first turn, to a standalone server (a cluster's configuration endpoint needs the cluster client, not built yet); a +store that cannot be reached is a `503 transcript store unreachable - …`. + +Every write sets the record's time to live to `ttl`, so a run that waits for a person keeps its transcript: make +`ttl` at least as long as the longest approval wait. When the model ends the run (done, a refusal, the context +window), the record takes `retention` instead: `0` deletes it, and anything longer keeps it closed for audit, under +the store's access control, because a transcript may hold personal data. A run the loop stops itself (a turn cut off +at `max_tokens`, or the loop's own limits) keeps its `ttl` and expires with it. + ## Configuration `resources/application.yml`, or `-Dkey=value` at run time. A call's `params` win over these keys. @@ -143,6 +228,10 @@ the 422 above, not an empty stream. Raise `params.max_tokens` or lower `params.e | `llm.effort` | unset | The default effort. | | `llm.log.batches` | `false` | Log each streamed batch's number, size and arrival time (never its text). | | `llm.provider` | `anthropic` | Only `anthropic`; any other value is a 400. | +| `llm.transcript.store` | unset | `memory` or `redis`; a loop turn needs one (see The transcript store). | +| `llm.transcript.ttl` | `7d` | A loop turn's `ttl` when the request carries none. | +| `llm.transcript.retention` | `0` | A loop turn's `retention` when the request carries none. | +| `llm.transcript.redis.host`, `.port`, `.username`, `.password`, `.ssl`, `.database`, `.timeout.ms` | `127.0.0.1`, `6379`, none, none, `false`, `0`, `5000` | The Redis connection: the engines' `redis.*` set under this store's own prefix. The password comes from the environment (`${REDIS_PASSWORD}`). | The credential is never configured here. `ANTHROPIC_API_KEY` (or whatever the SDK resolves: an auth token, or a profile) comes from the environment for the Claude API; the AWS credential chain, @@ -203,23 +292,34 @@ is a `403 ... is not available for this account` on a model the console's playgr name the model as a profile id. The same message on both surfaces, with the model agreement in place, is an account enablement for that model generation - a conversation with the AWS account team, not a setting. `llm.health` reports `api` beside `backend`, `model` and `region`. Bedrock sends no `request_id` the helper reads, so a reply carries none. +A loop turn's top-level cache breakpoint works on both surfaces with the current models; Bedrock's legacy integration +for Opus 4.6 and earlier rejects it with a 400. ## What it does not do -No tools or function calling, no images or documents, no `system` turns inside `messages`, no -sampling parameters, no prompt caching controls, and no provider other than Claude. Those are -deliberate: the helper is the bounded AI node of a graph, not an agent runtime. +No tool ever runs here: the engines' agent loop dispatches each call to a route. No images or +documents, no `system` turns inside `messages`, no sampling parameters, no prompt caching controls +for a caller (a loop turn is cached by the helper), and no provider other than Claude. Those are +deliberate: the helper is the bounded AI node of a graph and the model turn of an agent loop, not +an agent runtime. ## Tests `tests/test_llm_helper.py` runs the shared `tests/vectors/llm-helper-vectors.json` against a fake of the SDK (68 cases: request validation, the exact SDK call, replies, the error contract, streaming, -the shape of a Bedrock call on either surface), plus the tests a vector cannot express (token batches are -never held back, no prompt text in a log, the deadline cancels the call, trace annotations, health, the -backend seam, the Bedrock clients with their region and credential checks). No token is spent and no credential -is needed. +the shape of a Bedrock call on either surface) and the shared `tests/vectors/llm-loop-vectors.json` (66 cases of +one or more turns: the loop fields, the exact SDK call of each turn with the transcript sent back, the reply, and the +stored record after each step), the loop cases once on the memory store and again on a Redis fake that runs the +store's Lua. Then come the tests a vector cannot express (token batches are never held back, no prompt or tool text +in a log, the deadline cancels the call, trace annotations, health, the backend seam, the Bedrock clients with their +region and credential checks, the stored blocks as the SDK sends them back, the Redis store's keys, +compare-and-set and connection settings). No token is spent and no credential is needed. ```bash pip install -e '.[dev]' pytest tests/test_llm_helper.py +LLM_TRANSCRIPT_REDIS_URL=redis://127.0.0.1:6379/15 pytest tests/test_llm_helper.py -k live_redis # on a real Redis ``` + +The live run touches only the keys of the vectors' own conversations; mercury-composable's +`helpers/redis-standalone` is a local Redis without Docker. diff --git a/extensions/llm-helper/llm_helper.py b/extensions/llm-helper/llm_helper.py index 90df2de..a6737d1 100644 --- a/extensions/llm-helper/llm_helper.py +++ b/extensions/llm-helper/llm_helper.py @@ -12,9 +12,11 @@ the Claude API, the AWS credential chain (or AWS_BEARER_TOKEN_BEDROCK) for AWS Bedrock. Settings live in resources/application.yml (or -Dkey=value): llm.backend (anthropic or bedrock), llm.aws.api (mantle or runtime), llm.aws.region, llm.model, llm.max.tokens, -llm.timeout.ms, llm.max.retries, llm.fallbacks, llm.effort. The contract, the keys and the backends are documented in +llm.timeout.ms, llm.max.retries, llm.fallbacks, llm.effort, and for the agent loop's turns +llm.transcript.store (memory or redis), llm.transcript.ttl, llm.transcript.retention and the +llm.transcript.redis.* connection keys. The contract, the keys and the backends are documented in README.md next to this file. The Node.js twin (mercury-nodejs examples/llm-helper) speaks -the same contract, and both are pinned by one shared vector file. +the same contract, and both are pinned by shared vector files. Then map the routes from a Mercury engine application (event-over-http.yaml): @@ -31,10 +33,12 @@ import asyncio import importlib.util import json +import re import time +import uuid from collections.abc import Callable -from dataclasses import dataclass -from typing import Any +from dataclasses import dataclass, field, replace +from typing import Any, Protocol try: import anthropic @@ -100,6 +104,37 @@ # session" without "AWS"); matched without case on the words both carry AWS_CREDENTIAL_RUNTIME_ERROR = ("could not resolve", "credentials from session") +# --- the loop contract's defaults and limits (RFC-0011) ----------------------------------- +# a request that carries tools is one turn of an agent loop; the other loop fields need tools +LOOP_FIELDS = ("conversation", "turn", "results", "ttl", "retention") +LOOP_KEYS = ("tools", *LOOP_FIELDS) +TOOL_KEYS = ("name", "description", "input_schema", "strict") +# the API's rule for an advertised name: a route name such as orders.search is mapped by the +# loop's catalog to an API-safe name, never advertised +TOOL_NAME = re.compile(r"[a-zA-Z0-9_-]{1,128}") +CONVERSATION_ID = re.compile(r"[0-9a-f]{32}") +DURATION = re.compile(r"([0-9]+)([smhd]?)") +DURATION_SECONDS = {"": 1, "s": 1, "m": 60, "h": 3600, "d": 86400} +DEFAULT_TRANSCRIPT_TTL = "7d" +DEFAULT_TRANSCRIPT_RETENTION = "0" +STORE_MEMORY = "memory" +STORE_REDIS = "redis" +STORES = (STORE_MEMORY, STORE_REDIS) +# the store's key prefix, so a Redis ACL can confine this app's user to ~transcript:* +TRANSCRIPT_PREFIX = "transcript:" +DONE_STOPS = ("end_turn", "stop_sequence") +# a turn cut off by a limit: the loop names which one in its envelope +CAPS = {"max_tokens": "max_tokens", "model_context_window_exceeded": "context"} +CACHE_CONTROL = {"type": "ephemeral"} +MISSING_STORE = ( + "transcript store not configured - set llm.transcript.store " + "(redis, or memory for a single process)" +) +MISSING_REDIS_SDK = ( + "The redis transcript store needs redis - pip install 'mercury-composable[redis]'" +) +STREAM_LOOP = "tools and the loop fields are not part of the streaming contract - use llm.chat" + def _prop(key: str, default: str | None = None) -> str | None: return app_config().get_property(key, default) @@ -122,6 +157,7 @@ class LlmRequest: stop_sequences: list[str] | None schema: dict[str, Any] | None fallbacks: bool + tools: list[dict[str, Any]] | None = None # a loop turn's tools, as the API takes them def _whole_number(name: str, value: Any, minimum: int) -> int: @@ -224,15 +260,19 @@ def _fallbacks_enabled() -> bool: return mode == "default" -def prepare(body: Body, *, streaming: bool) -> LlmRequest: +def prepare(body: Body, *, streaming: bool, continuation: bool = False) -> LlmRequest: """Validate a request body into an LlmRequest, or raise AppException(400). - The checks run in a fixed order, the same in the Node.js twin: body, params and provider, - the numbers, the turns, system, effort, stop sequences, schema, the fallbacks switch. + The checks run in a fixed order, the same in the Node.js twin: the loop fields on the + streaming route, body, params and provider, the numbers, the turns, system, effort, stop + sequences, schema, the fallbacks switch. A loop turn after the first (``continuation``) + carries no turns of its own: the transcript holds them. """ + if streaming and isinstance(body, dict) and any(key in body for key in LOOP_KEYS): + raise AppException(400, STREAM_LOOP) turns = body.get("messages") if isinstance(body, dict) else None has_turns = isinstance(turns, list) and bool(turns) - if not isinstance(body, dict) or not (body.get("prompt") or has_turns): + if not isinstance(body, dict) or not (continuation or body.get("prompt") or has_turns): raise AppException(400, "missing 'prompt' or 'messages'") params = _params(body) _check_provider(params) @@ -255,7 +295,7 @@ def prepare(body: Body, *, streaming: bool) -> LlmRequest: max_tokens=max_tokens, timeout_ms=timeout_ms, max_retries=max_retries, - messages=_turns(body), + messages=[] if continuation else _turns(body), system=_system(body), effort=_effort(params), stop_sequences=_stop_sequences(params), @@ -264,6 +304,162 @@ def prepare(body: Body, *, streaming: bool) -> LlmRequest: ) +# --- the loop request: one turn of an agent loop (RFC-0011) ------------------------------ +# +# The agent loop (llm.run, in the engines) calls llm.chat once per model turn. The transcript stays +# in this app's store and the loop holds only a reference to it, so no provider block ever crosses +# an engine codec, and the record is appended to, never edited - the current models' thinking +# blocks are bound to the conversation that produced them. + + +@dataclass(frozen=True) +class LoopTurn: + """The validated loop fields of a turn. The rest of the request is an LlmRequest.""" + + tools: list[dict[str, Any]] + conversation: str | None # None on the first turn + turn: int # the turn count the loop expects the record to hold + results: list[dict[str, Any]] | None + ttl_s: int + retention_s: int + + +def _is_loop(body: dict[str, Any]) -> bool: + return any(key in body for key in LOOP_KEYS) + + +def _tools(raw: Any) -> list[dict[str, Any]]: + """The tools as the API takes them; the loop's catalog keeps the routes and approval marks.""" + if not isinstance(raw, list) or not raw: + raise AppException( + 400, "tools must be a non-empty list of {name, description, input_schema}" + ) + tools: list[dict[str, Any]] = [] + names: set[str] = set() + for index, tool in enumerate(raw): + if not isinstance(tool, dict): + raise AppException(400, f"tools[{index}] must be a map") + unknown = sorted(str(key) for key in tool if key not in TOOL_KEYS) + if unknown: + raise AppException( + 400, + f"tools[{index}]: unsupported keys: {', '.join(unknown)} - " + f"supported: {', '.join(TOOL_KEYS)}", + ) + name = tool.get("name") + if not isinstance(name, str) or not TOOL_NAME.fullmatch(name): + raise AppException(400, f"tools[{index}].name must match ^[a-zA-Z0-9_-]{{1,128}}$") + if name in names: + raise AppException(400, f"tools[{index}].name '{name}' is not unique") + names.add(name) + description = tool.get("description") + if not isinstance(description, str) or not description: + raise AppException(400, f"tools[{index}].description must be a non-empty string") + input_schema = tool.get("input_schema") + if not isinstance(input_schema, dict): + raise AppException(400, f"tools[{index}].input_schema must be a JSON schema map") + strict = tool.get("strict") + if strict is not None and not isinstance(strict, bool): + raise AppException(400, f"tools[{index}].strict must be true or false") + entry: dict[str, Any] = { + "name": name, + "description": description, + "input_schema": input_schema, + } + if strict: + entry["strict"] = True + tools.append(entry) + return tools + + +def _conversation(raw: Any) -> str | None: + if raw is None: + return None + if not isinstance(raw, str) or not CONVERSATION_ID.fullmatch(raw): + raise AppException(400, "conversation must be the reference a previous turn returned") + return raw + + +def _results(raw: Any) -> list[dict[str, Any]] | None: + """The answers to the previous turn's calls: {id, output} or {id, error}.""" + if raw is None: + return None + if not isinstance(raw, list): + raise AppException(400, "results must be a list of {id, output} or {id, error}") + for index, result in enumerate(raw): + if not isinstance(result, dict): + raise AppException(400, f"results[{index}] must be a map") + unknown = sorted(str(key) for key in result if key not in ("id", "output", "error")) + if unknown: + raise AppException( + 400, + f"results[{index}]: unsupported keys: {', '.join(unknown)} - supported: " + "id, output, error", + ) + call_id = result.get("id") + if not isinstance(call_id, str) or not call_id: + raise AppException(400, f"results[{index}].id must be a call id") + if ("output" in result) == ("error" in result): + raise AppException(400, f"results[{index}] must carry either output or error") + return raw + + +def _duration(name: str, value: Any, minimum: int) -> int: + """Seconds from a whole number or a duration such as 30m, 12h or 2d, or a 400 naming it.""" + text = value.strip() if isinstance(value, str) else None + if isinstance(value, int) and not isinstance(value, bool): + text = str(value) + match = DURATION.fullmatch(text) if text is not None else None + seconds = int(match.group(1)) * DURATION_SECONDS[match.group(2)] if match else -1 + if seconds < minimum: + raise AppException( + 400, + f"{name} must be a duration such as 30m, 12h or 2d (seconds, s, m, h, d) >= {minimum}", + ) + return seconds + + +def prepare_loop(body: dict[str, Any]) -> LoopTurn: + """Validate the loop fields of a turn, or raise AppException(400). + + The checks run in a fixed order, the same in the Node.js twin: tools, conversation, turn, + results, ttl, retention, then what a first turn and a later turn may carry. + """ + if "tools" not in body: + loop_field = next(key for key in LOOP_FIELDS if key in body) + raise AppException(400, f"{loop_field} is a field of the loop contract - it needs tools") + tools = _tools(body.get("tools")) + conversation = _conversation(body.get("conversation")) + raw_turn = body.get("turn") + turn = None if raw_turn is None else _whole_number("turn", raw_turn, 0) + results = _results(body.get("results")) + ttl = body.get("ttl") + if ttl is None: + ttl = _prop("llm.transcript.ttl", DEFAULT_TRANSCRIPT_TTL) + retention = body.get("retention") + if retention is None: + retention = _prop("llm.transcript.retention", DEFAULT_TRANSCRIPT_RETENTION) + ttl_s = _duration("ttl", ttl, 1) + retention_s = _duration("retention", retention, 0) + if conversation is None: + if turn: + raise AppException(400, "turn must be 0 or absent on a first turn") + if results is not None: + raise AppException(400, "results answer a previous turn - a first turn carries none") + else: + if turn is None: + raise AppException( + 400, "turn is required with conversation - the turn count the record should hold" + ) + if body.get("prompt") or body.get("messages"): + raise AppException( + 400, + "a later turn carries results, never prompt or messages - the transcript " + "holds the conversation", + ) + return LoopTurn(tools, conversation, turn or 0, results, ttl_s, retention_s) + + # --- the backend seam ------------------------------------------------------------------ @@ -398,6 +594,16 @@ def _plan(request: LlmRequest, backend: Backend) -> tuple[Any, dict[str, Any]]: } if request.system: kwargs["system"] = request.system + if request.tools is not None: + # a loop turn: tools and system render first and stay the same across turns, so one + # marker caches them and the top-level one caches the conversation as it grows. No + # stored block is ever marked - the transcript is sent back exactly as it was returned + kwargs["tools"] = request.tools + kwargs["cache_control"] = dict(CACHE_CONTROL) + if request.system: + kwargs["system"] = [ + {"type": "text", "text": request.system, "cache_control": dict(CACHE_CONTROL)} + ] output: dict[str, Any] = {} if request.schema is not None: output["format"] = {"type": "json_schema", "schema": request.schema} @@ -407,8 +613,11 @@ def _plan(request: LlmRequest, backend: Backend) -> tuple[Any, dict[str, Any]]: kwargs["output_config"] = output if request.stop_sequences: kwargs["stop_sequences"] = request.stop_sequences + # never on a loop turn: the record pins the model at the first turn, and a fallback would move + # the conversation to another model whose replay rules edit the history if ( request.fallbacks + and request.tools is None and backend.supports_fallbacks and request.model.startswith(FALLBACK_MODEL_PREFIXES) ): @@ -430,6 +639,22 @@ class Completion: output_tokens: int request_id: str | None refusal: dict[str, Any] | None + # every content block as the API returned it (thinking, text, tool_use), for a loop's record + content: list[dict[str, Any]] = field(default_factory=list) + cache_creation_input_tokens: int = 0 + cache_read_input_tokens: int = 0 + + +def _block(block: Any) -> dict[str, Any]: + """A content block as JSON, the way the SDK sends a returned block back: the fields the API + set and nothing else. Stored, and later sent, exactly so.""" + to_dict: Any = getattr(block, "to_dict", None) + if to_dict is not None: + as_json: dict[str, Any] = to_dict(mode="json") + return as_json + if isinstance(block, dict): + return dict(block) + return dict(vars(block)) def _refusal(details: Any) -> dict[str, Any] | None: @@ -465,14 +690,18 @@ async def _complete( if text: on_text(text) message = await stream.get_final_message() + usage = message.usage return Completion( text="".join(b.text for b in message.content if getattr(b, "type", "") == "text"), model=message.model, stop_reason=message.stop_reason or "", - input_tokens=message.usage.input_tokens, - output_tokens=message.usage.output_tokens, + input_tokens=usage.input_tokens, + output_tokens=usage.output_tokens, request_id=request_id or getattr(message, "_request_id", None), refusal=_refusal(getattr(message, "stop_details", None)), + content=[_block(b) for b in message.content], + cache_creation_input_tokens=getattr(usage, "cache_creation_input_tokens", None) or 0, + cache_read_input_tokens=getattr(usage, "cache_read_input_tokens", None) or 0, ) @@ -589,6 +818,374 @@ def _summary(route: str, done: Completion, started: float) -> None: ) +# --- the transcript store (RFC-0011) ------------------------------------------------------- +# +# Only this app reads or writes a transcript. A record holds the model pinned at the first turn, +# the messages in the provider's format, the turn count and the ids of the calls the last turn +# made. A write is a compare-and-set on the turn count, so a duplicated turn can never fork a +# conversation, and every write sets the record's time to live. + + +@dataclass(frozen=True) +class Transcript: + model: str + messages: list[dict[str, Any]] + turn: int + pending: list[str] # the ids of the last turn's tool calls, in call order + closed: bool = False # the run ended; a retained record is kept for audit, never continued + + def to_json(self) -> str: + record = { + "v": 1, + "model": self.model, + "turn": self.turn, + "pending": self.pending, + "closed": self.closed, + "messages": self.messages, + } + return json.dumps(record, ensure_ascii=False, separators=(",", ":")) + + @staticmethod + def from_json(text: str) -> Transcript: + record = json.loads(text) + return Transcript( + model=record["model"], + messages=record["messages"], + turn=record["turn"], + pending=record["pending"], + closed=record["closed"], + ) + + +class TranscriptStore(Protocol): + name: str + + async def load(self, conversation: str) -> Transcript | None: + """The record, or None when it never existed, ended or expired.""" + ... + + async def save( + self, conversation: str, expected_turn: int, record: Transcript, ttl_s: int + ) -> bool: + """Write the record if the stored one holds expected_turn (0: none may exist yet), with + ttl_s to live; ttl_s 0 deletes it instead. False when another write came first.""" + ... + + +class MemoryStore: + """One process: for development and tests. A second replica cannot see these records.""" + + name = STORE_MEMORY + + def __init__(self, clock: Callable[[], float] = time.monotonic) -> None: + self._clock = clock + self._records: dict[str, tuple[float, int, str]] = {} # expiry, turn, record + + def _live(self, conversation: str) -> tuple[float, int, str] | None: + entry = self._records.get(conversation) + if entry is not None and entry[0] <= self._clock(): + del self._records[conversation] + return None + return entry + + async def load(self, conversation: str) -> Transcript | None: + entry = self._live(conversation) + return None if entry is None else Transcript.from_json(entry[2]) + + async def save( + self, conversation: str, expected_turn: int, record: Transcript, ttl_s: int + ) -> bool: + # no await between the check and the write, so the event loop makes it atomic + entry = self._live(conversation) + held = 0 if entry is None else entry[1] + if held != expected_turn or (entry is not None and expected_turn == 0): + return False + if ttl_s == 0: + self._records.pop(conversation, None) + else: + self._records[conversation] = (self._clock() + ttl_s, record.turn, record.to_json()) + return True + + +# one script, one key: atomic on a standalone server and safe on a cluster +REDIS_SAVE = """ +local held = redis.call('HGET', KEYS[1], 'turn') +if ARGV[1] == '0' then + if held then return 0 end +elseif held ~= ARGV[1] then + return 0 +end +if ARGV[4] == '0' then + redis.call('DEL', KEYS[1]) +else + redis.call('HSET', KEYS[1], 'turn', ARGV[2], 'record', ARGV[3]) + redis.call('EXPIRE', KEYS[1], ARGV[4]) +end +return 1 +""" + + +class RedisStore: + """A hash per conversation under transcript:{conversation}: the turn count and the record.""" + + name = STORE_REDIS + + def __init__(self, client: Any) -> None: + self._client = client # redis.asyncio.Redis with decode_responses=True + + async def load(self, conversation: str) -> Transcript | None: + text = await self._client.hget(TRANSCRIPT_PREFIX + conversation, "record") + return None if text is None else Transcript.from_json(text) + + async def save( + self, conversation: str, expected_turn: int, record: Transcript, ttl_s: int + ) -> bool: + key = TRANSCRIPT_PREFIX + conversation + args = (str(expected_turn), str(record.turn), record.to_json(), str(ttl_s)) + return bool(await self._client.eval(REDIS_SAVE, 1, key, *args)) + + +def _redis_store() -> RedisStore: + """The Redis client from the llm.transcript.redis.* keys - the engines' redis.* connection + set under this store's own prefix. It connects on the first command.""" + if importlib.util.find_spec("redis") is None: + raise AppException(503, MISSING_REDIS_SDK) + import redis.asyncio as redis_async # the [redis] extra; checked above + + timeout_s = ( + _whole_number( + "llm.transcript.redis.timeout.ms", _prop("llm.transcript.redis.timeout.ms", "5000"), 1 + ) + / 1000 + ) + client = redis_async.Redis( + host=str(_prop("llm.transcript.redis.host", "127.0.0.1")), + port=_whole_number( + "llm.transcript.redis.port", _prop("llm.transcript.redis.port", "6379"), 1 + ), + username=_prop("llm.transcript.redis.username") or None, + password=_prop("llm.transcript.redis.password") or None, + ssl=str(_prop("llm.transcript.redis.ssl", "false")).lower() == "true", + db=_whole_number( + "llm.transcript.redis.database", _prop("llm.transcript.redis.database", "0"), 0 + ), + socket_timeout=timeout_s, + socket_connect_timeout=timeout_s, + decode_responses=True, + ) + return RedisStore(client) + + +_store: TranscriptStore | None = None # built on first use - module-level so tests can inject + + +def get_store() -> TranscriptStore: + global _store + store = _store + if store is None: + name = str(_prop("llm.transcript.store", "") or "").lower() + if not name: + raise AppException(503, MISSING_STORE) + if name not in STORES: + raise AppException(500, f"llm.transcript.store must be one of: {', '.join(STORES)}") + store = MemoryStore() if name == STORE_MEMORY else _redis_store() + _store = store + return store + + +def _store_failure(exc: Exception) -> AppException: + """A store that cannot be reached is the deployment's problem, named as such.""" + return AppException(503, f"transcript store unreachable - {type(exc).__name__}: {exc}") + + +def _new_conversation_id() -> str: + return uuid.uuid4().hex + + +# --- one loop turn --------------------------------------------------------------------------- + + +def _check_turn( + conversation: str, record: Transcript | None, loop: LoopTurn, body: dict[str, Any] +) -> Transcript: + """The append check: the record exists, is open, holds the turn the loop expects, and keeps + its model.""" + if record is None: + raise AppException(404, f"conversation {conversation} not found - it ended or expired") + if record.closed: + raise AppException(409, f"conversation {conversation} has ended - start a new one") + if record.turn != loop.turn: + raise AppException( + 409, + f"conversation {conversation} holds turn {record.turn}, not {loop.turn} - " + "a stale or duplicated turn", + ) + params = body.get("params") + model = params.get("model") if isinstance(params, dict) else None + if model is not None and str(model) != record.model: + raise AppException(400, f"conversation {conversation} is pinned to model {record.model}") + return record + + +def _result_text(value: Any) -> str: + """A tool's output or error as tool_result text; JSON is written compactly, keys in order.""" + if isinstance(value, str): + return value + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + + +def _tool_results(pending: list[str], results: list[dict[str, Any]] | None) -> dict[str, Any]: + """The one user message that answers the last turn's calls, in call order.""" + if results is None: + raise AppException( + 400, f"results required - the conversation waits for {len(pending)} tool result(s)" + ) + by_id: dict[str, dict[str, Any]] = {} + for result in results: + call_id = result["id"] + if call_id in by_id: + raise AppException(400, f"results: duplicate id '{call_id}'") + if call_id not in pending: + raise AppException( + 400, + f"results: unknown id '{call_id}' - the conversation waits for: " + f"{', '.join(pending)}", + ) + by_id[call_id] = result + missing = [call_id for call_id in pending if call_id not in by_id] + if missing: + raise AppException(400, f"results: missing id(s): {', '.join(missing)}") + blocks: list[dict[str, Any]] = [] + for call_id in pending: + result = by_id[call_id] + block: dict[str, Any] = {"type": "tool_result", "tool_use_id": call_id} + if "error" in result: + block["content"] = _result_text(result["error"]) + block["is_error"] = True + else: + block["content"] = _result_text(result["output"]) + blocks.append(block) + return {"role": "user", "content": blocks} + + +def _loop_outcome(request: LlmRequest, done: Completion) -> dict[str, Any]: + """The reply's outcome for the turn's stop reason: done, tool_use, refusal or cap. A cut-off + turn returns no calls, so none of its truncated calls can run.""" + stop = done.stop_reason + if stop == "tool_use": + calls = [ + {"id": block["id"], "name": block["name"], "input": block["input"]} + for block in done.content + if block.get("type") == "tool_use" + ] + if not calls: + raise AppException(502, "LLM stop_reason=tool_use carried no tool call") + return {"status": "tool_use", "calls": calls} + if stop in DONE_STOPS: + data = _judge(request, done, has_content=bool(done.text)) + if request.schema is not None: + return {"status": "done", "data": data} + return {"status": "done", "text": done.text} + if stop == "refusal": + return {"status": "refusal", "refusal": {"category": (done.refusal or {}).get("category")}} + if stop in CAPS: + return {"status": "cap", "cap": CAPS[stop]} + raise AppException(502, f"LLM stop_reason={stop} is not part of the loop contract") + + +async def _loop_turn(body: dict[str, Any]) -> dict[str, Any]: + """One model turn of an agent loop: load and check the record, append the tool results, call + the model, append its turn, and save under the compare-and-set.""" + loop = prepare_loop(body) + request = prepare(body, streaming=False, continuation=loop.conversation is not None) + store = get_store() + try: + if loop.conversation is None: + conversation = _new_conversation_id() + record = Transcript(request.model, list(request.messages), 0, []) + messages = list(record.messages) + else: + conversation = loop.conversation + record = _check_turn(conversation, await store.load(conversation), loop, body) + messages = [*record.messages, _tool_results(record.pending, loop.results)] + except AppException: + raise + except Exception as exc: # the store's own failure - a connection, a timeout + raise _store_failure(exc) from exc + request = replace(request, model=record.model, messages=messages, tools=loop.tools) + started = time.perf_counter() + try: + done = await asyncio.wait_for( + _complete(request, get_backend()), timeout=request.timeout_ms / 1000 + ) + except Exception as exc: # anything unmapped is re-raised below + failure = _failure(exc, request.timeout_ms) + if failure is None: + raise + log.warning("llm.chat failed - status=%d turn=%d", failure.status, record.turn + 1) + raise failure from exc + outcome = _loop_outcome(request, done) + status = outcome["status"] + calls = outcome.get("calls", []) + turn = record.turn + 1 + saved = Transcript( + model=record.model, + messages=[*messages, {"role": "assistant", "content": done.content}], + turn=turn, + pending=[call["id"] for call in calls], + closed=status != "tool_use", + ) + # a run the model ended takes the retention; a run the loop stops (a cut-off turn, or the + # loop's own limits) keeps its time to live and expires with it + model_ended = status in ("done", "refusal") or outcome.get("cap") == "context" + ttl_s = loop.retention_s if model_ended else loop.ttl_s + try: + written = await store.save(conversation, record.turn, saved, ttl_s) + except Exception as exc: # the store's own failure - a connection, a timeout + raise _store_failure(exc) from exc + if not written: + raise AppException( + 409, + f"conversation {conversation} moved on before turn {turn} was saved - " + "a duplicated turn", + ) + _annotate(done) + annotate_trace("llm_status", status) + annotate_trace("llm_turn", str(turn)) + annotate_trace("llm_cache_creation_input_tokens", str(done.cache_creation_input_tokens)) + annotate_trace("llm_cache_read_input_tokens", str(done.cache_read_input_tokens)) + # status, turn and usage only - no prompt, completion, tool argument or result reaches a log + log.info( + "llm.chat model=%s status=%s turn=%d stop_reason=%s input_tokens=%d output_tokens=%d " + "cache_read_input_tokens=%d request_id=%s elapsed_ms=%d", + done.model, + status, + turn, + done.stop_reason, + done.input_tokens, + done.output_tokens, + done.cache_read_input_tokens, + done.request_id or "none", + round((time.perf_counter() - started) * 1000), + ) + reply: dict[str, Any] = { + **outcome, + "conversation": conversation, + "turn": turn, + "model": done.model, + "stop_reason": done.stop_reason, + "usage": { + "input_tokens": done.input_tokens, + "output_tokens": done.output_tokens, + "cache_creation_input_tokens": done.cache_creation_input_tokens, + "cache_read_input_tokens": done.cache_read_input_tokens, + }, + } + if done.request_id: + reply["request_id"] = done.request_id + return reply + + # --- the functions ----------------------------------------------------------------------- @@ -608,7 +1205,14 @@ async def llm_chat(_headers: dict[str, str], body: Body) -> dict[str, Any]: ``params.timeout_ms`` bounds the whole call, SDK retries included - the x-ttl pattern. A reply that carries nothing usable (refused, empty, or a schema reply cut off) is a 422. + + A request that carries ``tools`` is one turn of an agent loop (RFC-0011): ``conversation``, + ``turn``, ``results``, ``ttl`` and ``retention`` come with it, and the reply carries + ``status`` (done, tool_use, refusal or cap), ``calls``, ``conversation`` and ``turn``. The + transcript stays in this app's store; see README.md, "The loop contract". """ + if isinstance(body, dict) and _is_loop(body): + return await _loop_turn(body) request = prepare(body, streaming=False) started = time.perf_counter() try: @@ -726,6 +1330,9 @@ async def llm_health(headers: dict[str, str], _body: Body) -> Any: info["region"] = region # which region a Bedrock call is signed for if backend.api: info["api"] = backend.api # which Bedrock surface answers + store = _prop("llm.transcript.store") + if store: + info["transcript_store"] = store # where a loop's transcripts are kept return info problem = backend.credential_problem() if problem: diff --git a/extensions/llm-helper/resources/application.yml b/extensions/llm-helper/resources/application.yml index f9ac7af..1d32e19 100644 --- a/extensions/llm-helper/resources/application.yml +++ b/extensions/llm-helper/resources/application.yml @@ -21,7 +21,7 @@ mandatory.health.dependencies: 'llm.health' # Opt-in lists for the /env endpoint - nothing is ever dumped wholesale. show.env.variables: 'LOG_LEVEL' -show.application.properties: 'application.name, rest.server.port, llm.backend, llm.model' +show.application.properties: 'application.name, rest.server.port, llm.backend, llm.model, llm.transcript.store' # --- the helper's settings (a call's params.* win over these) ----------------------------- # The credential is never configured here: ANTHROPIC_API_KEY (the Claude API) or the AWS @@ -48,6 +48,23 @@ llm.fallbacks: 'default' # its text), so the hop that holds batches back can be told apart from the one that forwards them # llm.log.batches: true +# --- the agent loop's turns (llm.chat with tools): the transcript store ------------------- +# memory keeps the transcripts in this process - development and a single replica. With more +# than one replica a turn may land on any of them: set redis (the [redis] extra) +llm.transcript.store: 'memory' +# a loop turn's defaults when the engine's agent loop sends none; it normally sends both +# llm.transcript.ttl: '7d' +# llm.transcript.retention: '0' +# the Redis connection for llm.transcript.store: redis - the engines' redis.* set under this +# store's own prefix; the password comes from the environment, never from this file +# llm.transcript.redis.host: '127.0.0.1' +# llm.transcript.redis.port: 6379 +# llm.transcript.redis.username: '' +# llm.transcript.redis.password: '${REDIS_PASSWORD:}' +# llm.transcript.redis.ssl: false +# llm.transcript.redis.database: 0 +# llm.transcript.redis.timeout.ms: 5000 + # OpenTelemetry forwarder (opt-in) - the engines' opentelemetry-forwarder twin. Switch it on # at run time with -Dotel.forwarding=true; the endpoint and credential come from the # environment (a Dynatrace endpoint ends in /api/v2/otlp/v1/traces; the header is the whole diff --git a/pyproject.toml b/pyproject.toml index fcce8bf..90db96a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,18 @@ llm = ["anthropic>=1,<2"] # AWS Bedrock as the helper's backend (llm.backend: bedrock) - boto3 signs each call with the # AWS credential chain; the Claude API install stays lean without it bedrock = ["anthropic[bedrock]>=1,<2"] -dev = ["pytest>=8", "pytest-asyncio>=0.23", "ruff>=0.9", "anthropic[bedrock]>=1,<2"] +# Redis as the helper's transcript store (llm.transcript.store: redis) - an agent loop's turns +# can then land on any replica; a single process keeps its transcripts in memory without it +redis = ["redis>=5,<9"] +dev = [ + "pytest>=8", + "pytest-asyncio>=0.23", + "ruff>=0.9", + "anthropic[bedrock]>=1,<2", + "redis>=5,<9", + # the store's compare-and-set is a Lua script; the [lua] extra runs it in the fake + "fakeredis[lua]>=2.26", +] [project.scripts] mercury-serve = "mercury_composable.cli:main" diff --git a/tests/test_llm_helper.py b/tests/test_llm_helper.py index 44f5c34..16ef985 100644 --- a/tests/test_llm_helper.py +++ b/tests/test_llm_helper.py @@ -1,30 +1,34 @@ """The LLM helper (extensions/llm-helper/llm_helper.py) - token-free. -One shared contract file, tests/vectors/llm-helper-vectors.json (byte-identical in -mercury-nodejs), drives every case below against a fake of the Anthropic SDK: the exact SDK -call, the reply map, the error contract and the streaming segments are pinned without -spending a token or needing a credential, and the Node.js twin runs the same file, so the two -helpers cannot drift apart. The tests after the vector runs pin what a vector cannot: token -batches are never held back, no prompt text reaches a log, the trace annotations, the health -route and the backend seam. +Two shared contract files, tests/vectors/llm-helper-vectors.json and llm-loop-vectors.json +(byte-identical in mercury-nodejs), drive every case below against a fake of the Anthropic +SDK: the exact SDK call, the reply map, the error contract, the streaming segments and the +loop's stored record are pinned without spending a token or needing a credential, and the +Node.js twin runs the same files, so the two helpers cannot drift apart. The tests after the +vector runs pin what a vector cannot: token batches are never held back, no prompt text +reaches a log, the trace annotations, the health route, the backend seam and the Redis store. """ from __future__ import annotations import asyncio +import copy import hashlib import importlib.util import json import logging +import os import sys from pathlib import Path from types import SimpleNamespace from typing import Any import anthropic +import fakeredis import httpx2 import pytest from anthropic.lib.bedrock import AsyncAnthropicBedrock, AsyncAnthropicBedrockMantle +from anthropic.types import RedactedThinkingBlock, TextBlock, ThinkingBlock, ToolUseBlock from mercury_composable import AppException @@ -47,6 +51,14 @@ CHAT_CASES: list[dict[str, Any]] = [c for c in VECTORS["cases"] if c["route"] == "llm.chat"] STREAM_CASES: list[dict[str, Any]] = [c for c in VECTORS["cases"] if c["route"] == "llm.stream"] +_LOOP_VECTORS = _TESTS / "vectors" / "llm-loop-vectors.json" +# the Node.js twin pins the same digest: change the file in both packs or in neither +_LOOP_VECTORS_SHA256 = "8b6644861643daf36ec7879b825dd6e0268a8bae53007c678043e31d9171f8aa" +LOOP_VECTORS: dict[str, Any] = json.loads(_LOOP_VECTORS.read_text(encoding="utf-8")) +LOOP_CASES: list[dict[str, Any]] = LOOP_VECTORS["cases"] +# the cases a Redis server can run as well: its clock is the wall clock, so none that moves time +LOOP_CASES_ON_REDIS = [c for c in LOOP_CASES if not any("advance_s" in s for s in c["steps"])] + _REQUEST = httpx2.Request("POST", "https://api.anthropic.com/v1/messages") _STATUS_CLASSES: dict[int, type[anthropic.APIStatusError]] = { 400: anthropic.BadRequestError, @@ -82,6 +94,14 @@ def __init__(self) -> None: self.cancelled = False +class _FakeBlock(SimpleNamespace): + """A content block as the SDK returns it: attributes, and to_dict() for the fields set.""" + + def to_dict(self, *, mode: str = "python") -> dict[str, Any]: + assert mode == "json", "a stored block is JSON" + return copy.deepcopy(vars(self)) + + class _FakeStream: """What ``client.messages.stream(...)`` returns: an async context manager.""" @@ -132,10 +152,13 @@ async def batches() -> Any: async def get_final_message(self) -> Any: provider = self._provider details: Any = provider.get("stop_details") + if "content" in provider: # a loop turn's blocks: thinking, text, tool_use + content: list[Any] = [_FakeBlock(**copy.deepcopy(b)) for b in provider["content"]] + else: + text = provider.get("text", "".join(provider["deltas"])) + content = [SimpleNamespace(type="text", text=text)] return SimpleNamespace( - content=[ - SimpleNamespace(type="text", text=provider.get("text", "".join(provider["deltas"]))) - ], + content=content, model=provider["model"], stop_reason=provider["stop_reason"], usage=SimpleNamespace(**provider["usage"]), @@ -291,6 +314,375 @@ async def test_stream_contract(case: dict[str, Any], monkeypatch: pytest.MonkeyP assert out.error is None, out.error +# --- the shared contract: a loop turn of llm.chat (RFC-0011) ----------------------------- + + +class _Clock: + """The memory store's clock; a step's advance_s moves it.""" + + def __init__(self) -> None: + self.now = 1000.0 + + def __call__(self) -> float: + return self.now + + +def _fake_redis() -> Any: + """A Redis of its own per test, with the Lua the store's compare-and-set runs.""" + return fakeredis.FakeAsyncRedis(server=fakeredis.FakeServer(), decode_responses=True) + + +async def _stored(store: Any, clock: _Clock, conversation: str | None) -> dict[str, Any] | None: + """The record a vector's store expectation describes, read from either store.""" + if store is None or conversation is None: + return None + if isinstance(store, helper.MemoryStore): + entry = store._records.get(conversation) + if entry is None or entry[0] <= clock.now: + return None + record = helper.Transcript.from_json(entry[2]) + ttl_s = round(entry[0] - clock.now) + else: + key = helper.TRANSCRIPT_PREFIX + conversation + text = await store._client.hget(key, "record") + if text is None: + return None + record = helper.Transcript.from_json(text) + ttl_s = round(await store._client.pttl(key) / 1000) + return { + "turn": record.turn, + "closed": record.closed, + "pending": record.pending, + "ttl_s": ttl_s, + "messages": len(record.messages), + } + + +async def _loop_step( + route: str, body: Any, monkeypatch: pytest.MonkeyPatch +) -> tuple[Any, AppException | None]: + if route == "llm.stream": + out = _install_writer(monkeypatch) + await helper.llm_stream({}, _event(body)) + return None, out.error + try: + return await helper.llm_chat({}, body), None + except AppException as exc: + return None, exc + + +async def _run_loop_case( + case: dict[str, Any], monkeypatch: pytest.MonkeyPatch, kind: str, redis_client: Any = None +) -> None: + """Run a case's steps against the memory store, or a Redis store (a fake unless a client is + given) where the case configures the memory one.""" + config: dict[str, str] = case["config"] + clock = _Clock() + store: Any = None + if config.get("llm.transcript.store") == "memory" and kind == "memory": + store = helper.MemoryStore(clock) + elif config.get("llm.transcript.store") == "memory": + store = helper.RedisStore(redis_client or _fake_redis()) + monkeypatch.setattr(helper, "_store", store) + ids = iter(case["conversation_ids"]) + monkeypatch.setattr(helper, "_new_conversation_id", lambda: next(ids)) + first = case["conversation_ids"][0] if case["conversation_ids"] else None + for index, step in enumerate(case["steps"]): + clock.now += step.get("advance_s", 0) + client = _FakeClient(step.get("provider")) + _install_backend(monkeypatch, client, config) + body = step["body"] + result, error = await _loop_step(case["route"], body, monkeypatch) + expect = step["expect"] + _assert_sdk(client, {"expect": expect, "provider": step.get("provider")}) + if "error" in expect: + _assert_error(error, expect["error"]) + else: + assert error is None, error + assert result == expect["result"], f"step {index}" + named = body.get("conversation") if isinstance(body, dict) else None + conversation = step.get("conversation") or named or first + assert await _stored(store, clock, conversation) == expect["store"], f"step {index}" + + +def test_the_loop_vector_file_is_the_one_the_node_twin_runs() -> None: + digest = hashlib.sha256(_LOOP_VECTORS.read_bytes()).hexdigest() + assert digest == _LOOP_VECTORS_SHA256, "the shared loop vector file changed - update both packs" + assert LOOP_VECTORS["contract"] == "llm-chat-loop" + assert LOOP_VECTORS["version"] == 1 + assert len({c["name"] for c in LOOP_CASES}) == len(LOOP_CASES), "case names are unique" + + +@pytest.mark.parametrize("case", LOOP_CASES, ids=[c["name"] for c in LOOP_CASES]) +async def test_loop_contract(case: dict[str, Any], monkeypatch: pytest.MonkeyPatch) -> None: + await _run_loop_case(case, monkeypatch, "memory") + + +@pytest.mark.parametrize("case", LOOP_CASES_ON_REDIS, ids=[c["name"] for c in LOOP_CASES_ON_REDIS]) +async def test_loop_contract_on_the_redis_store( + case: dict[str, Any], monkeypatch: pytest.MonkeyPatch +) -> None: + """The same turns with the Redis store under them: the store is invisible to the contract.""" + await _run_loop_case(case, monkeypatch, "redis") + + +# --- the loop turn: what a vector cannot pin ----------------------------------------------- + + +def test_a_stored_block_is_the_json_the_sdk_would_send_back() -> None: + """The record keeps each block as the SDK sends a returned block back - the fields the API + set and nothing else - so a thinking block's signature returns to the API unchanged.""" + blocks = [ + ThinkingBlock(type="thinking", thinking="", signature="sig-opaque"), + RedactedThinkingBlock(type="redacted_thinking", data="opaque"), + TextBlock(type="text", text="Checking the limit."), + ToolUseBlock(type="tool_use", id="toolu_01", name="credit_limit", input={"id": "C-1"}), + ] + assert [helper._block(b) for b in blocks] == [ + {"type": "thinking", "thinking": "", "signature": "sig-opaque"}, + {"type": "redacted_thinking", "data": "opaque"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}, + ] + + +_TURN = { + "kind": "message", "model": "claude-opus-5-5", "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "INPUT-MARKER-1"}}, + ], + "usage": {"input_tokens": 40, "output_tokens": 9, "cache_creation_input_tokens": 30, + "cache_read_input_tokens": 0}, + "request_id": "req_loop_trace", +} # fmt: skip +_TOOL = {"name": "credit_limit", "description": "A customer's credit limit.", + "input_schema": {"type": "object"}} # fmt: skip +_LOOP_CONFIG = {"llm.transcript.store": "memory"} + + +async def test_a_loop_turn_logs_status_and_usage_and_never_content( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setattr(helper, "_store", helper.MemoryStore()) + _install_backend(monkeypatch, _FakeClient(_TURN), _LOOP_CONFIG) + caplog.set_level(logging.DEBUG) + reply = await helper.llm_chat( + {}, {"prompt": "PROMPT-MARKER-2", "system": "SYSTEM-MARKER-3", "tools": [_TOOL]} + ) + final = {**_TURN, "stop_reason": "end_turn", + "content": [{"type": "text", "text": "REPLY-MARKER-4"}]} # fmt: skip + _install_backend(monkeypatch, _FakeClient(final), _LOOP_CONFIG) + results = [{"id": "toolu_01", "output": "RESULT-MARKER-5"}] + body = {"tools": [_TOOL], "conversation": reply["conversation"], "turn": 1, "results": results} + await helper.llm_chat({}, body) + assert "llm.chat model=claude-opus-5-5 status=tool_use turn=1" in caplog.text + assert "llm.chat model=claude-opus-5-5 status=done turn=2" in caplog.text + for marker in ("PROMPT", "SYSTEM", "INPUT", "REPLY", "RESULT"): + assert f"{marker}-MARKER" not in caplog.text, marker + assert reply["conversation"] not in caplog.text, "the reference to the transcript stays off" + + +async def test_a_loop_turn_annotates_its_status_turn_and_cache_usage( + monkeypatch: pytest.MonkeyPatch, +) -> None: + seen: dict[str, Any] = {} + monkeypatch.setattr(helper, "annotate_trace", lambda key, value: seen.__setitem__(key, value)) + monkeypatch.setattr(helper, "_store", helper.MemoryStore()) + _install_backend(monkeypatch, _FakeClient(_TURN), _LOOP_CONFIG) + await helper.llm_chat({}, {"prompt": "x", "tools": [_TOOL]}) + assert seen == { + "llm_model": "claude-opus-5-5", + "llm_stop_reason": "tool_use", + "llm_input_tokens": "40", + "llm_output_tokens": "9", + "llm_request_id": "req_loop_trace", + "llm_status": "tool_use", + "llm_turn": "1", + "llm_cache_creation_input_tokens": "30", + "llm_cache_read_input_tokens": "0", + } + + +class _BrokenStore: + """A store whose server is gone, or whose compare-and-set another writer won.""" + + name = "broken" + + def __init__(self, *, load: Exception | None = None, save: Exception | bool = True) -> None: + self._load = load + self._save = save + + async def load(self, _conversation: str) -> Any: + if self._load is not None: + raise self._load + return None + + async def save(self, _conversation: str, _turn: int, _record: Any, _ttl_s: int) -> bool: + if isinstance(self._save, Exception): + raise self._save + return self._save + + +async def test_a_store_that_cannot_be_reached_is_a_503_by_name( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _install_backend(monkeypatch, _FakeClient(_TURN), _LOOP_CONFIG) + refused = ConnectionError("Connection refused") + monkeypatch.setattr(helper, "_store", _BrokenStore(save=refused)) + with pytest.raises(AppException) as raised: + await helper.llm_chat({}, {"prompt": "x", "tools": [_TOOL]}) + assert raised.value.status == 503 + assert raised.value.message == ( + "transcript store unreachable - ConnectionError: Connection refused" + ) + monkeypatch.setattr(helper, "_store", _BrokenStore(load=TimeoutError("Timeout reading"))) + body = {"tools": [_TOOL], "conversation": "0" * 32, "turn": 1, "results": []} + with pytest.raises(AppException) as raised: + await helper.llm_chat({}, body) + assert raised.value.status == 503 + assert "TimeoutError: Timeout reading" in raised.value.message + + +async def test_a_turn_that_loses_the_compare_and_set_is_a_409( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Two copies of one turn: both pass the read check, the second write is refused, so the + conversation never forks - the model was called, and the turn is reported as duplicated.""" + client = _FakeClient(_TURN) + _install_backend(monkeypatch, client, _LOOP_CONFIG) + monkeypatch.setattr(helper, "_store", _BrokenStore(save=False)) + monkeypatch.setattr(helper, "_new_conversation_id", lambda: "0" * 32) + with pytest.raises(AppException) as raised: + await helper.llm_chat({}, {"prompt": "x", "tools": [_TOOL]}) + assert raised.value.status == 409 + assert "moved on before turn 1 was saved - a duplicated turn" in raised.value.message + assert len(client.ledger.calls) == 1 + + +async def test_health_reports_the_transcript_store(monkeypatch: pytest.MonkeyPatch) -> None: + _install_backend(monkeypatch, _FakeClient(None), {"llm.transcript.store": "redis"}) + info = await helper.llm_health({"type": "info"}, None) + assert info["transcript_store"] == "redis" + + +def test_the_memory_store_is_built_once(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(helper, "_store", None) + monkeypatch.setattr(helper, "_prop", lambda key, default=None: _LOOP_CONFIG.get(key, default)) + store = helper.get_store() + assert isinstance(store, helper.MemoryStore) + assert helper.get_store() is store + + +# --- the Redis transcript store ------------------------------------------------------------ + + +def _record(turn: int, closed: bool = False) -> Any: + return helper.Transcript( + "claude-opus-5-5", [{"role": "user", "content": "x"}], turn, [], closed + ) + + +async def test_the_redis_store_keeps_one_hash_under_the_transcript_prefix() -> None: + """A Redis ACL can confine the helper's user to ~transcript:*, beside the graph's suspend + store under graph:*; the hash holds the turn count the compare-and-set reads.""" + client = _fake_redis() + store = helper.RedisStore(client) + assert await store.save("a" * 32, 0, _record(1), 1800) is True + assert await client.keys("*") == ["transcript:" + "a" * 32] + assert await client.hget("transcript:" + "a" * 32, "turn") == "1" + assert 1799 <= await client.ttl("transcript:" + "a" * 32) <= 1800 + loaded = await store.load("a" * 32) + assert loaded == _record(1) + + +async def test_the_redis_store_writes_only_over_the_turn_it_expects() -> None: + store = helper.RedisStore(_fake_redis()) + conversation = "b" * 32 + assert await store.save(conversation, 0, _record(1), 60) is True + assert await store.save(conversation, 0, _record(1), 60) is False, "a new record exists" + assert await store.save(conversation, 2, _record(3), 60) is False, "the record holds turn 1" + assert await store.save(conversation, 1, _record(2), 60) is True + assert (await store.load(conversation)) == _record(2) + assert await store.save("c" * 32, 1, _record(2), 60) is False, "no record to append to" + + +async def test_the_redis_store_deletes_a_record_saved_with_no_time_to_live() -> None: + """Retention 0: the run's end removes the record instead of writing it.""" + client = _fake_redis() + store = helper.RedisStore(client) + conversation = "d" * 32 + assert await store.save(conversation, 0, _record(1), 60) is True + assert await store.save(conversation, 1, _record(2, closed=True), 0) is True + assert await client.exists("transcript:" + conversation) == 0 + assert await store.load(conversation) is None + + +def test_the_redis_store_is_built_from_its_connection_keys(monkeypatch: pytest.MonkeyPatch) -> None: + settings = { + "llm.transcript.store": "redis", + "llm.transcript.redis.host": "redis.internal", + "llm.transcript.redis.port": "6380", + "llm.transcript.redis.username": "llm-helper", + "llm.transcript.redis.password": "test-only-password", + "llm.transcript.redis.ssl": "true", + "llm.transcript.redis.database": "2", + "llm.transcript.redis.timeout.ms": "1500", + } + monkeypatch.setattr(helper, "_store", None) + monkeypatch.setattr(helper, "_prop", lambda key, default=None: settings.get(key, default)) + store = helper.get_store() + assert isinstance(store, helper.RedisStore) + pool = store._client.connection_pool + kwargs = pool.connection_kwargs + assert (kwargs["host"], kwargs["port"], kwargs["db"]) == ("redis.internal", 6380, 2) + assert (kwargs["username"], kwargs["password"]) == ("llm-helper", "test-only-password") + assert kwargs["socket_timeout"] == 1.5 + assert kwargs["decode_responses"] is True + assert pool.connection_class.__name__ == "SSLConnection" + + +def test_the_redis_store_without_redis_is_a_503_that_names_the_extra( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(helper, "_store", None) + monkeypatch.setattr( + helper, + "_prop", + lambda key, default=None: "redis" if key == "llm.transcript.store" else default, + ) + monkeypatch.setattr(helper.importlib.util, "find_spec", lambda _name: None) + with pytest.raises(AppException) as raised: + helper.get_store() + assert raised.value.status == 503 + assert "mercury-composable[redis]" in raised.value.message + + +@pytest.mark.skipif( + not os.environ.get("LLM_TRANSCRIPT_REDIS_URL"), + reason="a live Redis: set LLM_TRANSCRIPT_REDIS_URL, e.g. redis://127.0.0.1:6379/15", +) +@pytest.mark.parametrize("case", LOOP_CASES_ON_REDIS, ids=[c["name"] for c in LOOP_CASES_ON_REDIS]) +async def test_loop_contract_on_a_live_redis( + case: dict[str, Any], monkeypatch: pytest.MonkeyPatch +) -> None: + """The same turns on a real server: the Lua the fake runs is the script Redis runs. Only the + keys of the vectors' own conversations are touched.""" + import redis.asyncio as redis_async + + client = redis_async.from_url(os.environ["LLM_TRANSCRIPT_REDIS_URL"], decode_responses=True) + named = [s["body"].get("conversation") for s in case["steps"] if isinstance(s["body"], dict)] + keys = {helper.TRANSCRIPT_PREFIX + c for c in [*case["conversation_ids"], *named] if c} + try: + if keys: + await client.delete(*keys) + await _run_loop_case(case, monkeypatch, "redis", client) + finally: + if keys: + await client.delete(*keys) + await client.aclose() + + # --- progressive rendering: a token batch is never held back ----------------------------- diff --git a/tests/vectors/llm-loop-vectors.json b/tests/vectors/llm-loop-vectors.json new file mode 100644 index 0000000..314e2da --- /dev/null +++ b/tests/vectors/llm-loop-vectors.json @@ -0,0 +1,6296 @@ +{ + "contract": "llm-chat-loop", + "version": 1, + "description": "The loop contract of llm.chat (RFC-0011): one model turn of an agent loop per call, the transcript in the helper's own store. mercury-python and mercury-nodejs ship this file byte-identical and run every case against their own fake of the Anthropic SDK and their memory transcript store. A case names the route, the config keys in force (the harness installs the fake backend under llm.backend, as for llm-helper-vectors.json, and a memory store when llm.transcript.store is memory), the ids the helper draws for new conversations, in order, and its steps. A step may first advance the store's clock by advance_s seconds; it names the request body and what the fake provider does (the provider kinds of llm-helper-vectors.json, a message carrying content blocks that the fake returns as the SDK does); the expectation is the exact SDK call when one is made (namespace, parameters, options), then either the reply map or the error (status and message fragments), and the stored record of the step's conversation afterwards (the step's conversation field, else its body's, else the case's first id): null when none is held, else its turn count, whether it is closed, the pending call ids, the seconds it has left to live and its number of messages.", + "cases": [ + { + "name": "loop/prep/a-loop-field-without-tools", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": {"prompt": "May C-1 order 4200 EUR?", "conversation": "0123456789abcdef0123456789abcdef"}, + "expect": { + "error": { + "status": 400, + "message_contains": ["conversation is a field of the loop contract - it needs tools"] + }, + "store": null + } + } + ] + }, + { + "name": "loop/prep/tools-not-a-list", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": {"prompt": "May C-1 order 4200 EUR?", "tools": {"name": "credit_limit"}}, + "expect": { + "error": { + "status": 400, + "message_contains": ["tools must be a non-empty list of {name, description, input_schema}"] + }, + "store": null + } + } + ] + }, + { + "name": "loop/prep/tools-empty", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": {"prompt": "May C-1 order 4200 EUR?", "tools": []}, + "expect": { + "error": {"status": 400, "message_contains": ["tools must be a non-empty list"]}, + "store": null + } + } + ] + }, + { + "name": "loop/prep/tool-not-a-map", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": {"prompt": "May C-1 order 4200 EUR?", "tools": ["credit_limit"]}, + "expect": {"error": {"status": 400, "message_contains": ["tools[0] must be a map"]}, "store": null} + } + ] + }, + { + "name": "loop/prep/tool-carries-its-route-and-approval", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true, + "route": "credit.limit.lookup", + "approval": "required" + } + ] + }, + "expect": { + "error": { + "status": 400, + "message_contains": [ + "tools[0]: unsupported keys: approval, route", + "supported: name, description, input_schema, strict" + ] + }, + "store": null + } + } + ] + }, + { + "name": "loop/prep/tool-name-is-a-route-name", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit.limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + } + ] + }, + "expect": { + "error": {"status": 400, "message_contains": ["tools[0].name must match ^[a-zA-Z0-9_-]{1,128}$"]}, + "store": null + } + } + ] + }, + { + "name": "loop/prep/tool-name-too-long", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + } + ] + }, + "expect": { + "error": {"status": 400, "message_contains": ["tools[0].name must match"]}, + "store": null + } + } + ] + }, + { + "name": "loop/prep/tool-name-not-unique", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + }, + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + } + ] + }, + "expect": { + "error": {"status": 400, "message_contains": ["tools[2].name 'credit_limit' is not unique"]}, + "store": null + } + } + ] + }, + { + "name": "loop/prep/tool-without-description", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + } + ] + }, + "expect": { + "error": {"status": 400, "message_contains": ["tools[0].description must be a non-empty string"]}, + "store": null + } + } + ] + }, + { + "name": "loop/prep/tool-schema-not-a-map", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": "object", + "strict": true + } + ] + }, + "expect": { + "error": {"status": 400, "message_contains": ["tools[0].input_schema must be a JSON schema map"]}, + "store": null + } + } + ] + }, + { + "name": "loop/prep/tool-strict-not-a-boolean", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": "yes" + } + ] + }, + "expect": { + "error": {"status": 400, "message_contains": ["tools[0].strict must be true or false"]}, + "store": null + } + } + ] + }, + { + "name": "loop/prep/conversation-not-a-reference", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "transcript:*", + "turn": 1, + "results": [] + }, + "expect": { + "error": { + "status": 400, + "message_contains": ["conversation must be the reference a previous turn returned"] + }, + "store": null + } + } + ] + }, + { + "name": "loop/prep/conversation-in-upper-case", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789ABCDEF0123456789ABCDEF", + "turn": 1, + "results": [] + }, + "expect": { + "error": { + "status": 400, + "message_contains": ["conversation must be the reference a previous turn returned"] + }, + "store": null + } + } + ] + }, + { + "name": "loop/prep/turn-negative", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": -1, + "results": [] + }, + "expect": { + "error": {"status": 400, "message_contains": ["turn must be a whole number >= 0"]}, + "store": null + } + } + ] + }, + { + "name": "loop/prep/results-not-a-list", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": {"id": "toolu_01"} + }, + "expect": { + "error": { + "status": 400, + "message_contains": ["results must be a list of {id, output} or {id, error}"] + }, + "store": null + } + } + ] + }, + { + "name": "loop/prep/result-without-id", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"output": 5000}] + }, + "expect": { + "error": {"status": 400, "message_contains": ["results[0].id must be a call id"]}, + "store": null + } + } + ] + }, + { + "name": "loop/prep/result-with-output-and-error", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": 5000, "error": "timeout"}] + }, + "expect": { + "error": {"status": 400, "message_contains": ["results[0] must carry either output or error"]}, + "store": null + } + } + ] + }, + { + "name": "loop/prep/result-with-neither", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01"}] + }, + "expect": { + "error": {"status": 400, "message_contains": ["results[0] must carry either output or error"]}, + "store": null + } + } + ] + }, + { + "name": "loop/prep/result-with-another-key", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": 5000, "route": "credit.limit.lookup"}] + }, + "expect": { + "error": { + "status": 400, + "message_contains": ["results[0]: unsupported keys: route - supported: id, output, error"] + }, + "store": null + } + } + ] + }, + { + "name": "loop/prep/ttl-not-a-duration", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "ttl": "2 days" + }, + "expect": { + "error": {"status": 400, "message_contains": ["ttl must be a duration such as 30m, 12h or 2d"]}, + "store": null + } + } + ] + }, + { + "name": "loop/prep/ttl-zero", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "ttl": 0 + }, + "expect": { + "error": {"status": 400, "message_contains": ["ttl must be a duration", ">= 1"]}, + "store": null + } + } + ] + }, + { + "name": "loop/prep/ttl-a-boolean", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "ttl": true + }, + "expect": {"error": {"status": 400, "message_contains": ["ttl must be a duration"]}, "store": null} + } + ] + }, + { + "name": "loop/prep/retention-not-a-duration", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "retention": "forever" + }, + "expect": { + "error": { + "status": 400, + "message_contains": ["retention must be a duration such as 30m, 12h or 2d", ">= 0"] + }, + "store": null + } + } + ] + }, + { + "name": "loop/prep/first-turn-with-results", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "results": [] + }, + "expect": { + "error": { + "status": 400, + "message_contains": ["results answer a previous turn - a first turn carries none"] + }, + "store": null + } + } + ] + }, + { + "name": "loop/prep/first-turn-with-a-turn-count", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "turn": 2 + }, + "expect": { + "error": {"status": 400, "message_contains": ["turn must be 0 or absent on a first turn"]}, + "store": null + } + } + ] + }, + { + "name": "loop/prep/first-turn-without-prompt", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "expect": { + "error": {"status": 400, "message_contains": ["missing 'prompt' or 'messages'"]}, + "store": null + } + } + ] + }, + { + "name": "loop/prep/later-turn-without-turn", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "results": [] + }, + "expect": { + "error": {"status": 400, "message_contains": ["turn is required with conversation"]}, + "store": null + } + } + ] + }, + { + "name": "loop/prep/later-turn-with-a-prompt", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [], + "prompt": "And customer C-2?" + }, + "expect": { + "error": { + "status": 400, + "message_contains": ["a later turn carries results, never prompt or messages"] + }, + "store": null + } + } + ] + }, + { + "name": "loop/prep/params-are-checked-as-on-any-call", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "params": {"temperature": 0.2} + }, + "expect": { + "error": {"status": 400, "message_contains": ["unsupported params: temperature"]}, + "store": null + } + } + ] + }, + { + "name": "loop/prep/no-store-configured", + "route": "llm.chat", + "config": {}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "expect": { + "error": { + "status": 503, + "message_contains": ["transcript store not configured - set llm.transcript.store"] + }, + "store": null + } + } + ] + }, + { + "name": "loop/prep/an-unknown-store", + "route": "llm.chat", + "config": {"llm.transcript.store": "postgres"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "expect": { + "error": { + "status": 500, + "message_contains": ["llm.transcript.store must be one of: memory, redis"] + }, + "store": null + } + } + ] + }, + { + "name": "loop/prep/tools-on-the-streaming-route", + "route": "llm.stream", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "expect": { + "error": { + "status": 400, + "message_contains": ["tools and the loop fields are not part of the streaming contract - use llm.chat"] + }, + "store": null + } + } + ] + }, + { + "name": "loop/prep/a-conversation-on-the-streaming-route", + "route": "llm.stream", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": {"prompt": "May C-1 order 4200 EUR?", "conversation": "0123456789abcdef0123456789abcdef"}, + "expect": { + "error": {"status": 400, "message_contains": ["not part of the streaming contract"]}, + "store": null + } + } + ] + }, + { + "name": "loop/first-turn-asks-for-a-tool", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + } + ] + }, + { + "name": "loop/run-to-done-sends-the-transcript-back-unchanged", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": {"limit": 5000, "currency": "EUR"}}] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-2"}, + {"type": "text", "text": "4200 EUR is within the limit of 5000 EUR."} + ], + "usage": { + "input_tokens": 520, + "output_tokens": 40, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 380 + }, + "request_id": "req_loop_002" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [ + {"role": "user", "content": "May C-1 order 4200 EUR?"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + { + "type": "tool_use", + "id": "toolu_01", + "name": "credit_limit", + "input": {"id": "C-1"} + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": "{\"limit\":5000,\"currency\":\"EUR\"}" + } + ] + } + ], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "done", + "text": "4200 EUR is within the limit of 5000 EUR.", + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 2, + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 520, + "output_tokens": 40, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 380 + }, + "request_id": "req_loop_002" + }, + "store": null + } + } + ] + }, + { + "name": "loop/parallel-calls-are-answered-in-call-order", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}, + { + "type": "tool_use", + "id": "toolu_02", + "name": "orders_search", + "input": {"id": "C-1"} + } + ], + "usage": { + "input_tokens": 412, + "output_tokens": 80, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [ + {"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}, + {"id": "toolu_02", "name": "orders_search", "input": {"id": "C-1"}} + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 80, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "store": { + "turn": 1, + "closed": false, + "pending": ["toolu_01", "toolu_02"], + "ttl_s": 604800, + "messages": 2 + } + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [ + {"id": "toolu_02", "output": []}, + {"id": "toolu_01", "error": {"status": 404, "message": "no such customer"}} + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-2"}, + {"type": "text", "text": "The customer is unknown."} + ], + "usage": { + "input_tokens": 600, + "output_tokens": 20, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [ + {"role": "user", "content": "May C-1 order 4200 EUR?"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + { + "type": "tool_use", + "id": "toolu_01", + "name": "credit_limit", + "input": {"id": "C-1"} + }, + { + "type": "tool_use", + "id": "toolu_02", + "name": "orders_search", + "input": {"id": "C-1"} + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": "{\"status\":404,\"message\":\"no such customer\"}", + "is_error": true + }, + {"type": "tool_result", "tool_use_id": "toolu_02", "content": "[]"} + ] + } + ], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "done", + "text": "The customer is unknown.", + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 2, + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 600, + "output_tokens": 20, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "store": null + } + } + ] + }, + { + "name": "loop/text-output-and-text-error-are-sent-as-written", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}, + { + "type": "tool_use", + "id": "toolu_02", + "name": "orders_search", + "input": {"id": "C-1"} + } + ], + "usage": { + "input_tokens": 412, + "output_tokens": 80, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [ + {"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}, + {"id": "toolu_02", "name": "orders_search", "input": {"id": "C-1"}} + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 80, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "store": { + "turn": 1, + "closed": false, + "pending": ["toolu_01", "toolu_02"], + "ttl_s": 604800, + "messages": 2 + } + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [ + {"id": "toolu_01", "output": "5000 EUR"}, + {"id": "toolu_02", "error": "orders.search timed out after 3000 ms"} + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-2"}, + {"type": "text", "text": "The customer is unknown."} + ], + "usage": { + "input_tokens": 600, + "output_tokens": 20, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [ + {"role": "user", "content": "May C-1 order 4200 EUR?"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + { + "type": "tool_use", + "id": "toolu_01", + "name": "credit_limit", + "input": {"id": "C-1"} + }, + { + "type": "tool_use", + "id": "toolu_02", + "name": "orders_search", + "input": {"id": "C-1"} + } + ] + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_01", "content": "5000 EUR"}, + { + "type": "tool_result", + "tool_use_id": "toolu_02", + "content": "orders.search timed out after 3000 ms", + "is_error": true + } + ] + } + ], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "done", + "text": "The customer is unknown.", + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 2, + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 600, + "output_tokens": 20, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "store": null + } + } + ] + }, + { + "name": "loop/a-verdict-schema-after-a-tool-returns-data", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "schema": { + "type": "object", + "properties": {"decision": {"type": "boolean"}, "reason": {"type": "string"}}, + "required": ["decision", "reason"] + } + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ], + "output_config": { + "format": { + "type": "json_schema", + "schema": { + "additionalProperties": false, + "type": "object", + "properties": {"decision": {"type": "boolean"}, "reason": {"type": "string"}}, + "required": ["decision", "reason"] + } + } + } + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + } + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": {"limit": 5000, "currency": "EUR"}}], + "schema": { + "type": "object", + "properties": {"decision": {"type": "boolean"}, "reason": {"type": "string"}}, + "required": ["decision", "reason"] + } + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-2"}, + {"type": "text", "text": "{\"decision\":false,\"reason\":\"within the limit\"}"} + ], + "usage": { + "input_tokens": 520, + "output_tokens": 40, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 380 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [ + {"role": "user", "content": "May C-1 order 4200 EUR?"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + { + "type": "tool_use", + "id": "toolu_01", + "name": "credit_limit", + "input": {"id": "C-1"} + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": "{\"limit\":5000,\"currency\":\"EUR\"}" + } + ] + } + ], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ], + "output_config": { + "format": { + "type": "json_schema", + "schema": { + "additionalProperties": false, + "type": "object", + "properties": {"decision": {"type": "boolean"}, "reason": {"type": "string"}}, + "required": ["decision", "reason"] + } + } + } + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "done", + "data": {"decision": false, "reason": "within the limit"}, + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 2, + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 520, + "output_tokens": 40, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 380 + } + }, + "store": null + } + } + ] + }, + { + "name": "loop/retention-keeps-the-ended-run-closed", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "retention": "2d" + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": {"limit": 5000, "currency": "EUR"}}], + "retention": "2d" + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-2"}, + {"type": "text", "text": "4200 EUR is within the limit of 5000 EUR."} + ], + "usage": { + "input_tokens": 520, + "output_tokens": 40, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 380 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [ + {"role": "user", "content": "May C-1 order 4200 EUR?"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + { + "type": "tool_use", + "id": "toolu_01", + "name": "credit_limit", + "input": {"id": "C-1"} + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": "{\"limit\":5000,\"currency\":\"EUR\"}" + } + ] + } + ], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "done", + "text": "4200 EUR is within the limit of 5000 EUR.", + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 2, + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 520, + "output_tokens": 40, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 380 + } + }, + "store": {"turn": 2, "closed": true, "pending": [], "ttl_s": 172800, "messages": 4} + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 2, + "results": [], + "retention": "2d" + }, + "expect": { + "error": { + "status": 409, + "message_contains": ["conversation 0123456789abcdef0123456789abcdef has ended - start a new one"] + }, + "store": {"turn": 2, "closed": true, "pending": [], "ttl_s": 172800, "messages": 4} + } + } + ] + }, + { + "name": "loop/every-write-sets-the-time-to-live", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "ttl": "30m" + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 1800, "messages": 2} + } + }, + { + "advance_s": 600, + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": {"limit": 5000, "currency": "EUR"}}], + "ttl": "30m" + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-2"}, + { + "type": "tool_use", + "id": "toolu_02", + "name": "orders_search", + "input": {"id": "C-1"} + } + ], + "usage": { + "input_tokens": 530, + "output_tokens": 30, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [ + {"role": "user", "content": "May C-1 order 4200 EUR?"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + { + "type": "tool_use", + "id": "toolu_01", + "name": "credit_limit", + "input": {"id": "C-1"} + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": "{\"limit\":5000,\"currency\":\"EUR\"}" + } + ] + } + ], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_02", "name": "orders_search", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 2, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 530, + "output_tokens": 30, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "store": {"turn": 2, "closed": false, "pending": ["toolu_02"], "ttl_s": 1800, "messages": 4} + } + } + ] + }, + { + "name": "loop/the-configuration-supplies-ttl-and-retention", + "route": "llm.chat", + "config": { + "llm.transcript.store": "memory", + "llm.transcript.ttl": "12h", + "llm.transcript.retention": "1h" + }, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 43200, "messages": 2} + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": {"limit": 5000, "currency": "EUR"}}] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-2"}, + {"type": "text", "text": "4200 EUR is within the limit of 5000 EUR."} + ], + "usage": { + "input_tokens": 520, + "output_tokens": 40, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 380 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [ + {"role": "user", "content": "May C-1 order 4200 EUR?"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + { + "type": "tool_use", + "id": "toolu_01", + "name": "credit_limit", + "input": {"id": "C-1"} + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": "{\"limit\":5000,\"currency\":\"EUR\"}" + } + ] + } + ], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "done", + "text": "4200 EUR is within the limit of 5000 EUR.", + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 2, + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 520, + "output_tokens": 40, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 380 + } + }, + "store": {"turn": 2, "closed": true, "pending": [], "ttl_s": 3600, "messages": 4} + } + } + ] + }, + { + "name": "loop/a-stale-turn-is-a-409", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 2, + "results": [{"id": "toolu_01", "output": 5000}] + }, + "expect": { + "error": { + "status": 409, + "message_contains": [ + "conversation 0123456789abcdef0123456789abcdef holds turn 1, not 2 - a stale or duplicated turn" + ] + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + } + ] + }, + { + "name": "loop/a-duplicated-turn-never-forks-the-conversation", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": {"limit": 5000, "currency": "EUR"}}] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-2"}, + { + "type": "tool_use", + "id": "toolu_02", + "name": "orders_search", + "input": {"id": "C-1"} + } + ], + "usage": { + "input_tokens": 530, + "output_tokens": 30, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [ + {"role": "user", "content": "May C-1 order 4200 EUR?"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + { + "type": "tool_use", + "id": "toolu_01", + "name": "credit_limit", + "input": {"id": "C-1"} + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": "{\"limit\":5000,\"currency\":\"EUR\"}" + } + ] + } + ], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_02", "name": "orders_search", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 2, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 530, + "output_tokens": 30, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "store": {"turn": 2, "closed": false, "pending": ["toolu_02"], "ttl_s": 604800, "messages": 4} + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": {"limit": 5000, "currency": "EUR"}}] + }, + "expect": { + "error": { + "status": 409, + "message_contains": ["conversation 0123456789abcdef0123456789abcdef holds turn 2, not 1"] + }, + "store": {"turn": 2, "closed": false, "pending": ["toolu_02"], "ttl_s": 604800, "messages": 4} + } + } + ] + }, + { + "name": "loop/an-unknown-conversation-is-a-404", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": [], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "fedcba9876543210fedcba9876543210", + "turn": 1, + "results": [{"id": "toolu_01", "output": 5000}] + }, + "expect": { + "error": { + "status": 404, + "message_contains": ["conversation fedcba9876543210fedcba9876543210 not found - it ended or expired"] + }, + "store": null + } + } + ] + }, + { + "name": "loop/an-expired-conversation-is-a-404", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "ttl": "30m" + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 1800, "messages": 2} + } + }, + { + "advance_s": 1800, + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": 5000}], + "ttl": "30m" + }, + "expect": { + "error": { + "status": 404, + "message_contains": ["conversation 0123456789abcdef0123456789abcdef not found"] + }, + "store": null + } + } + ] + }, + { + "name": "loop/results-are-required-on-a-later-turn", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1 + }, + "expect": { + "error": { + "status": 400, + "message_contains": ["results required - the conversation waits for 1 tool result(s)"] + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + } + ] + }, + { + "name": "loop/a-result-for-an-unknown-call", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_99", "output": 5000}] + }, + "expect": { + "error": { + "status": 400, + "message_contains": ["results: unknown id 'toolu_99' - the conversation waits for: toolu_01"] + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + } + ] + }, + { + "name": "loop/a-missing-result", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}, + { + "type": "tool_use", + "id": "toolu_02", + "name": "orders_search", + "input": {"id": "C-1"} + } + ], + "usage": { + "input_tokens": 412, + "output_tokens": 80, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [ + {"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}, + {"id": "toolu_02", "name": "orders_search", "input": {"id": "C-1"}} + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 80, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "store": { + "turn": 1, + "closed": false, + "pending": ["toolu_01", "toolu_02"], + "ttl_s": 604800, + "messages": 2 + } + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": 5000}] + }, + "expect": { + "error": {"status": 400, "message_contains": ["results: missing id(s): toolu_02"]}, + "store": { + "turn": 1, + "closed": false, + "pending": ["toolu_01", "toolu_02"], + "ttl_s": 604800, + "messages": 2 + } + } + } + ] + }, + { + "name": "loop/a-duplicate-result", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}, + { + "type": "tool_use", + "id": "toolu_02", + "name": "orders_search", + "input": {"id": "C-1"} + } + ], + "usage": { + "input_tokens": 412, + "output_tokens": 80, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [ + {"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}, + {"id": "toolu_02", "name": "orders_search", "input": {"id": "C-1"}} + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 80, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "store": { + "turn": 1, + "closed": false, + "pending": ["toolu_01", "toolu_02"], + "ttl_s": 604800, + "messages": 2 + } + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": 5000}, {"id": "toolu_01", "output": 1}] + }, + "expect": { + "error": {"status": 400, "message_contains": ["results: duplicate id 'toolu_01'"]}, + "store": { + "turn": 1, + "closed": false, + "pending": ["toolu_01", "toolu_02"], + "ttl_s": 604800, + "messages": 2 + } + } + } + ] + }, + { + "name": "loop/the-model-is-pinned-at-the-first-turn", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "params": {"model": "claude-sonnet-5-5"} + }, + "provider": { + "kind": "message", + "model": "claude-sonnet-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 300, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-sonnet-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-sonnet-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 300, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": 5000}], + "params": {"model": "claude-opus-5-5"} + }, + "expect": { + "error": { + "status": 400, + "message_contains": ["conversation 0123456789abcdef0123456789abcdef is pinned to model claude-sonnet-5-5"] + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": 5000}] + }, + "provider": { + "kind": "message", + "model": "claude-sonnet-5-5", + "stop_reason": "end_turn", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-2"}, + {"type": "text", "text": "4200 EUR is within the limit of 5000 EUR."} + ], + "usage": { + "input_tokens": 300, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-sonnet-5-5", + "max_tokens": 16000, + "messages": [ + {"role": "user", "content": "May C-1 order 4200 EUR?"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + { + "type": "tool_use", + "id": "toolu_01", + "name": "credit_limit", + "input": {"id": "C-1"} + } + ] + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": "5000"}] + } + ], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "done", + "text": "4200 EUR is within the limit of 5000 EUR.", + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 2, + "model": "claude-sonnet-5-5", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 300, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "store": null + } + } + ] + }, + { + "name": "loop/a-refusal-is-a-status-not-a-422", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "refusal", + "content": [], + "usage": { + "input_tokens": 412, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + }, + "stop_details": {"category": "cyber", "explanation": "declined"} + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "refusal", + "refusal": {"category": "cyber"}, + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "refusal", + "usage": { + "input_tokens": 412, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "store": null + } + } + ] + }, + { + "name": "loop/a-refusal-with-no-category", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "retention": "1d" + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "refusal", + "content": [], + "usage": { + "input_tokens": 412, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "refusal", + "refusal": {"category": null}, + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "refusal", + "usage": { + "input_tokens": 412, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "store": {"turn": 1, "closed": true, "pending": [], "ttl_s": 86400, "messages": 2} + } + } + ] + }, + { + "name": "loop/a-turn-cut-off-at-max-tokens-runs-no-call", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "max_tokens", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 16000, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "cap", + "cap": "max_tokens", + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "max_tokens", + "usage": { + "input_tokens": 412, + "output_tokens": 16000, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "store": {"turn": 1, "closed": true, "pending": [], "ttl_s": 604800, "messages": 2} + } + } + ] + }, + { + "name": "loop/the-context-window-is-a-cap", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": {"limit": 5000, "currency": "EUR"}}] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "model_context_window_exceeded", + "content": [], + "usage": { + "input_tokens": 990000, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [ + {"role": "user", "content": "May C-1 order 4200 EUR?"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + { + "type": "tool_use", + "id": "toolu_01", + "name": "credit_limit", + "input": {"id": "C-1"} + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": "{\"limit\":5000,\"currency\":\"EUR\"}" + } + ] + } + ], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "cap", + "cap": "context", + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 2, + "model": "claude-opus-5-5", + "stop_reason": "model_context_window_exceeded", + "usage": { + "input_tokens": 990000, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "store": null + } + } + ] + }, + { + "name": "loop/a-stop-sequence-ends-the-run", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "params": {"stop_sequences": ["END"]} + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "stop_sequence", + "content": [{"type": "text", "text": "Within the limit."}], + "usage": { + "input_tokens": 412, + "output_tokens": 9, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ], + "stop_sequences": ["END"] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "done", + "text": "Within the limit.", + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "stop_sequence", + "usage": { + "input_tokens": 412, + "output_tokens": 9, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "store": null + } + } + ] + }, + { + "name": "loop/an-empty-reply-is-a-422", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "content": [{"type": "thinking", "thinking": "", "signature": "sig-1"}], + "usage": { + "input_tokens": 412, + "output_tokens": 120, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "error": { + "status": 422, + "message_contains": ["LLM reply is empty - stop_reason=end_turn, output_tokens=120"] + }, + "store": null + } + } + ] + }, + { + "name": "loop/a-verdict-that-is-not-json-is-a-422", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "schema": { + "type": "object", + "properties": {"decision": {"type": "boolean"}, "reason": {"type": "string"}}, + "required": ["decision", "reason"] + } + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "content": [{"type": "text", "text": "Within the limit."}], + "usage": { + "input_tokens": 412, + "output_tokens": 9, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ], + "output_config": { + "format": { + "type": "json_schema", + "schema": { + "additionalProperties": false, + "type": "object", + "properties": {"decision": {"type": "boolean"}, "reason": {"type": "string"}}, + "required": ["decision", "reason"] + } + } + } + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "error": { + "status": 422, + "message_contains": ["LLM reply is not valid JSON for the requested schema"] + }, + "store": null + } + } + ] + }, + { + "name": "loop/a-provider-error-leaves-the-record-as-it-was", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": {"limit": 5000, "currency": "EUR"}}] + }, + "provider": { + "kind": "status", + "status": 429, + "type": "rate_limit_error", + "message": "Number of request tokens has exceeded your rate limit", + "request_id": "req_loop_429" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [ + {"role": "user", "content": "May C-1 order 4200 EUR?"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + { + "type": "tool_use", + "id": "toolu_01", + "name": "credit_limit", + "input": {"id": "C-1"} + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": "{\"limit\":5000,\"currency\":\"EUR\"}" + } + ] + } + ], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "error": {"status": 429, "message_contains": ["LLM provider rate limit - 429 rate_limit_error"]}, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": {"limit": 5000, "currency": "EUR"}}] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-2"}, + {"type": "text", "text": "4200 EUR is within the limit of 5000 EUR."} + ], + "usage": { + "input_tokens": 520, + "output_tokens": 40, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 380 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [ + {"role": "user", "content": "May C-1 order 4200 EUR?"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + { + "type": "tool_use", + "id": "toolu_01", + "name": "credit_limit", + "input": {"id": "C-1"} + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": "{\"limit\":5000,\"currency\":\"EUR\"}" + } + ] + } + ], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "done", + "text": "4200 EUR is within the limit of 5000 EUR.", + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 2, + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 520, + "output_tokens": 40, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 380 + } + }, + "store": null + } + } + ] + }, + { + "name": "loop/a-first-turn-that-fails-creates-no-record", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": {"kind": "connection"}, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "error": {"status": 503, "message_contains": ["LLM provider unreachable"]}, + "store": null + } + } + ] + }, + { + "name": "loop/pause-turn-is-not-part-of-the-contract", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "pause_turn", + "content": [{"type": "thinking", "thinking": "", "signature": "sig-1"}], + "usage": { + "input_tokens": 412, + "output_tokens": 10, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "error": { + "status": 502, + "message_contains": ["LLM stop_reason=pause_turn is not part of the loop contract"] + }, + "store": null + } + } + ] + }, + { + "name": "loop/tool-use-without-a-call-is-a-502", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [{"type": "text", "text": "I will look it up."}], + "usage": { + "input_tokens": 412, + "output_tokens": 10, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "error": {"status": 502, "message_contains": ["LLM stop_reason=tool_use carried no tool call"]}, + "store": null + } + } + ] + }, + { + "name": "loop/messages-without-system-are-sent-as-written", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "messages": [ + {"role": "user", "content": "Customer C-1 wants to order."}, + {"role": "assistant", "content": "For how much?"}, + {"role": "user", "content": "4200 EUR."} + ], + "tools": [ + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + { + "type": "tool_use", + "id": "toolu_01", + "name": "orders_search", + "input": {"id": "C-1"} + } + ], + "usage": { + "input_tokens": 200, + "output_tokens": 20, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [ + {"role": "user", "content": "Customer C-1 wants to order."}, + {"role": "assistant", "content": "For how much?"}, + {"role": "user", "content": "4200 EUR."} + ], + "tools": [ + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"} + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "orders_search", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 200, + "output_tokens": 20, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 4} + } + } + ] + }, + { + "name": "loop/strict-false-is-not-sent", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": false + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "content": [{"type": "text", "text": "No."}], + "usage": { + "input_tokens": 100, + "output_tokens": 2, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + } + } + ], + "cache_control": {"type": "ephemeral"} + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "done", + "text": "No.", + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 100, + "output_tokens": 2, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + "store": null + } + } + ] + }, + { + "name": "loop/params-ride-every-turn", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "params": {"effort": "low", "max_tokens": 4000, "timeout_ms": 20000} + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 4000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ], + "output_config": {"effort": "low"} + }, + "options": {"timeout_ms": 20000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + } + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "results": [{"id": "toolu_01", "output": {"limit": 5000, "currency": "EUR"}}], + "params": {"effort": "low", "max_tokens": 4000, "timeout_ms": 20000} + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-2"}, + {"type": "text", "text": "4200 EUR is within the limit of 5000 EUR."} + ], + "usage": { + "input_tokens": 520, + "output_tokens": 40, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 380 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 4000, + "messages": [ + {"role": "user", "content": "May C-1 order 4200 EUR?"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + { + "type": "tool_use", + "id": "toolu_01", + "name": "credit_limit", + "input": {"id": "C-1"} + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": "{\"limit\":5000,\"currency\":\"EUR\"}" + } + ] + } + ], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ], + "output_config": {"effort": "low"} + }, + "options": {"timeout_ms": 20000, "max_retries": 2} + }, + "result": { + "status": "done", + "text": "4200 EUR is within the limit of 5000 EUR.", + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 2, + "model": "claude-opus-5-5", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 520, + "output_tokens": 40, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 380 + } + }, + "store": null + } + } + ] + }, + { + "name": "loop/bedrock-spells-the-model-its-way", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory", "llm.backend": "bedrock"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "anthropic.claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + } + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + } + ] + }, + { + "name": "loop/two-runs-have-two-conversations", + "route": "llm.chat", + "config": {"llm.transcript.store": "memory"}, + "conversation_ids": ["0123456789abcdef0123456789abcdef", "fedcba9876543210fedcba9876543210"], + "steps": [ + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "0123456789abcdef0123456789abcdef", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + }, + "request_id": "req_loop_001" + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + } + }, + { + "body": { + "system": "Check orders against the credit limit. Use the tools.", + "prompt": "May C-1 order 4200 EUR?", + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ] + }, + "provider": { + "kind": "message", + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig-1"}, + {"type": "text", "text": "Checking the limit."}, + {"type": "tool_use", "id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}} + ], + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + } + }, + "expect": { + "sdk": { + "namespace": "messages", + "params": { + "model": "claude-opus-5-5", + "max_tokens": 16000, + "messages": [{"role": "user", "content": "May C-1 order 4200 EUR?"}], + "tools": [ + { + "name": "credit_limit", + "description": "A customer's credit limit.", + "input_schema": { + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + "additionalProperties": false + }, + "strict": true + }, + { + "name": "orders_search", + "description": "A customer's open orders.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}} + } + ], + "cache_control": {"type": "ephemeral"}, + "system": [ + { + "type": "text", + "text": "Check orders against the credit limit. Use the tools.", + "cache_control": {"type": "ephemeral"} + } + ] + }, + "options": {"timeout_ms": 60000, "max_retries": 2} + }, + "result": { + "status": "tool_use", + "calls": [{"id": "toolu_01", "name": "credit_limit", "input": {"id": "C-1"}}], + "conversation": "fedcba9876543210fedcba9876543210", + "turn": 1, + "model": "claude-opus-5-5", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 412, + "output_tokens": 61, + "cache_creation_input_tokens": 380, + "cache_read_input_tokens": 0 + } + }, + "store": {"turn": 1, "closed": false, "pending": ["toolu_01"], "ttl_s": 604800, "messages": 2} + }, + "conversation": "fedcba9876543210fedcba9876543210" + } + ] + } + ] +}