Cap the reasoning and text the investigation trace keeps

CHANGELOG.md

@@ -149,6 +149,15 @@ Notable changes to Cloudfall. The format follows
 
 ### Changed
 
+- The raw trace of `agent investigate` keeps at most 8000 characters of a
+  turn's reasoning and of its text, like the host output a tool result hands
+  the model: a longer one keeps its end after a `[first N characters cut]`
+  mark, in the investigation record (`spec.trace`) and in the stream's `turn`
+  event alike. A model's reasoning often quotes the host output it just
+  read, and the record no longer keeps it unbounded.
+  `agent-investigation.schema.json` bounds both fields with `maxLength`, and
+  says that a step's parsed arguments are what ran while
+  `toolCalls[].arguments` is only the model's text (#50)
 - treaty 1.0.0rc38 instead of 1.0.0rc29. `dashboard serve` and
   `operator run` write each event in `json` and `jsonl` as a bare JSON line
   with `_seq` instead of an envelope with `data` and `meta.seq`, and end

config/schemas/v1/agent-investigation.schema.json

@@ -123,15 +123,26 @@
             "type": "object",
             "properties": {
               "turn": {"type": "integer", "minimum": 1},
-              "text": {"type": "string"},
-              "reasoning": {"type": "string"},
+              "text": {
+                "type": "string",
+                "maxLength": 8000,
+                "description": "The turn's words; a longer text keeps its end, after a mark naming how many characters were cut."
+              },
+              "reasoning": {
+                "type": "string",
+                "maxLength": 8000,
+                "description": "The model's reasoning, as the endpoint returned it; a longer reasoning keeps its end, after a mark naming how many characters were cut."
+              },
               "toolCalls": {
                 "type": "array",
                 "items": {
                   "type": "object",
                   "properties": {
                     "name": {"type": "string"},
-                    "arguments": {"type": "string"}
+                    "arguments": {
+                      "type": "string",
+                      "description": "The arguments exactly as the model wrote them. The step's parsed arguments are what ran; this raw string is only the model's text."
+                    }
                   },
                   "required": ["name", "arguments"],
                   "additionalProperties": false

sdk/src/cloudfall/investigation.py

@@ -38,6 +38,8 @@
 DEFAULT_MAX_TURNS = 12
 TOOL_TEXT_LIMIT = 8000
 """Most characters of host output one tool result hands the model."""
+TRACE_TEXT_LIMIT = TOOL_TEXT_LIMIT
+"""Most characters of a turn's text or reasoning the trace keeps, cut mark included."""
 TOOL_NAME_LIMIT = 128
 """Most characters of a tool name the record keeps, as its schema allows."""
 REPLY_ID_LIMIT = 256
@@ -534,10 +536,25 @@ def host_text(text: str) -> str:
     """Keep the end of a long host output, where a play reports, and say so."""
     if len(text) <= TOOL_TEXT_LIMIT:
         return text
-    return (
-        f"[first {len(text) - TOOL_TEXT_LIMIT} characters cut]\n"
-        + text[-TOOL_TEXT_LIMIT:]
-    )
+    return _cut_mark(len(text) - TOOL_TEXT_LIMIT) + text[-TOOL_TEXT_LIMIT:]
+
+
+def _trace_text(text: str) -> str:
+    """Keep the end of a long turn text or reasoning, where it concludes, and say so.
+
+    The model's reasoning often quotes the host output it just read, so the
+    trace bounds it as the tool result does. The mark counts against the
+    limit: what is kept fits the schema's ``maxLength``.
+    """
+    if len(text) <= TRACE_TEXT_LIMIT:
+        return text
+    # The mark for the whole length is at least as long as the one written.
+    kept = TRACE_TEXT_LIMIT - len(_cut_mark(len(text)))
+    return _cut_mark(len(text) - kept) + text[-kept:]
+
+
+def _cut_mark(cut: int) -> str:
+    return f"[first {cut} characters cut]\n"
 
 
 def _tool_definition(tool: AgentTool) -> dict[str, object]:
@@ -565,11 +582,12 @@ def _turn(
     details = details if isinstance(details, Mapping) else {}
     reasoning = message.get("reasoning_content") or message.get("reasoning") or ""
     response_id = reply.body.get("id")
-    # The ids are the endpoint's text; the record keeps what its schema takes.
+    # The ids, the text and the reasoning are the endpoint's text; the record
+    # keeps what its schema takes.
     return Turn(
         turn=number,
-        text=str(message.get("content") or "").strip(),
-        reasoning=str(reasoning).strip(),
+        text=_trace_text(str(message.get("content") or "").strip()),
+        reasoning=_trace_text(str(reasoning).strip()),
         tool_calls=tuple(_call_text(call) for call in calls),
         response_id=(
             response_id[:REPLY_ID_LIMIT] if isinstance(response_id, str) else None

sdk/tests/test_investigation.py

@@ -12,6 +12,7 @@
 from cloudfall.cli import main
 from cloudfall.investigation import (
     ERROR_MODEL_REFUSED,
+    TRACE_TEXT_LIMIT,
     AgentTool,
     InvestigationError,
     InvestigationStatus,
@@ -25,6 +26,7 @@
 )
 from cloudfall.resources import default_schema_directory
 from cloudfall.validation import SchemaCatalog
+from jsonschema import ValidationError
 from test_catalog import _repository
 
 if TYPE_CHECKING:
@@ -870,6 +872,58 @@ def test_each_turn_keeps_the_raw_trace() -> None:
     assert recorded["trace"][1]["toolCalls"] == []
 
 
+def test_a_long_reasoning_is_cut_to_the_limit_and_marked(tmp_path: Path) -> None:
+    report, rotate = _tools()
+    # A reasoning quoting a long host output, past the limit, ending in its plan.
+    reasoning = "журнал " * 1500 + "The disk is full; read it first."
+    long_text = "x" * 9000 + " rotate the logs."
+    thinking = _call("disk-report", {})
+    first = thinking["choices"]
+    assert isinstance(first, list)
+    first[0]["message"]["reasoning_content"] = reasoning
+    script = _Script(thinking, _say(long_text))
+
+    done = investigate("alert", _tool_list(report, rotate), ENDPOINT, script)
+    saved = InvestigationStore(
+        directory=tmp_path / "investigations",
+        catalog=SchemaCatalog(default_schema_directory()),
+    ).save(done)
+
+    kept, said = saved.trace[0].reasoning, saved.trace[1].text
+    # Characters, not bytes: the Cyrillic reasoning is counted the same. The
+    # mark counts against the limit; a shorter count can leave it one under.
+    assert TRACE_TEXT_LIMIT == 8000
+    assert TRACE_TEXT_LIMIT - 1 <= len(kept) <= TRACE_TEXT_LIMIT
+    assert len(said) == TRACE_TEXT_LIMIT
+    mark, tail = kept.split("\n", 1)
+    assert mark == f"[first {len(reasoning) - len(tail)} characters cut]"
+    assert reasoning.endswith(tail)
+    assert tail.endswith("The disk is full; read it first.")
+    assert said.startswith("[first 1045 characters cut]\n")
+    assert said.endswith(" rotate the logs.")
+    # The answer the finding is read from is the model's whole text.
+    assert saved.answer == long_text
+
+
+def test_the_schema_refuses_a_trace_longer_than_the_limit() -> None:
+    report, rotate = _tools()
+    done = investigate(
+        "alert",
+        _tool_list(report, rotate),
+        ENDPOINT,
+        _Script(_say("ROOT CAUSE: disk\nPROPOSED: none\nWHY: read only.")),
+    )
+    document = done.as_document()
+    spec = document["spec"]
+    assert isinstance(spec, dict)
+    spec["trace"][0]["reasoning"] = "x" * (TRACE_TEXT_LIMIT + 1)
+
+    with pytest.raises(ValidationError):
+        SchemaCatalog(default_schema_directory()).validate_named(
+            "agent-investigation.schema.json", document
+        )
+
+
 class _LongIds(_Script):
     """An endpoint that serves its answers under over-long ids."""