From 29797eddf90b277f209c03bf5e4f7cf021b3561c Mon Sep 17 00:00:00 2001 From: "shipmill-romamo[bot]" <338557544+shipmill-romamo[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:20:08 +0300 Subject: [PATCH] Cap the reasoning and text the investigation trace keeps A model's reasoning often quotes the host output it just read, and since PR #45 the trace kept it verbatim with no bound. Each turn's reasoning and text now keep at most 8000 characters (TOOL_TEXT_LIMIT), their end after a [first N characters cut] mark that counts against the limit, and the schema bounds both with maxLength. The stream's turn event carries the same Turn, so it is capped alike. The schema also says the step's parsed arguments are what ran, and toolCalls[].arguments only the model's text. Fixes #50 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 ++++ .../v1/agent-investigation.schema.json | 17 ++++-- sdk/src/cloudfall/investigation.py | 32 ++++++++--- sdk/tests/test_investigation.py | 54 +++++++++++++++++++ 4 files changed, 102 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 038bcf5..8844790 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -149,6 +149,15 @@ Notable changes to Cloudfall. The format follows ### Changed +- The raw trace of `agent investigate` keeps at most 8000 characters of a + turn's reasoning and of its text, like the host output a tool result hands + the model: a longer one keeps its end after a `[first N characters cut]` + mark, in the investigation record (`spec.trace`) and in the stream's `turn` + event alike. A model's reasoning often quotes the host output it just + read, and the record no longer keeps it unbounded. + `agent-investigation.schema.json` bounds both fields with `maxLength`, and + says that a step's parsed arguments are what ran while + `toolCalls[].arguments` is only the model's text (#50) - treaty 1.0.0rc38 instead of 1.0.0rc29. `dashboard serve` and `operator run` write each event in `json` and `jsonl` as a bare JSON line with `_seq` instead of an envelope with `data` and `meta.seq`, and end diff --git a/config/schemas/v1/agent-investigation.schema.json b/config/schemas/v1/agent-investigation.schema.json index f691a7b..3d4c204 100644 --- a/config/schemas/v1/agent-investigation.schema.json +++ b/config/schemas/v1/agent-investigation.schema.json @@ -123,15 +123,26 @@ "type": "object", "properties": { "turn": {"type": "integer", "minimum": 1}, - "text": {"type": "string"}, - "reasoning": {"type": "string"}, + "text": { + "type": "string", + "maxLength": 8000, + "description": "The turn's words; a longer text keeps its end, after a mark naming how many characters were cut." + }, + "reasoning": { + "type": "string", + "maxLength": 8000, + "description": "The model's reasoning, as the endpoint returned it; a longer reasoning keeps its end, after a mark naming how many characters were cut." + }, "toolCalls": { "type": "array", "items": { "type": "object", "properties": { "name": {"type": "string"}, - "arguments": {"type": "string"} + "arguments": { + "type": "string", + "description": "The arguments exactly as the model wrote them. The step's parsed arguments are what ran; this raw string is only the model's text." + } }, "required": ["name", "arguments"], "additionalProperties": false diff --git a/sdk/src/cloudfall/investigation.py b/sdk/src/cloudfall/investigation.py index 5edb975..f8e2ac0 100644 --- a/sdk/src/cloudfall/investigation.py +++ b/sdk/src/cloudfall/investigation.py @@ -38,6 +38,8 @@ DEFAULT_MAX_TURNS = 12 TOOL_TEXT_LIMIT = 8000 """Most characters of host output one tool result hands the model.""" +TRACE_TEXT_LIMIT = TOOL_TEXT_LIMIT +"""Most characters of a turn's text or reasoning the trace keeps, cut mark included.""" TOOL_NAME_LIMIT = 128 """Most characters of a tool name the record keeps, as its schema allows.""" REPLY_ID_LIMIT = 256 @@ -534,10 +536,25 @@ def host_text(text: str) -> str: """Keep the end of a long host output, where a play reports, and say so.""" if len(text) <= TOOL_TEXT_LIMIT: return text - return ( - f"[first {len(text) - TOOL_TEXT_LIMIT} characters cut]\n" - + text[-TOOL_TEXT_LIMIT:] - ) + return _cut_mark(len(text) - TOOL_TEXT_LIMIT) + text[-TOOL_TEXT_LIMIT:] + + +def _trace_text(text: str) -> str: + """Keep the end of a long turn text or reasoning, where it concludes, and say so. + + The model's reasoning often quotes the host output it just read, so the + trace bounds it as the tool result does. The mark counts against the + limit: what is kept fits the schema's ``maxLength``. + """ + if len(text) <= TRACE_TEXT_LIMIT: + return text + # The mark for the whole length is at least as long as the one written. + kept = TRACE_TEXT_LIMIT - len(_cut_mark(len(text))) + return _cut_mark(len(text) - kept) + text[-kept:] + + +def _cut_mark(cut: int) -> str: + return f"[first {cut} characters cut]\n" def _tool_definition(tool: AgentTool) -> dict[str, object]: @@ -565,11 +582,12 @@ def _turn( details = details if isinstance(details, Mapping) else {} reasoning = message.get("reasoning_content") or message.get("reasoning") or "" response_id = reply.body.get("id") - # The ids are the endpoint's text; the record keeps what its schema takes. + # The ids, the text and the reasoning are the endpoint's text; the record + # keeps what its schema takes. return Turn( turn=number, - text=str(message.get("content") or "").strip(), - reasoning=str(reasoning).strip(), + text=_trace_text(str(message.get("content") or "").strip()), + reasoning=_trace_text(str(reasoning).strip()), tool_calls=tuple(_call_text(call) for call in calls), response_id=( response_id[:REPLY_ID_LIMIT] if isinstance(response_id, str) else None diff --git a/sdk/tests/test_investigation.py b/sdk/tests/test_investigation.py index e0b4786..fad60dd 100644 --- a/sdk/tests/test_investigation.py +++ b/sdk/tests/test_investigation.py @@ -12,6 +12,7 @@ from cloudfall.cli import main from cloudfall.investigation import ( ERROR_MODEL_REFUSED, + TRACE_TEXT_LIMIT, AgentTool, InvestigationError, InvestigationStatus, @@ -25,6 +26,7 @@ ) from cloudfall.resources import default_schema_directory from cloudfall.validation import SchemaCatalog +from jsonschema import ValidationError from test_catalog import _repository if TYPE_CHECKING: @@ -870,6 +872,58 @@ def test_each_turn_keeps_the_raw_trace() -> None: assert recorded["trace"][1]["toolCalls"] == [] +def test_a_long_reasoning_is_cut_to_the_limit_and_marked(tmp_path: Path) -> None: + report, rotate = _tools() + # A reasoning quoting a long host output, past the limit, ending in its plan. + reasoning = "журнал " * 1500 + "The disk is full; read it first." + long_text = "x" * 9000 + " rotate the logs." + thinking = _call("disk-report", {}) + first = thinking["choices"] + assert isinstance(first, list) + first[0]["message"]["reasoning_content"] = reasoning + script = _Script(thinking, _say(long_text)) + + done = investigate("alert", _tool_list(report, rotate), ENDPOINT, script) + saved = InvestigationStore( + directory=tmp_path / "investigations", + catalog=SchemaCatalog(default_schema_directory()), + ).save(done) + + kept, said = saved.trace[0].reasoning, saved.trace[1].text + # Characters, not bytes: the Cyrillic reasoning is counted the same. The + # mark counts against the limit; a shorter count can leave it one under. + assert TRACE_TEXT_LIMIT == 8000 + assert TRACE_TEXT_LIMIT - 1 <= len(kept) <= TRACE_TEXT_LIMIT + assert len(said) == TRACE_TEXT_LIMIT + mark, tail = kept.split("\n", 1) + assert mark == f"[first {len(reasoning) - len(tail)} characters cut]" + assert reasoning.endswith(tail) + assert tail.endswith("The disk is full; read it first.") + assert said.startswith("[first 1045 characters cut]\n") + assert said.endswith(" rotate the logs.") + # The answer the finding is read from is the model's whole text. + assert saved.answer == long_text + + +def test_the_schema_refuses_a_trace_longer_than_the_limit() -> None: + report, rotate = _tools() + done = investigate( + "alert", + _tool_list(report, rotate), + ENDPOINT, + _Script(_say("ROOT CAUSE: disk\nPROPOSED: none\nWHY: read only.")), + ) + document = done.as_document() + spec = document["spec"] + assert isinstance(spec, dict) + spec["trace"][0]["reasoning"] = "x" * (TRACE_TEXT_LIMIT + 1) + + with pytest.raises(ValidationError): + SchemaCatalog(default_schema_directory()).validate_named( + "agent-investigation.schema.json", document + ) + + class _LongIds(_Script): """An endpoint that serves its answers under over-long ids."""