diff --git a/CHANGELOG.md b/CHANGELOG.md index 038bcf5..8844790 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -149,6 +149,15 @@ Notable changes to Cloudfall. The format follows ### Changed +- The raw trace of `agent investigate` keeps at most 8000 characters of a + turn's reasoning and of its text, like the host output a tool result hands + the model: a longer one keeps its end after a `[first N characters cut]` + mark, in the investigation record (`spec.trace`) and in the stream's `turn` + event alike. A model's reasoning often quotes the host output it just + read, and the record no longer keeps it unbounded. + `agent-investigation.schema.json` bounds both fields with `maxLength`, and + says that a step's parsed arguments are what ran while + `toolCalls[].arguments` is only the model's text (#50) - treaty 1.0.0rc38 instead of 1.0.0rc29. `dashboard serve` and `operator run` write each event in `json` and `jsonl` as a bare JSON line with `_seq` instead of an envelope with `data` and `meta.seq`, and end diff --git a/config/schemas/v1/agent-investigation.schema.json b/config/schemas/v1/agent-investigation.schema.json index f691a7b..3d4c204 100644 --- a/config/schemas/v1/agent-investigation.schema.json +++ b/config/schemas/v1/agent-investigation.schema.json @@ -123,15 +123,26 @@ "type": "object", "properties": { "turn": {"type": "integer", "minimum": 1}, - "text": {"type": "string"}, - "reasoning": {"type": "string"}, + "text": { + "type": "string", + "maxLength": 8000, + "description": "The turn's words; a longer text keeps its end, after a mark naming how many characters were cut." + }, + "reasoning": { + "type": "string", + "maxLength": 8000, + "description": "The model's reasoning, as the endpoint returned it; a longer reasoning keeps its end, after a mark naming how many characters were cut." + }, "toolCalls": { "type": "array", "items": { "type": "object", "properties": { "name": {"type": "string"}, - "arguments": {"type": "string"} + "arguments": { + "type": "string", + "description": "The arguments exactly as the model wrote them. The step's parsed arguments are what ran; this raw string is only the model's text." + } }, "required": ["name", "arguments"], "additionalProperties": false diff --git a/sdk/src/cloudfall/investigation.py b/sdk/src/cloudfall/investigation.py index 5edb975..f8e2ac0 100644 --- a/sdk/src/cloudfall/investigation.py +++ b/sdk/src/cloudfall/investigation.py @@ -38,6 +38,8 @@ DEFAULT_MAX_TURNS = 12 TOOL_TEXT_LIMIT = 8000 """Most characters of host output one tool result hands the model.""" +TRACE_TEXT_LIMIT = TOOL_TEXT_LIMIT +"""Most characters of a turn's text or reasoning the trace keeps, cut mark included.""" TOOL_NAME_LIMIT = 128 """Most characters of a tool name the record keeps, as its schema allows.""" REPLY_ID_LIMIT = 256 @@ -534,10 +536,25 @@ def host_text(text: str) -> str: """Keep the end of a long host output, where a play reports, and say so.""" if len(text) <= TOOL_TEXT_LIMIT: return text - return ( - f"[first {len(text) - TOOL_TEXT_LIMIT} characters cut]\n" - + text[-TOOL_TEXT_LIMIT:] - ) + return _cut_mark(len(text) - TOOL_TEXT_LIMIT) + text[-TOOL_TEXT_LIMIT:] + + +def _trace_text(text: str) -> str: + """Keep the end of a long turn text or reasoning, where it concludes, and say so. + + The model's reasoning often quotes the host output it just read, so the + trace bounds it as the tool result does. The mark counts against the + limit: what is kept fits the schema's ``maxLength``. + """ + if len(text) <= TRACE_TEXT_LIMIT: + return text + # The mark for the whole length is at least as long as the one written. + kept = TRACE_TEXT_LIMIT - len(_cut_mark(len(text))) + return _cut_mark(len(text) - kept) + text[-kept:] + + +def _cut_mark(cut: int) -> str: + return f"[first {cut} characters cut]\n" def _tool_definition(tool: AgentTool) -> dict[str, object]: @@ -565,11 +582,12 @@ def _turn( details = details if isinstance(details, Mapping) else {} reasoning = message.get("reasoning_content") or message.get("reasoning") or "" response_id = reply.body.get("id") - # The ids are the endpoint's text; the record keeps what its schema takes. + # The ids, the text and the reasoning are the endpoint's text; the record + # keeps what its schema takes. return Turn( turn=number, - text=str(message.get("content") or "").strip(), - reasoning=str(reasoning).strip(), + text=_trace_text(str(message.get("content") or "").strip()), + reasoning=_trace_text(str(reasoning).strip()), tool_calls=tuple(_call_text(call) for call in calls), response_id=( response_id[:REPLY_ID_LIMIT] if isinstance(response_id, str) else None diff --git a/sdk/tests/test_investigation.py b/sdk/tests/test_investigation.py index e0b4786..fad60dd 100644 --- a/sdk/tests/test_investigation.py +++ b/sdk/tests/test_investigation.py @@ -12,6 +12,7 @@ from cloudfall.cli import main from cloudfall.investigation import ( ERROR_MODEL_REFUSED, + TRACE_TEXT_LIMIT, AgentTool, InvestigationError, InvestigationStatus, @@ -25,6 +26,7 @@ ) from cloudfall.resources import default_schema_directory from cloudfall.validation import SchemaCatalog +from jsonschema import ValidationError from test_catalog import _repository if TYPE_CHECKING: @@ -870,6 +872,58 @@ def test_each_turn_keeps_the_raw_trace() -> None: assert recorded["trace"][1]["toolCalls"] == [] +def test_a_long_reasoning_is_cut_to_the_limit_and_marked(tmp_path: Path) -> None: + report, rotate = _tools() + # A reasoning quoting a long host output, past the limit, ending in its plan. + reasoning = "журнал " * 1500 + "The disk is full; read it first." + long_text = "x" * 9000 + " rotate the logs." + thinking = _call("disk-report", {}) + first = thinking["choices"] + assert isinstance(first, list) + first[0]["message"]["reasoning_content"] = reasoning + script = _Script(thinking, _say(long_text)) + + done = investigate("alert", _tool_list(report, rotate), ENDPOINT, script) + saved = InvestigationStore( + directory=tmp_path / "investigations", + catalog=SchemaCatalog(default_schema_directory()), + ).save(done) + + kept, said = saved.trace[0].reasoning, saved.trace[1].text + # Characters, not bytes: the Cyrillic reasoning is counted the same. The + # mark counts against the limit; a shorter count can leave it one under. + assert TRACE_TEXT_LIMIT == 8000 + assert TRACE_TEXT_LIMIT - 1 <= len(kept) <= TRACE_TEXT_LIMIT + assert len(said) == TRACE_TEXT_LIMIT + mark, tail = kept.split("\n", 1) + assert mark == f"[first {len(reasoning) - len(tail)} characters cut]" + assert reasoning.endswith(tail) + assert tail.endswith("The disk is full; read it first.") + assert said.startswith("[first 1045 characters cut]\n") + assert said.endswith(" rotate the logs.") + # The answer the finding is read from is the model's whole text. + assert saved.answer == long_text + + +def test_the_schema_refuses_a_trace_longer_than_the_limit() -> None: + report, rotate = _tools() + done = investigate( + "alert", + _tool_list(report, rotate), + ENDPOINT, + _Script(_say("ROOT CAUSE: disk\nPROPOSED: none\nWHY: read only.")), + ) + document = done.as_document() + spec = document["spec"] + assert isinstance(spec, dict) + spec["trace"][0]["reasoning"] = "x" * (TRACE_TEXT_LIMIT + 1) + + with pytest.raises(ValidationError): + SchemaCatalog(default_schema_directory()).validate_named( + "agent-investigation.schema.json", document + ) + + class _LongIds(_Script): """An endpoint that serves its answers under over-long ids."""