From 4524679d6d004b1644ff54be086feed3b424e703 Mon Sep 17 00:00:00 2001 From: "shipmill-romamo[bot]" <338557544+shipmill-romamo[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:13:11 +0300 Subject: [PATCH] Cap the investigation's answer and raw tool-call arguments at 8000 characters spec.answer and trace[].toolCalls[].arguments are the model's text and can quote host output, yet the record kept both unbounded after #52 capped the trace's reasoning and text (D-4). Both now keep at most 8000 characters, cut mark included, keeping the start and ending with a [last N characters cut] mark (D-6). Only the stored copy is cut: the finding is read from the whole answer and the tool runs on the whole arguments. The schema bounds both fields with maxLength 8000. Fixes #57 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 + .../v1/agent-investigation.schema.json | 7 +- sdk/src/cloudfall/investigation.py | 30 +++- sdk/tests/test_investigation.py | 152 +++++++++++++++++- 4 files changed, 191 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40e3f8c..6a61708 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -183,6 +183,14 @@ Notable changes to Cloudfall. The format follows `agent-investigation.schema.json` bounds both fields with `maxLength`, and says that a step's parsed arguments are what ran while `toolCalls[].arguments` is only the model's text (#50) +- The investigation record of `agent investigate` keeps at most 8000 + characters of the model's answer (`spec.answer`) and of each tool call's + raw arguments (`spec.trace[].toolCalls[].arguments`). Unlike the trace's + reasoning, a longer one keeps its start, followed by a + `[last N characters cut]` mark, in the record and in the stream's events + alike. The finding is still read from the whole answer, and the tool + still runs on the whole arguments. `agent-investigation.schema.json` + bounds both fields with `maxLength` (#57) - treaty 1.0.0rc38 instead of 1.0.0rc29. `dashboard serve` and `operator run` write each event in `json` and `jsonl` as a bare JSON line with `_seq` instead of an envelope with `data` and `meta.seq`, and end diff --git a/config/schemas/v1/agent-investigation.schema.json b/config/schemas/v1/agent-investigation.schema.json index 887f383..9c65a55 100644 --- a/config/schemas/v1/agent-investigation.schema.json +++ b/config/schemas/v1/agent-investigation.schema.json @@ -116,7 +116,9 @@ } }, "answer": { - "type": "string" + "type": "string", + "maxLength": 8000, + "description": "The model's final words, which the finding is read from; a longer answer keeps its start, before a mark naming how many characters were cut. The finding is read from the whole answer." }, "trace": { "type": "array", @@ -142,7 +144,8 @@ "name": {"type": "string"}, "arguments": { "type": "string", - "description": "The arguments exactly as the model wrote them. The step's parsed arguments are what ran; this raw string is only the model's text." + "maxLength": 8000, + "description": "The arguments exactly as the model wrote them; longer ones keep their start, before a mark naming how many characters were cut. The step's parsed arguments are what ran; this raw string is only the model's text." } }, "required": ["name", "arguments"], diff --git a/sdk/src/cloudfall/investigation.py b/sdk/src/cloudfall/investigation.py index 296091d..6732196 100644 --- a/sdk/src/cloudfall/investigation.py +++ b/sdk/src/cloudfall/investigation.py @@ -40,6 +40,10 @@ """Most characters of host output one tool result hands the model.""" TRACE_TEXT_LIMIT = TOOL_TEXT_LIMIT """Most characters of a turn's text or reasoning the trace keeps, cut mark included.""" +ANSWER_LIMIT = TOOL_TEXT_LIMIT +"""Most characters of the model's answer the record keeps, cut mark included.""" +ARGUMENTS_LIMIT = TOOL_TEXT_LIMIT +"""Most characters of a tool call's raw arguments the trace keeps, cut mark included.""" TOOL_NAME_LIMIT = 128 """Most characters of a tool name the record keeps, as its schema allows.""" REPLY_ID_LIMIT = 256 @@ -497,7 +501,8 @@ def record(ending: InvestigationStatus, finding: Finding | None) -> Investigatio turns=turns, usage=usage, steps=tuple(steps), - answer=answer, + # The finding is read from the whole answer; the record keeps its start. + answer=_head_text(answer, ANSWER_LIMIT), finding=finding, replay=replay, trace=tuple(trace), @@ -579,10 +584,29 @@ def _trace_text(text: str) -> str: return _cut_mark(len(text) - kept) + text[-kept:] +def _head_text(text: str, limit: int) -> str: + """Keep the start of a long answer or raw arguments, where they begin, and say so. + + Both are the model's text and can quote host output, so the record + bounds them as it bounds the trace's reasoning, but keeps the start: an + answer opens with its finding. The mark counts against the limit: what + is kept fits the schema's ``maxLength``. + """ + if len(text) <= limit: + return text + # The mark for the whole length is at least as long as the one written. + kept = limit - len(_tail_mark(len(text))) + return text[:kept] + _tail_mark(len(text) - kept) + + def _cut_mark(cut: int) -> str: return f"[first {cut} characters cut]\n" +def _tail_mark(cut: int) -> str: + return f"\n[last {cut} characters cut]" + + def _tool_definition(tool: AgentTool) -> dict[str, object]: return { "type": "function", @@ -632,9 +656,11 @@ def _call_text(call: Mapping[str, object]) -> ToolCallText: function = call.get("function") function = function if isinstance(function, Mapping) else {} arguments = function.get("arguments", "") + text = arguments if isinstance(arguments, str) else json.dumps(arguments) + # Only the trace's copy is cut: the call itself runs on the whole arguments. return ToolCallText( name=str(function.get("name", "")), - arguments=arguments if isinstance(arguments, str) else json.dumps(arguments), + arguments=_head_text(text, ARGUMENTS_LIMIT), ) diff --git a/sdk/tests/test_investigation.py b/sdk/tests/test_investigation.py index c39b520..5e3cf4e 100644 --- a/sdk/tests/test_investigation.py +++ b/sdk/tests/test_investigation.py @@ -15,6 +15,8 @@ from cloudfall.app import app from cloudfall.cli import main from cloudfall.investigation import ( + ANSWER_LIMIT, + ARGUMENTS_LIMIT, ERROR_INVESTIGATIONS_UNWRITABLE, ERROR_MODEL_REFUSED, TRACE_TEXT_LIMIT, @@ -1096,8 +1098,10 @@ def test_a_long_reasoning_is_cut_to_the_limit_and_marked(tmp_path: Path) -> None assert tail.endswith("The disk is full; read it first.") assert said.startswith("[first 1045 characters cut]\n") assert said.endswith(" rotate the logs.") - # The answer the finding is read from is the model's whole text. - assert saved.answer == long_text + # The record's answer keeps the start of the same text (#57). + assert saved.answer.startswith("x" * 7000) + assert saved.answer.endswith("\n[last 1044 characters cut]") + assert len(saved.answer) == ANSWER_LIMIT def test_the_schema_refuses_a_trace_longer_than_the_limit() -> None: @@ -1119,6 +1123,150 @@ def test_the_schema_refuses_a_trace_longer_than_the_limit() -> None: ) +_FINDING = "ROOT CAUSE: disk\nPROPOSED: none\nWHY: read only.\n" + + +@pytest.mark.parametrize("length", [ANSWER_LIMIT, ANSWER_LIMIT + 1, 50_000]) +def test_a_long_answer_keeps_its_start_and_its_finding( + tmp_path: Path, length: int +) -> None: + report, rotate = _tools() + # The finding first, as asked, then a long quote of host output. + # Characters, not bytes: the Cyrillic quote is counted the same. + answer = (_FINDING + "журнал;" * length)[:length] + assert len(answer) == length + + done = investigate( + "alert", _tool_list(report, rotate), ENDPOINT, _Script(_say(answer)) + ) + saved = InvestigationStore( + directory=tmp_path / "investigations", + catalog=SchemaCatalog(default_schema_directory()), + ).save(done) + + assert ANSWER_LIMIT == 8000 + assert saved.status is InvestigationStatus.ANSWERED + assert saved.finding is not None + assert saved.finding.root_cause == "disk" + if length == ANSWER_LIMIT: + assert saved.answer == answer + return + # The mark counts against the limit; a shorter count can leave it under. + assert ANSWER_LIMIT - 3 <= len(saved.answer) <= ANSWER_LIMIT + head, mark = saved.answer.rsplit("\n", 1) + assert answer.startswith(head) + assert mark == f"[last {length - len(head)} characters cut]" + + +def test_a_finding_past_the_cut_is_still_read_from_the_whole_answer() -> None: + report, rotate = _tools() + answer = "x" * 9000 + "\n" + _FINDING + + done = investigate( + "alert", _tool_list(report, rotate), ENDPOINT, _Script(_say(answer)) + ) + + assert done.status is InvestigationStatus.ANSWERED + assert done.finding is not None + assert done.finding.why == "read only." + assert "ROOT CAUSE" not in done.answer + + +def test_the_cli_streams_the_answer_as_the_record_keeps_it( + tmp_path: Path, capsys: pytest.CaptureFixture[str], model_url: str +) -> None: + repository = _repository(tmp_path) + key = tmp_path / "model.key" + key.write_text("sk-test\n", encoding="utf-8") + _Model.replies[:] = [_say(_FINDING + "y" * 9000)] + + assert _investigate_against(repository, key, model_url) == 0 + + streamed = _streamed(capsys.readouterr().out)["data"]["investigation"] + [recorded] = (repository / "investigations").glob("*.json") + kept = json.loads(recorded.read_text(encoding="utf-8")) + assert streamed["spec"]["answer"] == kept["spec"]["answer"] + assert kept["spec"]["answer"].endswith(" characters cut]") + assert kept["spec"]["finding"]["rootCause"] == "disk" + + +class _Echo: + """A tool taking a note, which keeps the arguments it was called with.""" + + def __init__(self) -> None: + self.received: list[Mapping[str, object]] = [] + self.tool = AgentTool( + name="note", + description="note", + input_schema={ + "type": "object", + "properties": {"note": {"type": "string"}}, + "additionalProperties": False, + }, + call=self._call, + ) + + def _call(self, arguments: Mapping[str, object]) -> ToolResult: + self.received.append(arguments) + return ToolResult(StepOutcome.RAN, {"output": "noted"}) + + +@pytest.mark.parametrize("length", [ARGUMENTS_LIMIT, ARGUMENTS_LIMIT + 1, 50_000]) +def test_long_raw_arguments_keep_their_start_in_the_trace_only( + tmp_path: Path, length: int +) -> None: + echo = _Echo() + # {"note": "..."} with the note sized so the raw string is `length` long. + note = "n" * (length - len(json.dumps({"note": ""}))) + raw = json.dumps({"note": note}) + assert len(raw) == length + script = _Script(_call("note", {"note": note}), _say(_FINDING)) + + done = investigate("alert", [echo.tool], ENDPOINT, script) + saved = InvestigationStore( + directory=tmp_path / "investigations", + catalog=SchemaCatalog(default_schema_directory()), + ).save(done) + + # The tool ran on the whole arguments; only the trace's copy is cut. + assert echo.received == [{"note": note}] + assert saved.steps[0].arguments == {"note": note} + kept = saved.trace[0].tool_calls[0].arguments + assert ARGUMENTS_LIMIT == 8000 + if length == ARGUMENTS_LIMIT: + assert kept == raw + return + assert ARGUMENTS_LIMIT - 3 <= len(kept) <= ARGUMENTS_LIMIT + head, mark = kept.rsplit("\n", 1) + assert raw.startswith(head) + assert mark == f"[last {length - len(head)} characters cut]" + + +@pytest.mark.parametrize("field", ["answer", "arguments"]) +def test_the_schema_refuses_an_answer_or_arguments_past_the_limit( + field: str, +) -> None: + report, rotate = _tools() + done = investigate( + "alert", + _tool_list(report, rotate), + ENDPOINT, + _Script(_call("disk-report", {}), _say(_FINDING)), + ) + document = done.as_document() + spec = document["spec"] + assert isinstance(spec, dict) + catalog = SchemaCatalog(default_schema_directory()) + catalog.validate_named("agent-investigation.schema.json", document) + if field == "answer": + spec["answer"] = "x" * (ANSWER_LIMIT + 1) + else: + spec["trace"][0]["toolCalls"][0]["arguments"] = "x" * (ARGUMENTS_LIMIT + 1) + + with pytest.raises(ValidationError): + catalog.validate_named("agent-investigation.schema.json", document) + + class _LongIds(_Script): """An endpoint that serves its answers under over-long ids."""