diff --git a/demo/pyproject.toml b/demo/pyproject.toml index 493a567..dc10cc7 100644 --- a/demo/pyproject.toml +++ b/demo/pyproject.toml @@ -19,7 +19,7 @@ package = true [tool.uv.sources] # The demo is deployed from this directory alone, so Cloudfall comes from git, # pinned: bump the rev to ship a new Cloudfall to the demo. -cloudfall = { git = "https://github.com/cloudfall-dev/cloudfall", rev = "9e51dc569a758504d82828e157dc6f65b9978bd9" } +cloudfall = { git = "https://github.com/cloudfall-dev/cloudfall", rev = "6b4807884de7cfe800548bbd90ad868a1393267c" } [build-system] requires = ["hatchling>=1.27"] diff --git a/demo/src/cloudfall_demo/server.py b/demo/src/cloudfall_demo/server.py index f183a74..96ed393 100644 --- a/demo/src/cloudfall_demo/server.py +++ b/demo/src/cloudfall_demo/server.py @@ -5,7 +5,9 @@ recorded on a Hetzner host and streams its events to the page as they happen. The model is live, through Nebius Token Factory; the host is recorded, so nothing anyone does here can touch a server. Approving a -decision plays back the approval that was recorded on the same host. +decision plays back an approval recorded on a real host broken the same +way. Once a run ends, ``cloudfall why`` answers from its record what the +agent proposed and what it was thinking in the turn that asked. """ import asyncio @@ -142,6 +144,48 @@ async def start_run(request: Request) -> Response: ) +async def why(request: Request) -> Response: + """Answer from one run's record, as ``cloudfall why`` does, with the command.""" + run_id = request.query_params.get("run", "") + run = INCIDENT / RUNS / run_id + if not run_id.isalnum() or not run.is_dir(): + return JSONResponse({"error": "unknown run"}, status_code=404) + arguments = ( + "why", + *("--decisions", (RUNS / run_id / "decisions").as_posix()), + *("--investigations", (RUNS / run_id / "investigations").as_posix()), + ) + process = await asyncio.create_subprocess_exec( + _cloudfall(), + *arguments, + "--format", + "json", + cwd=INCIDENT, + env={**os.environ, "PATH": f"{_BIN}{os.pathsep}{os.environ['PATH']}"}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + output, _ = await process.communicate() + envelope = json.loads(output) + if not envelope["ok"]: + return JSONResponse({"error": envelope["error"]["message"]}, status_code=500) + return JSONResponse( + { + "command": " ".join(["cloudfall", *arguments]), + "answers": [ + { + "id": entry["id"], + "operation": entry["decision"]["spec"]["operation"]["id"], + "status": entry["decision"]["spec"]["status"], + "story": entry["story"], + "agent": entry.get("agent"), + } + for entry in envelope["data"]["answers"] + ], + } + ) + + async def approve(request: Request) -> Response: """Play back the recorded approval of the operation a decision proposes.""" body = await request.json() @@ -387,6 +431,7 @@ def _line(document: Mapping[str, object]) -> bytes: Route("/api/config", config), Route("/api/runs", start_run, methods=["POST"]), Route("/api/approve", approve, methods=["POST"]), + Route("/api/why", why), Route("/health", lambda _request: JSONResponse({"status": "ok"})), ] ) diff --git a/demo/src/cloudfall_demo/static/index.html b/demo/src/cloudfall_demo/static/index.html index 040b187..b905167 100644 --- a/demo/src/cloudfall_demo/static/index.html +++ b/demo/src/cloudfall_demo/static/index.html @@ -112,6 +112,13 @@ .command { font: .72rem var(--mono); background: var(--code); border: 1px solid var(--line); border-radius: 8px; padding: 8px 10px; margin-bottom: 12px; overflow-wrap: anywhere } .tools { display: flex; gap: 10px; align-items: center; flex-wrap: wrap; margin-top: 10px } .tools button { border: 1.5px solid var(--line); background: var(--card); color: var(--ink); border-radius: 8px; padding: 5px 12px; font: 600 .82rem var(--body); cursor: pointer } +#record[hidden] { display: none } +.answer { background: var(--card); border: 1px solid var(--line); border-radius: 12px; padding: 14px; margin-top: 12px } +.answer .top { display: flex; justify-content: space-between; gap: 8px; align-items: center; flex-wrap: wrap } +.answer b { font: 700 1rem var(--display) } +.answer ol { margin: 10px 0 0; padding-left: 22px; font-size: .9rem } +.answer li { margin-bottom: 5px; overflow-wrap: anywhere } +.answer li.agent { background: color-mix(in srgb, var(--sun) 10%, transparent); border-left: 3px solid var(--sun); padding: 4px 8px; border-radius: 0 8px 8px 0; list-style-position: inside; margin-left: -22px } @@ -143,6 +150,12 @@

Waiting for you

+ +

40 runs on a real broken host

The same alert, ten times per model, against a Hetzner server whose disk was really full. If the tools had run whatever the model asked:

@@ -343,6 +356,25 @@

How it works

$("#timeline").append(box); } +async function showRecord() { + const response = await fetch(`/api/why?run=${encodeURIComponent(state.run)}`); + const result = await response.json(); + if (!response.ok) return; + const shown = result.answers.filter((entry) => entry.status !== "ran"); + const answers = shown.map((entry) => el("div", { class: "answer" }, + el("div", { class: "top" }, el("b", {}, entry.operation), el("span", { class: `badge b-${entry.status}` }, entry.status)), + el("code", { class: "ids" }, entry.id), + el("ol", {}, ...entry.story.map((sentence) => + el("li", { class: /^(The agent |Its reasoning |That investigation )/.test(sentence) ? "agent" : "" }, sentence))))); + const reads = result.answers.length - shown.length; + $("#answers").replaceChildren( + el("div", { class: "command" }, el("span", { class: "label" }, "the server runs"), "$ " + result.command), + ...answers, + el("p", { class: "status" }, shown.length ? `${reads} read operation${reads === 1 ? "" : "s"} ran freely and are on the record too; the raw answer has them.` : "The model proposed no change; its reads are on the record."), + rawDetails("raw answer", JSON.stringify(result, null, 2))); + $("#record").hidden = false; +} + function download() { const blob = new Blob([state.raw.join("\n") + "\n"], { type: "application/x-ndjson" }); const link = el("a", { href: URL.createObjectURL(blob), download: `cloudfall-investigation-${state.run}.ndjson` }); @@ -351,10 +383,12 @@

How it works

async function run() { $("#run").disabled = true; + state.run = null; state.decisions.clear(); state.events.clear(); state.raw = []; $("#timeline").replaceChildren(); + $("#record").hidden = true; $("#decisions").replaceChildren(el("p", { class: "empty" }, "Nothing proposed yet.")); $("#status").textContent = "Running…"; thinking(true); @@ -385,6 +419,7 @@

How it works

thinking(false); $("#run").disabled = false; } + if (state.run) await showRecord(); } function handle(text, line) { diff --git a/demo/tests/test_demo.py b/demo/tests/test_demo.py index ca8066d..9c57663 100644 --- a/demo/tests/test_demo.py +++ b/demo/tests/test_demo.py @@ -148,3 +148,71 @@ def test_a_decision_outside_the_runs_is_refused() -> None: assert response.status_code == 400 assert json.loads(response.text)["error"] == "unknown decision" + + +def _investigated(run: str, decision: str) -> None: + """Record, in a run, the investigation whose second turn asked for it.""" + investigations = server.INCIDENT / server.RUNS / run / "investigations" + investigations.mkdir(parents=True, exist_ok=True) + record = { + "apiVersion": "cloudfall/v1", + "kind": "AgentInvestigation", + "metadata": {"id": "investigation-20261011120000"}, + "spec": { + "alert": server.ALERT, + "model": {"endpoint": server.BASE_URL, "name": "nvidia/nemotron"}, + "startedAt": "2026-10-11T12:00:00Z", + "finishedAt": "2026-10-11T12:00:09Z", + "status": "answered", + "turns": 2, + "tokens": {"prompt": 100, "completion": 10}, + "steps": [ + { + "tool": "operation_shop_logrotate", + "arguments": {"target": "hz1"}, + "outcome": "proposed", + "decision": decision, + "turn": 2, + } + ], + "answer": "", + "trace": [ + { + "turn": 2, + "text": "", + "reasoning": "The shop logs filled the disk.", + "toolCalls": [], + "elapsedMs": 900, + "tokens": {"prompt": 100, "completion": 10, "reasoning": 6}, + "responseId": "chatcmpl-2", + } + ], + }, + } + (investigations / "investigation-20261011120000.json").write_text( + json.dumps(record), encoding="utf-8" + ) + + +def test_the_record_tells_which_turn_asked_for_a_change() -> None: + decision = _proposed("testrun4", "shop-logrotate") + _investigated("testrun4", decision) + try: + result = CLIENT.get("/api/why", params={"run": "testrun4"}).json() + finally: + shutil.rmtree(server.INCIDENT / server.RUNS / "testrun4") + + assert result["command"].startswith("cloudfall why --decisions tmp/runs/testrun4/") + [answer] = result["answers"] + assert (answer["operation"], answer["status"]) == ("shop-logrotate", "proposed") + assert answer["agent"]["turn"] == 2 + assert any( + sentence.startswith("The agent nvidia/nemotron") and "in turn 2" in sentence + for sentence in answer["story"] + ) + assert any("The shop logs filled the disk." in line for line in answer["story"]) + + +def test_the_record_of_an_unknown_run_is_refused() -> None: + assert CLIENT.get("/api/why", params={"run": "nosuchrun"}).status_code == 404 + assert CLIENT.get("/api/why", params={"run": "../x"}).status_code == 404 diff --git a/demo/uv.lock b/demo/uv.lock index 23a88c2..c27db69 100644 --- a/demo/uv.lock +++ b/demo/uv.lock @@ -136,8 +136,8 @@ wheels = [ [[package]] name = "cloudfall" -version = "0.5.1" -source = { git = "https://github.com/cloudfall-dev/cloudfall?rev=9e51dc569a758504d82828e157dc6f65b9978bd9#9e51dc569a758504d82828e157dc6f65b9978bd9" } +version = "0.6.0" +source = { git = "https://github.com/cloudfall-dev/cloudfall?rev=6b4807884de7cfe800548bbd90ad868a1393267c#6b4807884de7cfe800548bbd90ad868a1393267c" } dependencies = [ { name = "ansible-core" }, { name = "jsonschema", extra = ["format"] }, @@ -164,7 +164,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "cloudfall", git = "https://github.com/cloudfall-dev/cloudfall?rev=9e51dc569a758504d82828e157dc6f65b9978bd9" }, + { name = "cloudfall", git = "https://github.com/cloudfall-dev/cloudfall?rev=6b4807884de7cfe800548bbd90ad868a1393267c" }, { name = "starlette", specifier = ">=0.46,<2" }, { name = "uvicorn", specifier = ">=0.34,<1" }, ]