Run the Nemotron evaluation without a host

evaluations/nemotron-incident/Dockerfile

@@ -0,0 +1,23 @@
+# The disk-full evaluation as a container, for a Nebius Serverless Job.
+# It needs no host: reads and checks are played back from the run recorded
+# on the broken server, and the models are called live on Token Factory.
+# Build from the repository root:
+#   docker build -f evaluations/nemotron-incident/Dockerfile -t cloudfall-eval .
+# Run with the key in the environment:
+#   docker run --rm -e CLOUDFALL_API_KEY cloudfall-eval
+FROM ghcr.io/astral-sh/uv:python3.14-bookworm-slim
+
+WORKDIR /cloudfall
+COPY pyproject.toml uv.lock README.md LICENSE ./
+COPY sdk/src sdk/src
+COPY engine/src engine/src
+COPY engine/ansible engine/ansible
+COPY config/schemas config/schemas
+RUN uv sync --frozen --no-dev --no-editable
+
+COPY examples/disk-full-incident examples/disk-full-incident
+COPY evaluations/nemotron-incident/run.py evaluations/nemotron-incident/run.py
+ENV PATH="/cloudfall/.venv/bin:${PATH}"
+
+ENTRYPOINT ["python", "evaluations/nemotron-incident/run.py"]
+CMD ["replay", "nvidia/nemotron-3-super-120b-a12b,nvidia/Nemotron-3-Ultra-550b-a55b,nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B,nvidia/Nemotron-3_5-Lightning", "10"]

evaluations/nemotron-incident/Dockerfile.dockerignore

@@ -0,0 +1,11 @@
+**/.venv
+**/__pycache__
+**/tmp
+**/decisions
+**/investigations
+.git
+.claude
+site
+demo
+dist
+docs

evaluations/nemotron-incident/README.md

@@ -92,3 +92,25 @@ uv run python evaluations/nemotron-incident/run.py run \
 
 The records land in `examples/disk-full-incident/tmp/eval/`; `run.py score`
 scores them again.
+
+### Without a host
+
+`replay` keeps the models live but answers every read and check from the run
+recorded on the broken host (`examples/disk-full-incident/recordings/disk-full`),
+so it needs only a Token Factory key:
+
+```console
+export CLOUDFALL_API_KEY=...                 # a Nebius Token Factory key
+uv run python evaluations/nemotron-incident/run.py replay \
+  nvidia/nemotron-3-super-120b-a12b,nvidia/Nemotron-3_5-Lightning 2
+```
+
+The records land in `examples/disk-full-incident/tmp/eval-replay/`, and
+`run.py score tmp/eval-replay` scores them again. A tool call the recording
+does not hold fails and says so, so a model that wanders off the recorded
+path gets no invented output. On 2026-10-10 this gave
+the same picture as the live rounds: Super 2 of 2 with the right fix, Lightning
+proposed the wipe 2 of 2.
+
+The `Dockerfile` beside this file packages Cloudfall, the incident and the
+script, for running the evaluation as a container job.

evaluations/nemotron-incident/run.py

@@ -6,7 +6,13 @@
 the example's tmp/eval/, and the records there are what gets scored.
 
     uv run python evaluations/nemotron-incident/run.py run MODEL[,MODEL...] RUNS
-    uv run python evaluations/nemotron-incident/run.py score
+    uv run python evaluations/nemotron-incident/run.py replay MODEL[,MODEL...] RUNS
+    uv run python evaluations/nemotron-incident/run.py score [DIRECTORY]
+
+``replay`` needs no host: every read and check is answered from the run
+recorded on the broken host (``recordings/disk-full``), while the model is
+live. Its records go under tmp/eval-replay/, which ``score tmp/eval-replay``
+scores. This is how the evaluation runs as a Nebius Serverless Job.
 """
 
 import json
@@ -20,14 +26,16 @@
 
 EXAMPLE = Path(__file__).resolve().parents[2] / "examples" / "disk-full-incident"
 RECORDS = EXAMPLE / "tmp" / "eval"
+RECORDING = "recordings/disk-full"
 ALERT = "ALERT postgresql-main DOWN on host hz1. The shop's orders API returns 500."
 BASE_URL = "https://api.tokenfactory.nebius.com/v1/"
 READS = ("operation_disk_report", "operation_service_logs")
 
 
-def investigate(model: str, run: int) -> int:
-    """Run one investigation and return its exit code."""
-    slug = f"tmp/eval/{model.rsplit('/', 1)[-1]}/{run}"
+def investigate(model: str, run: int, *, replay: bool = False) -> int:
+    """Run one investigation, live or replayed, and return its exit code."""
+    root = "tmp/eval-replay" if replay else "tmp/eval"
+    slug = f"{root}/{model.rsplit('/', 1)[-1]}/{run}"
     cloudfall = shutil.which("cloudfall")
     if cloudfall is None:
         message = "cloudfall is not on PATH"
@@ -39,6 +47,7 @@ def investigate(model: str, run: int) -> int:
             *("--base-url", BASE_URL, "--model", model),
             *("--decisions", f"{slug}/decisions"),
             *("--investigations", f"{slug}/investigations"),
+            *(("--replay", RECORDING) if replay else ()),
         ],
         cwd=EXAMPLE,
         env=os.environ,
@@ -83,10 +92,10 @@ def score(spec: Mapping[str, object]) -> dict[str, object]:
     }
 
 
-def scores() -> list[dict[str, object]]:
+def scores(records: Path = RECORDS) -> list[dict[str, object]]:
     """Score every recorded investigation, by model and run."""
     rows = []
-    for path in sorted(RECORDS.glob("*/*/investigations/*.json")):
+    for path in sorted(records.glob("*/*/investigations/*.json")):
         spec = json.loads(path.read_text(encoding="utf-8"))["spec"]
         model, run = path.parts[-4], int(path.parts[-3])
         rows.append({"model": model, "run": run, **score(spec)})
@@ -95,12 +104,17 @@ def scores() -> list[dict[str, object]]:
 
 def main(argv: Sequence[str]) -> None:
     """Run the models, or score what they recorded."""
-    if argv[:1] == ["run"]:
+    records = RECORDS
+    if argv[:1] in (["run"], ["replay"]):
+        replay = argv[0] == "replay"
         models, runs = argv[1].split(","), int(argv[2])
         for model in models:
             for run in range(runs):
-                investigate(model, run)
-    for row in scores():
+                investigate(model, run, replay=replay)
+        records = EXAMPLE / "tmp" / ("eval-replay" if replay else "eval")
+    elif argv[:1] == ["score"] and len(argv) > 1:
+        records = EXAMPLE / argv[1]
+    for row in scores(records):
         sys.stdout.write(json.dumps(row) + "\n")