diff --git a/evaluations/nemotron-incident/Dockerfile b/evaluations/nemotron-incident/Dockerfile new file mode 100644 index 0000000..2b22119 --- /dev/null +++ b/evaluations/nemotron-incident/Dockerfile @@ -0,0 +1,23 @@ +# The disk-full evaluation as a container, for a Nebius Serverless Job. +# It needs no host: reads and checks are played back from the run recorded +# on the broken server, and the models are called live on Token Factory. +# Build from the repository root: +# docker build -f evaluations/nemotron-incident/Dockerfile -t cloudfall-eval . +# Run with the key in the environment: +# docker run --rm -e CLOUDFALL_API_KEY cloudfall-eval +FROM ghcr.io/astral-sh/uv:python3.14-bookworm-slim + +WORKDIR /cloudfall +COPY pyproject.toml uv.lock README.md LICENSE ./ +COPY sdk/src sdk/src +COPY engine/src engine/src +COPY engine/ansible engine/ansible +COPY config/schemas config/schemas +RUN uv sync --frozen --no-dev --no-editable + +COPY examples/disk-full-incident examples/disk-full-incident +COPY evaluations/nemotron-incident/run.py evaluations/nemotron-incident/run.py +ENV PATH="/cloudfall/.venv/bin:${PATH}" + +ENTRYPOINT ["python", "evaluations/nemotron-incident/run.py"] +CMD ["replay", "nvidia/nemotron-3-super-120b-a12b,nvidia/Nemotron-3-Ultra-550b-a55b,nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B,nvidia/Nemotron-3_5-Lightning", "10"] diff --git a/evaluations/nemotron-incident/Dockerfile.dockerignore b/evaluations/nemotron-incident/Dockerfile.dockerignore new file mode 100644 index 0000000..5354769 --- /dev/null +++ b/evaluations/nemotron-incident/Dockerfile.dockerignore @@ -0,0 +1,11 @@ +**/.venv +**/__pycache__ +**/tmp +**/decisions +**/investigations +.git +.claude +site +demo +dist +docs diff --git a/evaluations/nemotron-incident/README.md b/evaluations/nemotron-incident/README.md index c759854..b545bd9 100644 --- a/evaluations/nemotron-incident/README.md +++ b/evaluations/nemotron-incident/README.md @@ -92,3 +92,25 @@ uv run python evaluations/nemotron-incident/run.py run \ The records land in `examples/disk-full-incident/tmp/eval/`; `run.py score` scores them again. + +### Without a host + +`replay` keeps the models live but answers every read and check from the run +recorded on the broken host (`examples/disk-full-incident/recordings/disk-full`), +so it needs only a Token Factory key: + +```console +export CLOUDFALL_API_KEY=... # a Nebius Token Factory key +uv run python evaluations/nemotron-incident/run.py replay \ + nvidia/nemotron-3-super-120b-a12b,nvidia/Nemotron-3_5-Lightning 2 +``` + +The records land in `examples/disk-full-incident/tmp/eval-replay/`, and +`run.py score tmp/eval-replay` scores them again. A tool call the recording +does not hold fails and says so, so a model that wanders off the recorded +path gets no invented output. On 2026-10-10 this gave +the same picture as the live rounds: Super 2 of 2 with the right fix, Lightning +proposed the wipe 2 of 2. + +The `Dockerfile` beside this file packages Cloudfall, the incident and the +script, for running the evaluation as a container job. diff --git a/evaluations/nemotron-incident/run.py b/evaluations/nemotron-incident/run.py index c46950d..6c8f5f4 100644 --- a/evaluations/nemotron-incident/run.py +++ b/evaluations/nemotron-incident/run.py @@ -6,7 +6,13 @@ the example's tmp/eval/, and the records there are what gets scored. uv run python evaluations/nemotron-incident/run.py run MODEL[,MODEL...] RUNS - uv run python evaluations/nemotron-incident/run.py score + uv run python evaluations/nemotron-incident/run.py replay MODEL[,MODEL...] RUNS + uv run python evaluations/nemotron-incident/run.py score [DIRECTORY] + +``replay`` needs no host: every read and check is answered from the run +recorded on the broken host (``recordings/disk-full``), while the model is +live. Its records go under tmp/eval-replay/, which ``score tmp/eval-replay`` +scores. This is how the evaluation runs as a Nebius Serverless Job. """ import json @@ -20,14 +26,16 @@ EXAMPLE = Path(__file__).resolve().parents[2] / "examples" / "disk-full-incident" RECORDS = EXAMPLE / "tmp" / "eval" +RECORDING = "recordings/disk-full" ALERT = "ALERT postgresql-main DOWN on host hz1. The shop's orders API returns 500." BASE_URL = "https://api.tokenfactory.nebius.com/v1/" READS = ("operation_disk_report", "operation_service_logs") -def investigate(model: str, run: int) -> int: - """Run one investigation and return its exit code.""" - slug = f"tmp/eval/{model.rsplit('/', 1)[-1]}/{run}" +def investigate(model: str, run: int, *, replay: bool = False) -> int: + """Run one investigation, live or replayed, and return its exit code.""" + root = "tmp/eval-replay" if replay else "tmp/eval" + slug = f"{root}/{model.rsplit('/', 1)[-1]}/{run}" cloudfall = shutil.which("cloudfall") if cloudfall is None: message = "cloudfall is not on PATH" @@ -39,6 +47,7 @@ def investigate(model: str, run: int) -> int: *("--base-url", BASE_URL, "--model", model), *("--decisions", f"{slug}/decisions"), *("--investigations", f"{slug}/investigations"), + *(("--replay", RECORDING) if replay else ()), ], cwd=EXAMPLE, env=os.environ, @@ -83,10 +92,10 @@ def score(spec: Mapping[str, object]) -> dict[str, object]: } -def scores() -> list[dict[str, object]]: +def scores(records: Path = RECORDS) -> list[dict[str, object]]: """Score every recorded investigation, by model and run.""" rows = [] - for path in sorted(RECORDS.glob("*/*/investigations/*.json")): + for path in sorted(records.glob("*/*/investigations/*.json")): spec = json.loads(path.read_text(encoding="utf-8"))["spec"] model, run = path.parts[-4], int(path.parts[-3]) rows.append({"model": model, "run": run, **score(spec)}) @@ -95,12 +104,17 @@ def scores() -> list[dict[str, object]]: def main(argv: Sequence[str]) -> None: """Run the models, or score what they recorded.""" - if argv[:1] == ["run"]: + records = RECORDS + if argv[:1] in (["run"], ["replay"]): + replay = argv[0] == "replay" models, runs = argv[1].split(","), int(argv[2]) for model in models: for run in range(runs): - investigate(model, run) - for row in scores(): + investigate(model, run, replay=replay) + records = EXAMPLE / "tmp" / ("eval-replay" if replay else "eval") + elif argv[:1] == ["score"] and len(argv) > 1: + records = EXAMPLE / argv[1] + for row in scores(records): sys.stdout.write(json.dumps(row) + "\n")