Make lcode bench --compare say when attempts are missing

CHANGELOG.md

@@ -13,6 +13,10 @@ All notable changes to lcode are documented here. The format follows
   run them, and `lcode -p --output json` reported them as passed. A run whose exit code isn't the check's
   now counts only if its output shows a test summary with nothing failing. Otherwise lcode runs the
   checks itself.
+- **`lcode bench --compare`** took a task with one attempt as certain, so its noise came out too small,
+  and it said nothing when a run had far fewer attempts: a real run lost 18 of its 72 attempts and
+  compared as usual. A single attempt's pass rate now comes from both runs, and `--compare` warns when
+  the two runs didn't have the same number of attempts.
 
 ## [0.61.0] - 2026-11-05
 

docs/public-benchmarks.md

@@ -127,9 +127,11 @@ limits, so part of a difference can come from the machine rather than from lcode
 
 The noise is how far pass@1 moves between runs of the same setup, estimated from how much each
 task's attempts disagree: a task that always passes or always fails adds nothing, one that passes
-half the time the most. A change counts when it's more than twice the standard error of the
-difference. On the dev slice, with three attempts per task, that's about 9 points. `--compare` exits
-with 1 when the second result dropped beyond the noise, so a script can stop on it.
+half the time the most. One attempt can't disagree with itself, so that task's pass rate comes from
+both runs. A change counts when it's more than twice the standard error of the difference. On the dev
+slice, with three attempts per task, that's about 9 points. `--compare` warns when the two runs didn't
+have the same number of attempts (a container that didn't start, say), and exits with 1 when the second
+result dropped beyond the noise, so a script can stop on it.
 
 **The release rule:** a release that changes lcode's prompts, tools or context handling ships only
 if the dev slice doesn't drop beyond the noise against the last baseline, and the comparison goes on

src/lcode/harbor_bench.py

@@ -602,10 +602,14 @@ def pass_at_1(passed: list[list[bool]]) -> float:
     return statistics.fmean(statistics.fmean(p) for p in passed)
 
 
-def standard_error(passed: list[list[bool]]) -> float:
+def standard_error(passed: list[list[bool]], other: list[list[bool]]) -> float:
     """How much pass@1 would move between runs, from how much each task's attempts disagree: a task that always
-    passes or always fails adds nothing, one that passes half the time the most."""
-    variance = sum(statistics.fmean(p) * (1 - statistics.fmean(p)) / max(len(p) - 1, 1) for p in passed)
+    passes or always fails adds nothing, one that passes half the time the most. One attempt can't disagree with
+    itself, so that task's pass rate comes from both runs (`other` is the other run's attempts, task by task)."""
+    variance = 0.0
+    for attempts, others in zip(passed, other, strict=True):
+        rate = statistics.fmean(attempts if len(attempts) > 1 else attempts + others)
+        variance += rate * (1 - rate) / max(len(attempts) - 1, 1)
     return math.sqrt(variance) / len(passed)
 
 
@@ -638,8 +642,15 @@ def compare(old: Saved, new: Saved) -> tuple[list[str], bool]:
             lines.append(f"  {task}: {sum(before)}/{len(before)} → {sum(after)}/{len(after)}")
     if not both:
         return [*lines, "[red]No task ran in both, so there's nothing to compare.[/]"], False
-    a, b = pass_at_1([old.passed[t] for t in both]), pass_at_1([new.passed[t] for t in both])
-    noise = 2 * math.hypot(standard_error([old.passed[t] for t in both]), standard_error([new.passed[t] for t in both]))
+    runs_before, runs_after = [old.passed[t] for t in both], [new.passed[t] for t in both]
+    ran_before, ran_after = sum(map(len, runs_before)), sum(map(len, runs_after))
+    if ran_before != ran_after:
+        lines.append(
+            f"[yellow]Not the same number of attempts ran: {ran_before} before and {ran_after} after, over the tasks "
+            "in both. With fewer attempts, a result says less.[/]"
+        )
+    a, b = pass_at_1(runs_before), pass_at_1(runs_after)
+    noise = 2 * math.hypot(standard_error(runs_before, runs_after), standard_error(runs_after, runs_before))
     change = f"{(b - a) * 100:+.0f} points; the noise is ±{noise * 100:.0f}"
     if a - b > noise + 1e-9:
         verdict = f"[red]a drop beyond the noise ({change})[/]"

tests/test_harbor_bench.py

@@ -456,6 +456,19 @@ def test_compare_reads_both_runs_again_from_harbors_results(tmp_path):
     assert any("Model speed" in line for line in near[0]) and not any("slower" in line for line in near[0])
 
 
+def test_compare_doesnt_take_one_attempt_as_certain(tmp_path):
+    before = {f"t{n}": SOMETIMES for n in range(6)}
+    code, text = compared(tmp_path, before, {f"t{n}": [True] for n in range(6)})
+    assert code == 0 and "Not the same number of attempts ran: 18 before and 6 after" in text
+    assert "pass@1 67% → 100% over the 6 tasks in both: within the noise (+33 points; the noise is ±45)" in text
+
+
+def test_compare_of_single_attempts_that_disagree_isnt_a_drop(tmp_path):
+    code, text = compared(tmp_path, {"t0": [True]}, {"t0": [False]})
+    assert code == 0 and "within the noise (-100 points; the noise is ±141)" in text
+    assert "Not the same number of attempts" not in text
+
+
 def test_compare_needs_two_results(tmp_path):
     console = Console(file=io.StringIO(), width=200)
     (tmp_path / "other.json").write_text('{"schema": 1, "runs": []}')