From 65665550a58e0806d3c6a76e78cd7a03091a782f Mon Sep 17 00:00:00 2001 From: Naser Derakhshan <9349840+nasser1941@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:09:01 +0100 Subject: [PATCH] Make lcode bench --compare say when attempts are missing A task with one attempt counted as certain, so the noise came out too small, and a run that lost 18 of its 72 attempts compared as usual. A single attempt's pass rate now comes from both runs, and --compare warns when the two runs didn't have the same number of attempts. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 +++++++ docs/public-benchmarks.md | 8 +++++--- src/lcode/harbor_bench.py | 21 ++++++++++++++++----- tests/test_harbor_bench.py | 13 +++++++++++++ 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22694f0..4bcaa2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ All notable changes to lcode are documented here. The format follows ## [Unreleased] +### Fixed + +- **`lcode bench --compare`** took a task with one attempt as certain, so its noise came out too small, + and it said nothing when a run had far fewer attempts: a real run lost 18 of its 72 attempts and + compared as usual. A single attempt's pass rate now comes from both runs, and `--compare` warns when + the two runs didn't have the same number of attempts. + ## [0.61.0] - 2026-11-05 ### Added diff --git a/docs/public-benchmarks.md b/docs/public-benchmarks.md index bfef554..2e0b9dd 100644 --- a/docs/public-benchmarks.md +++ b/docs/public-benchmarks.md @@ -127,9 +127,11 @@ limits, so part of a difference can come from the machine rather than from lcode The noise is how far pass@1 moves between runs of the same setup, estimated from how much each task's attempts disagree: a task that always passes or always fails adds nothing, one that passes -half the time the most. A change counts when it's more than twice the standard error of the -difference. On the dev slice, with three attempts per task, that's about 9 points. `--compare` exits -with 1 when the second result dropped beyond the noise, so a script can stop on it. +half the time the most. One attempt can't disagree with itself, so that task's pass rate comes from +both runs. A change counts when it's more than twice the standard error of the difference. On the dev +slice, with three attempts per task, that's about 9 points. `--compare` warns when the two runs didn't +have the same number of attempts (a container that didn't start, say), and exits with 1 when the second +result dropped beyond the noise, so a script can stop on it. **The release rule:** a release that changes lcode's prompts, tools or context handling ships only if the dev slice doesn't drop beyond the noise against the last baseline, and the comparison goes on diff --git a/src/lcode/harbor_bench.py b/src/lcode/harbor_bench.py index 62b03a0..6a08efe 100644 --- a/src/lcode/harbor_bench.py +++ b/src/lcode/harbor_bench.py @@ -602,10 +602,14 @@ def pass_at_1(passed: list[list[bool]]) -> float: return statistics.fmean(statistics.fmean(p) for p in passed) -def standard_error(passed: list[list[bool]]) -> float: +def standard_error(passed: list[list[bool]], other: list[list[bool]]) -> float: """How much pass@1 would move between runs, from how much each task's attempts disagree: a task that always - passes or always fails adds nothing, one that passes half the time the most.""" - variance = sum(statistics.fmean(p) * (1 - statistics.fmean(p)) / max(len(p) - 1, 1) for p in passed) + passes or always fails adds nothing, one that passes half the time the most. One attempt can't disagree with + itself, so that task's pass rate comes from both runs (`other` is the other run's attempts, task by task).""" + variance = 0.0 + for attempts, others in zip(passed, other, strict=True): + rate = statistics.fmean(attempts if len(attempts) > 1 else attempts + others) + variance += rate * (1 - rate) / max(len(attempts) - 1, 1) return math.sqrt(variance) / len(passed) @@ -638,8 +642,15 @@ def compare(old: Saved, new: Saved) -> tuple[list[str], bool]: lines.append(f" {task}: {sum(before)}/{len(before)} → {sum(after)}/{len(after)}") if not both: return [*lines, "[red]No task ran in both, so there's nothing to compare.[/]"], False - a, b = pass_at_1([old.passed[t] for t in both]), pass_at_1([new.passed[t] for t in both]) - noise = 2 * math.hypot(standard_error([old.passed[t] for t in both]), standard_error([new.passed[t] for t in both])) + runs_before, runs_after = [old.passed[t] for t in both], [new.passed[t] for t in both] + ran_before, ran_after = sum(map(len, runs_before)), sum(map(len, runs_after)) + if ran_before != ran_after: + lines.append( + f"[yellow]Not the same number of attempts ran: {ran_before} before and {ran_after} after, over the tasks " + "in both. With fewer attempts, a result says less.[/]" + ) + a, b = pass_at_1(runs_before), pass_at_1(runs_after) + noise = 2 * math.hypot(standard_error(runs_before, runs_after), standard_error(runs_after, runs_before)) change = f"{(b - a) * 100:+.0f} points; the noise is ±{noise * 100:.0f}" if a - b > noise + 1e-9: verdict = f"[red]a drop beyond the noise ({change})[/]" diff --git a/tests/test_harbor_bench.py b/tests/test_harbor_bench.py index 1c692eb..516cae2 100644 --- a/tests/test_harbor_bench.py +++ b/tests/test_harbor_bench.py @@ -456,6 +456,19 @@ def test_compare_reads_both_runs_again_from_harbors_results(tmp_path): assert any("Model speed" in line for line in near[0]) and not any("slower" in line for line in near[0]) +def test_compare_doesnt_take_one_attempt_as_certain(tmp_path): + before = {f"t{n}": SOMETIMES for n in range(6)} + code, text = compared(tmp_path, before, {f"t{n}": [True] for n in range(6)}) + assert code == 0 and "Not the same number of attempts ran: 18 before and 6 after" in text + assert "pass@1 67% → 100% over the 6 tasks in both: within the noise (+33 points; the noise is ±45)" in text + + +def test_compare_of_single_attempts_that_disagree_isnt_a_drop(tmp_path): + code, text = compared(tmp_path, {"t0": [True]}, {"t0": [False]}) + assert code == 0 and "within the noise (-100 points; the noise is ±141)" in text + assert "Not the same number of attempts" not in text + + def test_compare_needs_two_results(tmp_path): console = Console(file=io.StringIO(), width=200) (tmp_path / "other.json").write_text('{"schema": 1, "runs": []}')