From 4c74cd7bf166044699f14883220faf02c1afc907 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 18:18:57 -0700 Subject: [PATCH 1/6] =?UTF-8?q?fix(reports):=201/2=20=E2=80=94=20exact=20S?= =?UTF-8?q?tudent-t=20p-values=20in=20welch=5Ft=5Ftest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the anti-conservative NormalDist z-approximation with an exact two-tailed Student-t p-value via the regularized incomplete beta (continued fraction, stdlib only) plus Welch-Satterthwaite degrees of freedom. The old approximation understated p at small df and manufactured significance in the small-replicate regime coder_eval runs in (t=2.5, df=4: exact 0.0668 vs approximated 0.0124). Also fixes the docstring's claim of a nonexistent exact fallback, and makes zero-variance-with-different-means return p=0.0 (deterministic difference) instead of 1.0. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/coder_eval/reports_stats.py | 82 ++++++++++++++++--- .../report_snapshots/experiment_2variant.md | 8 +- tests/test_experiment_reports.py | 52 ++++++++++++ 3 files changed, 128 insertions(+), 14 deletions(-) diff --git a/src/coder_eval/reports_stats.py b/src/coder_eval/reports_stats.py index 00598578..a2e30d5e 100644 --- a/src/coder_eval/reports_stats.py +++ b/src/coder_eval/reports_stats.py @@ -33,12 +33,75 @@ def stddev(values: list[float]) -> float: return _stats.stdev(values) if len(values) >= 2 else 0.0 +def _betacf(a: float, b: float, x: float) -> float: + """Continued fraction for the regularized incomplete beta (Lentz's method).""" + max_iterations = 200 + eps = 3e-12 + fpmin = 1e-300 + + qab, qap, qam = a + b, a + 1.0, a - 1.0 + c = 1.0 + d = 1.0 - qab * x / qap + if abs(d) < fpmin: + d = fpmin + d = 1.0 / d + h = d + for m in range(1, max_iterations + 1): + m2 = 2 * m + # Even step of the recurrence. + aa = m * (b - m) * x / ((qam + m2) * (a + m2)) + d = 1.0 + aa * d + if abs(d) < fpmin: + d = fpmin + c = 1.0 + aa / c + if abs(c) < fpmin: + c = fpmin + d = 1.0 / d + h *= d * c + # Odd step. + aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2)) + d = 1.0 + aa * d + if abs(d) < fpmin: + d = fpmin + c = 1.0 + aa / c + if abs(c) < fpmin: + c = fpmin + d = 1.0 / d + delta = d * c + h *= delta + if abs(delta - 1.0) < eps: + break + return h + + +def regularized_incomplete_beta(a: float, b: float, x: float) -> float: + """Regularized incomplete beta function I_x(a, b), for a, b > 0 and x in [0, 1].""" + if x <= 0.0: + return 0.0 + if x >= 1.0: + return 1.0 + ln_front = math.lgamma(a + b) - math.lgamma(a) - math.lgamma(b) + a * math.log(x) + b * math.log1p(-x) + front = math.exp(ln_front) + # Use the continued fraction directly where it converges fast, else via symmetry. + if x < (a + 1.0) / (a + b + 2.0): + return front * _betacf(a, b, x) / a + return 1.0 - front * _betacf(b, a, 1.0 - x) / b + + +def student_t_two_tailed_p(t_stat: float, df: float) -> float: + """Exact two-tailed p-value for Student's t: P(|T| >= |t|) = I_x(df/2, 1/2), x = df/(df + t^2).""" + if df <= 0: + return 1.0 + x = df / (df + t_stat * t_stat) + return regularized_incomplete_beta(df / 2.0, 0.5, x) + + def welch_t_test(a: list[float], b: list[float]) -> float | None: - """Two-tailed p-value using Welch's t-test via stdlib NormalDist approximation. + """Two-tailed p-value from Welch's unequal-variances t-test (exact t distribution). - Uses the normal approximation for the t-distribution when df is large (>30), - and falls back to exact Welch-Satterthwaite otherwise. Returns None if either - group has fewer than 2 observations. + Degrees of freedom via Welch-Satterthwaite; the t CDF is evaluated exactly + through the regularized incomplete beta (stdlib only, no scipy). Returns + None if either group has fewer than 2 observations. """ n_a, n_b = len(a), len(b) if n_a < 2 or n_b < 2: @@ -50,14 +113,13 @@ def welch_t_test(a: list[float], b: list[float]) -> float | None: se_sq = var_a / n_a + var_b / n_b if se_sq == 0: - return 1.0 + # Zero variance in both groups: identical constants (p=1) or a + # deterministic difference (p=0). + return 1.0 if mean_a == mean_b else 0.0 t_stat = abs(mean_a - mean_b) / math.sqrt(se_sq) - - # Conservative normal approximation: treat t as z-score. - # Overestimates p slightly for small df (heavier tails), but correct in - # direction and sufficient for display purposes (no incomplete-beta needed). - return 2.0 * _stats.NormalDist().cdf(-t_stat) + df = se_sq**2 / ((var_a / n_a) ** 2 / (n_a - 1) + (var_b / n_b) ** 2 / (n_b - 1)) + return student_t_two_tailed_p(t_stat, df) def fmt_mean_sd(values: list[float], fmt: str = ".3f") -> str: diff --git a/tests/_fixtures/report_snapshots/experiment_2variant.md b/tests/_fixtures/report_snapshots/experiment_2variant.md index c9d13502..abc08553 100644 --- a/tests/_fixtures/report_snapshots/experiment_2variant.md +++ b/tests/_fixtures/report_snapshots/experiment_2variant.md @@ -20,10 +20,10 @@ | - Cost budget | 0 | 1 | — | | Errors | 0 | 0 | — | | Success Rate | 50.0% | 50.0% | — | -| Score | 0.700 ± 0.283 | 0.825 ± 0.177 | 0.596 | -| Avg Duration (s) | 35.0 ± 7.1 | 55.0 ± 7.1 | 0.005 | -| Assistant Turns | 5.5 ± 0.7 | 7.5 ± 0.7 | 0.005 | -| Tokens | 1,100 ± 141 | 1,900 ± 141 | <0.001 | +| Score | 0.700 ± 0.283 | 0.825 ± 0.177 | 0.658 | +| Avg Duration (s) | 35.0 ± 7.1 | 55.0 ± 7.1 | 0.106 | +| Assistant Turns | 5.5 ± 0.7 | 7.5 ± 0.7 | 0.106 | +| Tokens | 1,100 ± 141 | 1,900 ± 141 | 0.030 | ## Win Rates diff --git a/tests/test_experiment_reports.py b/tests/test_experiment_reports.py index c3bc3f5e..e7892ed7 100644 --- a/tests/test_experiment_reports.py +++ b/tests/test_experiment_reports.py @@ -748,6 +748,58 @@ def test_welch_t_test_insufficient_data(self): assert welch_t_test([1.0], [2.0]) is None + def test_welch_t_test_exact_reference_value(self): + """Exact Student-t p-value, cross-checked against scipy ttest_ind(equal_var=False).""" + from coder_eval.reports_stats import welch_t_test + + # Equal variances 2.5, n=5 each ⇒ t=1.0, Welch-Satterthwaite df=8. + p = welch_t_test([1.0, 2.0, 3.0, 4.0, 5.0], [2.0, 3.0, 4.0, 5.0, 6.0]) + assert p is not None + assert abs(p - 0.34659) < 5e-4 + + def test_welch_t_test_zero_variance_different_means(self): + """Zero variance in both groups: deterministic difference ⇒ 0.0, identical ⇒ 1.0.""" + from coder_eval.reports_stats import welch_t_test + + assert welch_t_test([1.0, 1.0], [2.0, 2.0]) == 0.0 + assert welch_t_test([1.0, 1.0], [1.0, 1.0]) == 1.0 + + def test_student_t_two_tailed_p_small_df_not_normal(self): + """Small df must use t tails, not normal ones — the regression sensor for the old bug.""" + from coder_eval.reports_stats import student_t_two_tailed_p + + p = student_t_two_tailed_p(2.5, 4.0) + assert abs(p - 0.06677) < 5e-4 + # The old normal approximation reported 0.0124 here — falsely significant. + assert p > 0.05 + + def test_student_t_two_tailed_p_critical_values(self): + """Round-trip the standard t-table critical values.""" + from coder_eval.reports_stats import student_t_two_tailed_p + + assert abs(student_t_two_tailed_p(2.776, 4.0) - 0.05) < 1e-3 + assert abs(student_t_two_tailed_p(2.228, 10.0) - 0.05) < 1e-3 + assert abs(student_t_two_tailed_p(0.0, 7.0) - 1.0) < 1e-12 + + def test_student_t_two_tailed_p_large_df_matches_normal(self): + """At huge df the t distribution converges to the normal.""" + import statistics + + from coder_eval.reports_stats import student_t_two_tailed_p + + normal_p = 2.0 * statistics.NormalDist().cdf(-1.96) + assert abs(student_t_two_tailed_p(1.96, 1e6) - normal_p) < 1e-5 + + def test_regularized_incomplete_beta_closed_forms(self): + """I_x(a,b) against the closed forms it has for b=1, a=1, and the symmetric point.""" + from coder_eval.reports_stats import regularized_incomplete_beta + + assert abs(regularized_incomplete_beta(2.0, 1.0, 0.3) - 0.3**2) < 1e-10 + assert abs(regularized_incomplete_beta(1.0, 3.0, 0.4) - (1.0 - 0.6**3)) < 1e-10 + assert abs(regularized_incomplete_beta(5.0, 5.0, 0.5) - 0.5) < 1e-10 + assert regularized_incomplete_beta(2.0, 3.0, 0.0) == 0.0 + assert regularized_incomplete_beta(2.0, 3.0, 1.0) == 1.0 + def test_mean_and_stddev(self): """Basic mean and stddev calculations.""" from coder_eval.reports_stats import mean, stddev From 684c28363fc6624493bdeb4810a08b57543dd219 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 18:27:57 -0700 Subject: [PATCH 2/6] =?UTF-8?q?feat(reports):=202/2=20=E2=80=94=20add=20a?= =?UTF-8?q?=20Paired=20Comparison=20section=20to=20experiment=20reports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both variants run the same tasks, so an unpaired Welch test carries between-task difficulty variance in its denominator and is systematically underpowered. Add `paired_t_test` (one-sample t on per-task differences, reusing the Phase 1 exact t CDF) and render a `## Paired Comparison` markdown section for any 2-variant experiment. The existing paired score comparison moves out of `## Replicate Statistics` into that section and loses its `replicate_count > 1` gate, so single-replicate A/B experiments — the common case — get it too. Pairing is over per-task mean scores: replicate slots within a task share the task effect and are not independent, so pairing them individually would understate the standard error and manufacture significance (the very failure this change set exists to fix). The task is the unit of analysis, which is also what the section's note reports. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/coder_eval/reports_experiment.py | 103 ++++++---- src/coder_eval/reports_stats.py | 18 ++ .../report_snapshots/experiment_2variant.md | 5 + .../report_snapshots/experiment_replicates.md | 2 - tests/test_experiment_reports.py | 177 +++++++++++++++++- 5 files changed, 262 insertions(+), 43 deletions(-) diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py index 9f9935b7..5f1cf90b 100644 --- a/src/coder_eval/reports_experiment.py +++ b/src/coder_eval/reports_experiment.py @@ -21,7 +21,9 @@ fmt_mean_sd, fmt_p, load_variant_eval_results, + mean, paired_bootstrap_diff_ci, + paired_t_test, stddev, welch_t_test, wilson_interval, @@ -465,8 +467,7 @@ def _win_loss_lines(result: ExperimentResult) -> list[str]: @staticmethod def _replicate_stats_lines(result: ExperimentResult) -> list[str]: """The ``## Replicate Statistics`` block: per-variant bootstrap-CI / Wilson - pass-rate table + the 2-variant paired-bootstrap comparison. Returns ``[]`` - when no variant ran more than one replicate.""" + pass-rate table. Returns ``[]`` when no variant ran more than one replicate.""" # ── Replicate Statistics (only when any variant ran >1 replicate) ── if not any(ts.replicate_count > 1 for ts in result.task_summaries): return [] @@ -488,45 +489,68 @@ def _replicate_stats_lines(result: ExperimentResult) -> list[str]: + f" | {passes}/{len(all_scores)} [{wlo:.2f}, {whi:.2f}] |" ) - # Paired comparison for 2-variant experiments - if len(result.variant_ids) == 2: - vid_a, vid_b = result.variant_ids[0], result.variant_ids[1] - per_rep_a = result.per_replicate_scores.get(vid_a, {}) - per_rep_b = result.per_replicate_scores.get(vid_b, {}) - common_tasks = sorted(set(per_rep_a) & set(per_rep_b)) - a_scores: list[float] = [] - b_scores: list[float] = [] - skipped_tasks: list[str] = [] - for task_id in common_tasks: - rep_a = per_rep_a[task_id] - rep_b = per_rep_b[task_id] - if len(rep_a) == len(rep_b): - a_scores.extend(rep_a) - b_scores.extend(rep_b) - else: - skipped_tasks.append(task_id) - diff = paired_bootstrap_diff_ci(a_scores, b_scores) - if diff is not None: - mean_diff, d_lo, d_hi = diff - d_val = cohens_d(a_scores, b_scores) - d_str = f"{d_val:.2f}" if d_val is not None else "n/a" - suffix = f" ({len(skipped_tasks)} task(s) excluded: unequal replicate counts)" if skipped_tasks else "" - lines.extend( - [ - "", - f"**Paired mean diff ({vid_a} - {vid_b})**: {mean_diff:+.3f}" - + f" [95% CI {d_lo:+.3f}, {d_hi:+.3f}], Cohen's d = {d_str}{suffix}", - ] - ) - else: - lines.extend( - [ - "", - f"*Paired statistics skipped — unequal replicate counts between {vid_a} and {vid_b}.*", - ] - ) return lines + @staticmethod + def _paired_comparison_lines(result: ExperimentResult) -> list[str]: + """The ``## Paired Comparison`` block for 2-variant experiments. + + Pairs the two variants' *per-task mean* scores, which removes between-task + difficulty variance and is therefore more powerful than the pooled Welch + test in ``## Aggregate Metrics``. The task is the unit of analysis: replicate + slots within a task share the task effect and are not independent, so + pairing them individually would understate the standard error. Unlike + ``## Replicate Statistics`` this has no replicate gate — single-replicate + experiments pair task-by-task. Returns ``[]`` when there is nothing to pair. + """ + if len(result.variant_ids) != 2: + return [] + vid_a, vid_b = result.variant_ids[0], result.variant_ids[1] + per_rep_a = result.per_replicate_scores.get(vid_a, {}) + per_rep_b = result.per_replicate_scores.get(vid_b, {}) + common_tasks = sorted(set(per_rep_a) & set(per_rep_b)) + a_scores: list[float] = [] + b_scores: list[float] = [] + skipped_tasks: list[str] = [] + for task_id in common_tasks: + rep_a = per_rep_a[task_id] + rep_b = per_rep_b[task_id] + if len(rep_a) == len(rep_b): + a_scores.append(mean(rep_a)) + b_scores.append(mean(rep_b)) + else: + skipped_tasks.append(task_id) + + if len(a_scores) < 2: + if skipped_tasks: + return [ + "", + "## Paired Comparison", + "", + f"*Paired statistics skipped — unequal replicate counts between {vid_a} and {vid_b}.*", + ] + # Nothing to pair: 0/1 common tasks, or per_replicate_scores absent (old results). + return [] + + # Equal lengths and >=2 pairs are guaranteed above, so this never returns None. + diff = paired_bootstrap_diff_ci(a_scores, b_scores) + if diff is None: # pragma: no cover - defensive + return [] + mean_diff, d_lo, d_hi = diff + d_val = cohens_d(a_scores, b_scores) + d_str = f"{d_val:.2f}" if d_val is not None else "n/a" + suffix = f" ({len(skipped_tasks)} task(s) excluded: unequal replicate counts)" if skipped_tasks else "" + return [ + "", + "## Paired Comparison", + "", + f"*Paired over the per-task mean score of {len(a_scores)} task(s) common to both variants" + + " — removes between-task variance; more powerful than the pooled Welch test above.*", + f"**Paired mean diff ({vid_a} - {vid_b})**: {mean_diff:+.3f}" + + f" [95% CI {d_lo:+.3f}, {d_hi:+.3f}], Cohen's d = {d_str}" + + f", p = {fmt_p(paired_t_test(a_scores, b_scores))}{suffix}", + ] + @staticmethod def generate_experiment_report( result: ExperimentResult, @@ -549,6 +573,7 @@ def generate_experiment_report( lines += ExperimentReportGenerator._aggregate_metrics_lines(result) lines += ExperimentReportGenerator._win_loss_lines(result) lines += ExperimentReportGenerator._replicate_stats_lines(result) + lines += ExperimentReportGenerator._paired_comparison_lines(result) return "\n".join(lines) @staticmethod diff --git a/src/coder_eval/reports_stats.py b/src/coder_eval/reports_stats.py index a2e30d5e..2916a8a3 100644 --- a/src/coder_eval/reports_stats.py +++ b/src/coder_eval/reports_stats.py @@ -214,6 +214,24 @@ def cohens_d(a: list[float], b: list[float]) -> float | None: return (sum(diffs) / len(diffs)) / s if s > 0 else None +def paired_t_test(a: list[float], b: list[float]) -> float | None: + """Two-tailed p-value from a paired t-test on (a_i - b_i), exact t distribution. + + Equivalent to a one-sample t-test of the differences against 0, df = n - 1. + Returns None if lengths differ or n < 2. + """ + if len(a) != len(b) or len(a) < 2: + return None + diffs = [ai - bi for ai, bi in zip(a, b, strict=True)] + sd = stddev(diffs) + mean_diff = sum(diffs) / len(diffs) + if sd == 0: + # All diffs identical: no difference (p=1) or a deterministic shift (p=0). + return 1.0 if mean_diff == 0 else 0.0 + t_stat = abs(mean_diff) / (sd / math.sqrt(len(diffs))) + return student_t_two_tailed_p(t_stat, len(diffs) - 1) + + # --------------------------------------------------------------------------- # Prompt config + variant-result loaders # --------------------------------------------------------------------------- diff --git a/tests/_fixtures/report_snapshots/experiment_2variant.md b/tests/_fixtures/report_snapshots/experiment_2variant.md index abc08553..53d01048 100644 --- a/tests/_fixtures/report_snapshots/experiment_2variant.md +++ b/tests/_fixtures/report_snapshots/experiment_2variant.md @@ -41,3 +41,8 @@ - **task-b**: spread=0.450, best=mutated - **task-a**: spread=0.200, best=baseline + +## Paired Comparison + +*Paired over the per-task mean score of 2 task(s) common to both variants — removes between-task variance; more powerful than the pooled Welch test above.* +**Paired mean diff (baseline - mutated)**: -0.125 [95% CI -0.450, +0.200], Cohen's d = -0.27, p = 0.766 diff --git a/tests/_fixtures/report_snapshots/experiment_replicates.md b/tests/_fixtures/report_snapshots/experiment_replicates.md index fa1c31b8..c3dade35 100644 --- a/tests/_fixtures/report_snapshots/experiment_replicates.md +++ b/tests/_fixtures/report_snapshots/experiment_replicates.md @@ -39,5 +39,3 @@ |---------|-----------------|------------|--------|------------------------| | a | 3 | 0.900 | [0.850, 0.933] | 2/3 [0.21, 0.94] | | b | 3 | 0.650 | [0.600, 0.683] | 0/3 [0.00, 0.56] | - -**Paired mean diff (a - b)**: +0.250 [95% CI +0.200, +0.300], Cohen's d = 5.00 diff --git a/tests/test_experiment_reports.py b/tests/test_experiment_reports.py index e7892ed7..0a580aaf 100644 --- a/tests/test_experiment_reports.py +++ b/tests/test_experiment_reports.py @@ -800,6 +800,34 @@ def test_regularized_incomplete_beta_closed_forms(self): assert regularized_incomplete_beta(2.0, 3.0, 0.0) == 0.0 assert regularized_incomplete_beta(2.0, 3.0, 1.0) == 1.0 + def test_paired_t_test_exact_reference_value(self): + """Exact paired-t p-value, cross-checked against scipy ttest_rel.""" + from coder_eval.reports_stats import paired_t_test, welch_t_test + + a = [0.9, 0.5, 0.8, 0.7, 0.95] + b = [0.7, 0.55, 0.6, 0.72, 0.8] + p_paired = paired_t_test(a, b) + assert p_paired is not None + assert abs(p_paired - 0.15273) < 5e-4 + # Pairing removes between-task variance, so it is strictly more powerful. + p_welch = welch_t_test(a, b) + assert p_welch is not None + assert p_paired < p_welch + + def test_paired_t_test_constant_shift_and_identical(self): + """Zero-sd differences: deterministic shift ⇒ 0.0, identical lists ⇒ 1.0.""" + from coder_eval.reports_stats import paired_t_test + + assert paired_t_test([1.0, 2.0, 3.0, 4.0, 5.0], [2.0, 3.0, 4.0, 5.0, 6.0]) == 0.0 + assert paired_t_test([1.0, 2.0, 3.0], [1.0, 2.0, 3.0]) == 1.0 + + def test_paired_t_test_invalid_input(self): + """Length mismatch or fewer than 2 pairs returns None.""" + from coder_eval.reports_stats import paired_t_test + + assert paired_t_test([1.0, 2.0], [1.0]) is None + assert paired_t_test([1.0], [2.0]) is None + def test_mean_and_stddev(self): """Basic mean and stddev calculations.""" from coder_eval.reports_stats import mean, stddev @@ -1119,9 +1147,10 @@ def test_section_contains_per_variant_ci_columns(self): assert "Pass-rate" in md def test_paired_diff_line_for_two_variants_equal_counts(self): + # Two tasks: the paired section pairs their per-task means (n = 2 tasks). per_rep = { - "a": {"task-1": [0.9, 0.85, 0.95]}, - "b": {"task-1": [0.6, 0.65, 0.7]}, + "a": {"task-1": [0.9, 0.85, 0.95], "task-2": [0.7, 0.75, 0.8]}, + "b": {"task-1": [0.6, 0.65, 0.7], "task-2": [0.5, 0.55, 0.6]}, } result = self._make_result(replicate_count=3, per_replicate_scores=per_rep) md = ExperimentReportGenerator.generate_experiment_report(result) @@ -1169,6 +1198,146 @@ def test_variant_report_no_ci_when_replicate_count_is_one(self): assert "Score 95% CI" not in md +class TestPairedComparisonSection: + """Tests for the ``## Paired Comparison`` markdown section.""" + + @staticmethod + def _make_result( + variant_ids: list[str], + per_replicate_scores: dict[str, dict[str, list[float]]], + ) -> ExperimentResult: + task_ids = sorted({tid for per_task in per_replicate_scores.values() for tid in per_task}) + return ExperimentResult( + experiment_id="exp", + description="d", + variant_ids=variant_ids, + task_summaries=[ + TaskExperimentSummary( + task_id=task_id, + variant_results=[ + VariantResult( + variant_id=vid, + task_id=task_id, + weighted_score=per_replicate_scores.get(vid, {}).get(task_id, [0.0])[0], + final_status="SUCCESS", + duration_seconds=1.0, + ) + for vid in variant_ids + ], + best_variant=variant_ids[0], + score_spread=0.1, + ) + for task_id in task_ids + ], + variant_aggregates={ + vid: VariantAggregate( + variant_id=vid, + tasks_run=len(task_ids), + tasks_succeeded=len(task_ids), + tasks_failed=0, + tasks_error=0, + average_score=0.5, + average_duration=1.0, + ) + for vid in variant_ids + }, + total_duration_seconds=10.0, + per_replicate_scores=per_replicate_scores, + ) + + def test_paired_comparison_section_two_variants_single_replicate(self): + """A single-replicate 2-variant experiment gets the paired section (no replicate gate).""" + result = self._make_result( + ["a", "b"], + { + "a": {"t1": [0.9], "t2": [0.5], "t3": [0.8]}, + "b": {"t1": [0.7], "t2": [0.55], "t3": [0.6]}, + }, + ) + md = ExperimentReportGenerator.generate_experiment_report(result) + assert "## Paired Comparison" in md + assert "**Paired mean diff (a - b)**" in md + assert ", p = " in md + assert "Cohen's d" in md + # Single replicate ⇒ no Replicate Statistics section, but paired still renders. + assert "## Replicate Statistics" not in md + + def test_paired_comparison_section_absent(self): + """3 variants, or an empty per_replicate_scores, render no paired section.""" + three = self._make_result( + ["a", "b", "c"], + {"a": {"t1": [0.9]}, "b": {"t1": [0.7]}, "c": {"t1": [0.6]}}, + ) + assert "## Paired Comparison" not in ExperimentReportGenerator.generate_experiment_report(three) + + empty = self._make_result(["a", "b"], {}) + empty.task_summaries = [ + TaskExperimentSummary( + task_id="t1", + variant_results=[ + VariantResult( + variant_id=vid, + task_id="t1", + weighted_score=0.5, + final_status="SUCCESS", + duration_seconds=1.0, + ) + for vid in ("a", "b") + ], + best_variant="a", + score_spread=0.0, + ) + ] + assert "## Paired Comparison" not in ExperimentReportGenerator.generate_experiment_report(empty) + + def test_paired_comparison_averages_replicates_per_task(self): + """Replicates are averaged per task, so n is the task count — not the slot count.""" + result = self._make_result( + ["a", "b"], + { + "a": {"t1": [0.8, 1.0], "t2": [0.4, 0.6]}, + "b": {"t1": [0.5, 0.7], "t2": [0.2, 0.4]}, + }, + ) + md = ExperimentReportGenerator.generate_experiment_report(result) + assert "per-task mean score of 2 task(s) common to both variants" in md + # Task means: a = [0.9, 0.5], b = [0.6, 0.3] ⇒ diffs [+0.3, +0.2], mean +0.250. + assert "**Paired mean diff (a - b)**: +0.250" in md + + def test_paired_comparison_single_common_task(self): + """One common task ⇒ a single pair ⇒ no section (a paired test needs n >= 2 tasks).""" + result = self._make_result( + ["a", "b"], + {"a": {"t1": [0.9, 0.85, 0.95]}, "b": {"t1": [0.6, 0.65, 0.7]}}, + ) + assert "## Paired Comparison" not in ExperimentReportGenerator.generate_experiment_report(result) + + def test_paired_comparison_partial_skip(self): + """Some tasks skipped for unequal replicate counts, >=2 pairs left ⇒ suffix + reduced count.""" + result = self._make_result( + ["a", "b"], + { + "a": {"t1": [0.5, 0.6], "t2": [0.4], "t3": [0.8]}, + "b": {"t1": [0.5], "t2": [0.45], "t3": [0.7]}, + }, + ) + md = ExperimentReportGenerator.generate_experiment_report(result) + assert "## Paired Comparison" in md + assert "per-task mean score of 2 task(s) common to both variants" in md + assert "(1 task(s) excluded: unequal replicate counts)" in md + + def test_paired_comparison_all_tasks_skipped(self): + """Every common task has unequal replicate counts ⇒ section renders the skip message only.""" + result = self._make_result( + ["a", "b"], + {"a": {"t1": [0.5, 0.6]}, "b": {"t1": [0.5]}}, + ) + md = ExperimentReportGenerator.generate_experiment_report(result) + assert "## Paired Comparison" in md + assert "*Paired statistics skipped — unequal replicate counts between a and b.*" in md + assert "Paired mean diff" not in md + + class TestExperimentReportSnapshots: """Byte-identical characterization snapshots for generate_experiment_report — the safety net for its decomposition. Output is deterministic: bootstrap_mean_ci / @@ -1269,6 +1438,10 @@ def test_experiment_report_snapshot_2variant(self): ), }, total_duration_seconds=180.0, + per_replicate_scores={ + "baseline": {"task-a": [0.9], "task-b": [0.5]}, + "mutated": {"task-a": [0.7], "task-b": [0.95]}, + }, ) md = ExperimentReportGenerator.generate_experiment_report(result, experiment=experiment) assert_matches_snapshot(md, "experiment_2variant.md") From 67bcd94e2f584da28dc098d6985d6b0f8d212ba5 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 18:37:57 -0700 Subject: [PATCH 3/6] fix: code review fixes for welch-t-test-exact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Guard the t-test helpers against non-finite input (rubric item 15). student_t_two_tailed_p fails closed to p=1.0; welch_t_test and paired_t_test return None, which renders as "—" rather than a fabricated p-value. - Drop the "more powerful than the pooled Welch test" claim from the Paired Comparison note. Pairing loses degrees of freedom, so it is not unconditionally sharper — the 2-variant snapshot is itself a counter-example (Welch p=0.658 vs paired p=0.766). The note is now descriptive: pairing cancels between-task difficulty. - Stop excluding tasks whose two variants ran unequal replicate counts. That constraint existed to zip replicate slots; pairing per-task means has no such requirement, so the exclusion only discarded real data (and biased the comparison toward fully-completed tasks). - A 2-variant experiment with exactly one common task now says why it has no paired result instead of silently rendering nothing. - Note in-code that the bootstrap CI and the exact-t p-value are separate inference models and can disagree at small task counts. - Warn when the incomplete-beta continued fraction fails to converge rather than returning a partial value silently. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/coder_eval/reports_experiment.py | 67 ++++++++----------- src/coder_eval/reports_stats.py | 19 ++++-- .../report_snapshots/experiment_2variant.md | 2 +- .../report_snapshots/experiment_replicates.md | 4 ++ tests/test_experiment_reports.py | 59 ++++++++++------ 5 files changed, 87 insertions(+), 64 deletions(-) diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py index 5f1cf90b..e8d2986c 100644 --- a/src/coder_eval/reports_experiment.py +++ b/src/coder_eval/reports_experiment.py @@ -495,60 +495,51 @@ def _replicate_stats_lines(result: ExperimentResult) -> list[str]: def _paired_comparison_lines(result: ExperimentResult) -> list[str]: """The ``## Paired Comparison`` block for 2-variant experiments. - Pairs the two variants' *per-task mean* scores, which removes between-task - difficulty variance and is therefore more powerful than the pooled Welch - test in ``## Aggregate Metrics``. The task is the unit of analysis: replicate - slots within a task share the task effect and are not independent, so - pairing them individually would understate the standard error. Unlike - ``## Replicate Statistics`` this has no replicate gate — single-replicate - experiments pair task-by-task. Returns ``[]`` when there is nothing to pair. + Pairs the two variants' *per-task mean* scores. The task is the unit of + analysis: replicate slots within a task share the task effect and are not + independent, so pairing them individually would understate the standard + error. Unlike ``## Replicate Statistics`` there is no replicate gate — + single-replicate experiments pair task-by-task. Returns ``[]`` only when + the two variants have no scored task in common; when they have exactly + one, the section explains why no paired result is shown. """ if len(result.variant_ids) != 2: return [] vid_a, vid_b = result.variant_ids[0], result.variant_ids[1] per_rep_a = result.per_replicate_scores.get(vid_a, {}) per_rep_b = result.per_replicate_scores.get(vid_b, {}) - common_tasks = sorted(set(per_rep_a) & set(per_rep_b)) - a_scores: list[float] = [] - b_scores: list[float] = [] - skipped_tasks: list[str] = [] - for task_id in common_tasks: - rep_a = per_rep_a[task_id] - rep_b = per_rep_b[task_id] - if len(rep_a) == len(rep_b): - a_scores.append(mean(rep_a)) - b_scores.append(mean(rep_b)) - else: - skipped_tasks.append(task_id) - - if len(a_scores) < 2: - if skipped_tasks: - return [ - "", - "## Paired Comparison", - "", - f"*Paired statistics skipped — unequal replicate counts between {vid_a} and {vid_b}.*", - ] - # Nothing to pair: 0/1 common tasks, or per_replicate_scores absent (old results). + # Replicate counts need not match: a task's mean score is a well-defined + # pair member either way, so no task is excluded for having fewer runs. + common_tasks = sorted(t for t in set(per_rep_a) & set(per_rep_b) if per_rep_a[t] and per_rep_b[t]) + if not common_tasks: + # Nothing to pair: no shared task, or per_replicate_scores absent (old results). return [] + header = ["", "## Paired Comparison", ""] + if len(common_tasks) < 2: + return [ + *header, + f"*A paired comparison needs at least 2 tasks common to {vid_a} and {vid_b}; found 1.*", + ] + + a_scores = [mean(per_rep_a[task_id]) for task_id in common_tasks] + b_scores = [mean(per_rep_b[task_id]) for task_id in common_tasks] + # The interval is a percentile bootstrap while the p-value is an exact paired + # t — separate inference models, so they can disagree at small task counts. # Equal lengths and >=2 pairs are guaranteed above, so this never returns None. diff = paired_bootstrap_diff_ci(a_scores, b_scores) if diff is None: # pragma: no cover - defensive return [] - mean_diff, d_lo, d_hi = diff + mean_diff, ci_lo, ci_hi = diff d_val = cohens_d(a_scores, b_scores) d_str = f"{d_val:.2f}" if d_val is not None else "n/a" - suffix = f" ({len(skipped_tasks)} task(s) excluded: unequal replicate counts)" if skipped_tasks else "" return [ - "", - "## Paired Comparison", - "", - f"*Paired over the per-task mean score of {len(a_scores)} task(s) common to both variants" - + " — removes between-task variance; more powerful than the pooled Welch test above.*", + *header, + f"*Paired over the per-task mean score of {len(common_tasks)} task(s) common to both variants" + + " — pairing cancels between-task difficulty, which the pooled Welch test above cannot.*", f"**Paired mean diff ({vid_a} - {vid_b})**: {mean_diff:+.3f}" - + f" [95% CI {d_lo:+.3f}, {d_hi:+.3f}], Cohen's d = {d_str}" - + f", p = {fmt_p(paired_t_test(a_scores, b_scores))}{suffix}", + + f" [95% CI {ci_lo:+.3f}, {ci_hi:+.3f}], Cohen's d = {d_str}" + + f", p = {fmt_p(paired_t_test(a_scores, b_scores))}", ] @staticmethod diff --git a/src/coder_eval/reports_stats.py b/src/coder_eval/reports_stats.py index 2916a8a3..f27a29e0 100644 --- a/src/coder_eval/reports_stats.py +++ b/src/coder_eval/reports_stats.py @@ -70,7 +70,8 @@ def _betacf(a: float, b: float, x: float) -> float: delta = d * c h *= delta if abs(delta - 1.0) < eps: - break + return h + logger.warning("Incomplete beta continued fraction did not converge for a=%r, b=%r, x=%r", a, b, x) return h @@ -89,8 +90,11 @@ def regularized_incomplete_beta(a: float, b: float, x: float) -> float: def student_t_two_tailed_p(t_stat: float, df: float) -> float: - """Exact two-tailed p-value for Student's t: P(|T| >= |t|) = I_x(df/2, 1/2), x = df/(df + t^2).""" - if df <= 0: + """Exact two-tailed p-value for Student's t: P(|T| >= |t|) = I_x(df/2, 1/2), x = df/(df + t^2). + + Non-finite inputs fail closed to 1.0 — garbage must never read as significant. + """ + if not math.isfinite(t_stat) or not math.isfinite(df) or df <= 0: return 1.0 x = df / (df + t_stat * t_stat) return regularized_incomplete_beta(df / 2.0, 0.5, x) @@ -101,11 +105,14 @@ def welch_t_test(a: list[float], b: list[float]) -> float | None: Degrees of freedom via Welch-Satterthwaite; the t CDF is evaluated exactly through the regularized incomplete beta (stdlib only, no scipy). Returns - None if either group has fewer than 2 observations. + None if either group has fewer than 2 observations, or holds a non-finite + value (rendered as "—" rather than a fabricated p-value). """ n_a, n_b = len(a), len(b) if n_a < 2 or n_b < 2: return None + if not all(math.isfinite(v) for v in (*a, *b)): + return None mean_a, mean_b = _stats.mean(a), _stats.mean(b) var_a = _stats.variance(a) @@ -218,10 +225,12 @@ def paired_t_test(a: list[float], b: list[float]) -> float | None: """Two-tailed p-value from a paired t-test on (a_i - b_i), exact t distribution. Equivalent to a one-sample t-test of the differences against 0, df = n - 1. - Returns None if lengths differ or n < 2. + Returns None if lengths differ, n < 2, or any value is non-finite. """ if len(a) != len(b) or len(a) < 2: return None + if not all(math.isfinite(v) for v in (*a, *b)): + return None diffs = [ai - bi for ai, bi in zip(a, b, strict=True)] sd = stddev(diffs) mean_diff = sum(diffs) / len(diffs) diff --git a/tests/_fixtures/report_snapshots/experiment_2variant.md b/tests/_fixtures/report_snapshots/experiment_2variant.md index 53d01048..18830bac 100644 --- a/tests/_fixtures/report_snapshots/experiment_2variant.md +++ b/tests/_fixtures/report_snapshots/experiment_2variant.md @@ -44,5 +44,5 @@ ## Paired Comparison -*Paired over the per-task mean score of 2 task(s) common to both variants — removes between-task variance; more powerful than the pooled Welch test above.* +*Paired over the per-task mean score of 2 task(s) common to both variants — pairing cancels between-task difficulty, which the pooled Welch test above cannot.* **Paired mean diff (baseline - mutated)**: -0.125 [95% CI -0.450, +0.200], Cohen's d = -0.27, p = 0.766 diff --git a/tests/_fixtures/report_snapshots/experiment_replicates.md b/tests/_fixtures/report_snapshots/experiment_replicates.md index c3dade35..dfcd8988 100644 --- a/tests/_fixtures/report_snapshots/experiment_replicates.md +++ b/tests/_fixtures/report_snapshots/experiment_replicates.md @@ -39,3 +39,7 @@ |---------|-----------------|------------|--------|------------------------| | a | 3 | 0.900 | [0.850, 0.933] | 2/3 [0.21, 0.94] | | b | 3 | 0.650 | [0.600, 0.683] | 0/3 [0.00, 0.56] | + +## Paired Comparison + +*A paired comparison needs at least 2 tasks common to a and b; found 1.* diff --git a/tests/test_experiment_reports.py b/tests/test_experiment_reports.py index 0a580aaf..0f7da38e 100644 --- a/tests/test_experiment_reports.py +++ b/tests/test_experiment_reports.py @@ -780,6 +780,8 @@ def test_student_t_two_tailed_p_critical_values(self): assert abs(student_t_two_tailed_p(2.776, 4.0) - 0.05) < 1e-3 assert abs(student_t_two_tailed_p(2.228, 10.0) - 0.05) < 1e-3 assert abs(student_t_two_tailed_p(0.0, 7.0) - 1.0) < 1e-12 + # Degenerate df is not reachable through the t-tests, but the helper stays total. + assert student_t_two_tailed_p(1.0, 0.0) == 1.0 def test_student_t_two_tailed_p_large_df_matches_normal(self): """At huge df the t distribution converges to the normal.""" @@ -809,7 +811,8 @@ def test_paired_t_test_exact_reference_value(self): p_paired = paired_t_test(a, b) assert p_paired is not None assert abs(p_paired - 0.15273) < 5e-4 - # Pairing removes between-task variance, so it is strictly more powerful. + # On positively correlated pairs like these, pairing is the sharper test — + # but that is a property of this data, not a guarantee (it loses df). p_welch = welch_t_test(a, b) assert p_welch is not None assert p_paired < p_welch @@ -828,6 +831,19 @@ def test_paired_t_test_invalid_input(self): assert paired_t_test([1.0, 2.0], [1.0]) is None assert paired_t_test([1.0], [2.0]) is None + def test_non_finite_inputs_never_report_significance(self): + """NaN/inf must not produce a fabricated p-value — fail closed, never significant.""" + from coder_eval.reports_stats import fmt_p, paired_t_test, student_t_two_tailed_p, welch_t_test + + nan, inf = float("nan"), float("inf") + assert student_t_two_tailed_p(nan, 4.0) == 1.0 + assert student_t_two_tailed_p(2.5, nan) == 1.0 + assert student_t_two_tailed_p(inf, 4.0) == 1.0 + # The list-taking helpers can say "no result", which renders as an em dash. + assert welch_t_test([1.0, nan], [2.0, 3.0]) is None + assert paired_t_test([1.0, nan], [2.0, 3.0]) is None + assert fmt_p(welch_t_test([1.0, inf], [2.0, 3.0])) == "—" + def test_mean_and_stddev(self): """Basic mean and stddev calculations.""" from coder_eval.reports_stats import mean, stddev @@ -1157,16 +1173,17 @@ def test_paired_diff_line_for_two_variants_equal_counts(self): assert "Paired mean diff" in md assert "Cohen's d" in md - def test_paired_diff_skipped_when_unequal_counts_across_tasks(self): - # Variant a has 3 replicates for task-1, variant b has 5 → paired skipped + def test_paired_diff_needs_two_common_tasks(self): + # Unequal replicate counts no longer exclude a task, but one task is still + # a single pair — too few for a paired comparison. per_rep = { "a": {"task-1": [0.9, 0.85, 0.95]}, "b": {"task-1": [0.6, 0.65, 0.7, 0.75, 0.8]}, } result = self._make_result(replicate_count=3, per_replicate_scores=per_rep) md = ExperimentReportGenerator.generate_experiment_report(result) - assert "Paired statistics skipped" in md - assert "unequal replicate counts" in md + assert "needs at least 2 tasks" in md + assert "Paired mean diff" not in md def test_no_paired_diff_for_single_variant(self): per_rep = {"only": {"task-1": [0.7, 0.8, 0.9]}} @@ -1218,7 +1235,7 @@ def _make_result( VariantResult( variant_id=vid, task_id=task_id, - weighted_score=per_replicate_scores.get(vid, {}).get(task_id, [0.0])[0], + weighted_score=next(iter(per_replicate_scores.get(vid, {}).get(task_id, [])), 0.0), final_status="SUCCESS", duration_seconds=1.0, ) @@ -1305,37 +1322,39 @@ def test_paired_comparison_averages_replicates_per_task(self): assert "**Paired mean diff (a - b)**: +0.250" in md def test_paired_comparison_single_common_task(self): - """One common task ⇒ a single pair ⇒ no section (a paired test needs n >= 2 tasks).""" + """One common task ⇒ the section explains why there is no paired result.""" result = self._make_result( ["a", "b"], {"a": {"t1": [0.9, 0.85, 0.95]}, "b": {"t1": [0.6, 0.65, 0.7]}}, ) - assert "## Paired Comparison" not in ExperimentReportGenerator.generate_experiment_report(result) + md = ExperimentReportGenerator.generate_experiment_report(result) + assert "## Paired Comparison" in md + assert "*A paired comparison needs at least 2 tasks common to a and b; found 1.*" in md + assert "Paired mean diff" not in md - def test_paired_comparison_partial_skip(self): - """Some tasks skipped for unequal replicate counts, >=2 pairs left ⇒ suffix + reduced count.""" + def test_paired_comparison_keeps_tasks_with_unequal_replicate_counts(self): + """Pairing per-task means needs no equal replicate counts — no task is dropped.""" result = self._make_result( ["a", "b"], { - "a": {"t1": [0.5, 0.6], "t2": [0.4], "t3": [0.8]}, + "a": {"t1": [0.5, 0.7], "t2": [0.4], "t3": [0.8]}, "b": {"t1": [0.5], "t2": [0.45], "t3": [0.7]}, }, ) md = ExperimentReportGenerator.generate_experiment_report(result) - assert "## Paired Comparison" in md - assert "per-task mean score of 2 task(s) common to both variants" in md - assert "(1 task(s) excluded: unequal replicate counts)" in md + assert "per-task mean score of 3 task(s) common to both variants" in md + assert "excluded" not in md + # Task means: a = [0.6, 0.4, 0.8], b = [0.5, 0.45, 0.7] ⇒ diffs [+0.1, -0.05, +0.1]. + assert "**Paired mean diff (a - b)**: +0.050" in md - def test_paired_comparison_all_tasks_skipped(self): - """Every common task has unequal replicate counts ⇒ section renders the skip message only.""" + def test_paired_comparison_ignores_tasks_with_no_scores(self): + """A task with an empty replicate list on either side is not a pair.""" result = self._make_result( ["a", "b"], - {"a": {"t1": [0.5, 0.6]}, "b": {"t1": [0.5]}}, + {"a": {"t1": [0.9], "t2": [0.5], "t3": []}, "b": {"t1": [0.7], "t2": [0.6], "t3": [0.4]}}, ) md = ExperimentReportGenerator.generate_experiment_report(result) - assert "## Paired Comparison" in md - assert "*Paired statistics skipped — unequal replicate counts between a and b.*" in md - assert "Paired mean diff" not in md + assert "per-task mean score of 2 task(s) common to both variants" in md class TestExperimentReportSnapshots: From faabc5e1a1d70214332ddae20adaaa7d28a0431e Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 18:38:09 -0700 Subject: [PATCH 4/6] chore(harness): defer two guards from the welch-t-test-exact run Neither is a ~30-minute guard: the non-finite-input rule needs dataflow analysis rather than the syntactic matching the CExxx rules do, and the bootstrap percentile indexing is latent pre-existing code with no caller that can currently reach it. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/harness-candidates.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index cbfb4970..c0f4c7cd 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -37,3 +37,19 @@ Deferred lint/test guardrails surfaced during reviews. Promote to a `CExxx` rule caught them. The cleanup plan explicitly deferred this as YAGNI for the one-time purge, but any future doc rename/deletion re-opens the same blind spot — caught in the 2026-07-03 open-source-docs-cleanup implementation run. + +- [ ] Non-finite (NaN/inf) guards on numeric report helpers — `reports_stats.py`'s + t-test entry points now guard with `math.isfinite`, but nothing mechanically + enforces it for the next statistical helper added to that module. Review + criterion 15 states the rule; a CExxx rule would need to recognise "function + that feeds a value into `math.log`/`lgamma`/a division without a finiteness + check", which is a dataflow question rather than the syntactic pattern the + existing CE001–CE025 rules match — caught in the 2026-07-21 welch-t-test-exact + implementation run. +- [ ] Percentile indices in `bootstrap_mean_ci` (`reports_stats.py`) are unguarded + against `confidence` outside (0,1) or tiny `n_resamples`, so an out-of-range + `confidence` would raise `IndexError` mid-report. Unreachable today — no caller + passes a non-default `confidence`/`n_resamples` — so it is latent rather than + live, and it is pre-existing code untouched by this change. Guard it (or drop + the unused parameters) if a caller ever starts passing them — flagged by an + external reviewer during the 2026-07-21 welch-t-test-exact final review. From a4f87dc2f7304a882f5b0b387775ebb11a7173a7 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 18:43:02 -0700 Subject: [PATCH 5/6] fix(reports): validate confidence and n_resamples in bootstrap_mean_ci Out-of-domain values previously surfaced as a bare IndexError from the percentile lookup. Refuse them explicitly instead: clamping into range would quietly return an interval of the wrong width, which is worse than failing. No caller passes either parameter today, so nothing changes for existing report paths. Closes the harness candidate deferred earlier this run. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/harness-candidates.md | 7 ------- src/coder_eval/reports_stats.py | 8 ++++++++ tests/test_replicate_stats.py | 12 ++++++++++++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index c0f4c7cd..2ad5eb55 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -46,10 +46,3 @@ Deferred lint/test guardrails surfaced during reviews. Promote to a `CExxx` rule check", which is a dataflow question rather than the syntactic pattern the existing CE001–CE025 rules match — caught in the 2026-07-21 welch-t-test-exact implementation run. -- [ ] Percentile indices in `bootstrap_mean_ci` (`reports_stats.py`) are unguarded - against `confidence` outside (0,1) or tiny `n_resamples`, so an out-of-range - `confidence` would raise `IndexError` mid-report. Unreachable today — no caller - passes a non-default `confidence`/`n_resamples` — so it is latent rather than - live, and it is pre-existing code untouched by this change. Guard it (or drop - the unused parameters) if a caller ever starts passing them — flagged by an - external reviewer during the 2026-07-21 welch-t-test-exact final review. diff --git a/src/coder_eval/reports_stats.py b/src/coder_eval/reports_stats.py index f27a29e0..16dbcf01 100644 --- a/src/coder_eval/reports_stats.py +++ b/src/coder_eval/reports_stats.py @@ -165,7 +165,15 @@ def bootstrap_mean_ci( Returns (mean, ci_low, ci_high). When ``len(values) < 2``, returns (values[0], values[0], values[0]) or (0, 0, 0) for empty input. Uses ``random.Random(seed)`` for determinism. + + Raises ValueError for a ``confidence`` outside (0, 1) or a non-positive + ``n_resamples`` — clamping those would quietly return an interval of the + wrong width, which is worse than refusing. """ + if not 0.0 < confidence < 1.0: + raise ValueError(f"confidence must be in (0, 1), got {confidence!r}") + if n_resamples < 1: + raise ValueError(f"n_resamples must be >= 1, got {n_resamples!r}") if not values: return (0.0, 0.0, 0.0) m = sum(values) / len(values) diff --git a/tests/test_replicate_stats.py b/tests/test_replicate_stats.py index 2d1442ee..850996e8 100644 --- a/tests/test_replicate_stats.py +++ b/tests/test_replicate_stats.py @@ -1,5 +1,7 @@ """Unit tests for replicate statistics helpers in reports_stats.""" +import pytest + from coder_eval.reports_stats import ( bootstrap_mean_ci, cohens_d, @@ -15,6 +17,16 @@ def test_empty_returns_triple_zero(self): def test_single_value_returns_triple(self): assert bootstrap_mean_ci([0.7]) == (0.7, 0.7, 0.7) + @pytest.mark.parametrize("confidence", [-1.0, 0.0, 1.0, 1.1]) + def test_confidence_outside_unit_interval_raises(self, confidence): + """Out-of-domain confidence must refuse, not return a wrong-width interval.""" + with pytest.raises(ValueError, match="confidence must be in"): + bootstrap_mean_ci([0.1, 0.5, 0.9], confidence=confidence) + + def test_non_positive_n_resamples_raises(self): + with pytest.raises(ValueError, match="n_resamples must be"): + bootstrap_mean_ci([0.1, 0.5, 0.9], n_resamples=0) + def test_mean_is_correct(self): m, _lo, _hi = bootstrap_mean_ci([0.0, 1.0]) assert abs(m - 0.5) < 1e-9 From c31205221328f8c0971b38714e34a86e1b859570 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 18:54:13 -0700 Subject: [PATCH 6/6] fix(reports): one source of truth for variant series and paired stats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three related report-correctness fixes: - De-duplicate _collect_variant_series. The markdown and HTML reporters each had a copy and they had drifted: the markdown one divides duration_seconds by replicate_count (correct — VariantResult sums it across replicates) while the HTML one used it raw, so HTML showed replicate-inflated durations. Both now call one collect_variant_series in reports_stats. - Replace the paired percentile bootstrap with a Student-t interval. The section printed a bootstrap CI beside an exact-t p-value — two inference models on the same quantity, which could disagree, and the bootstrap was degenerate over the 2-3 task means it now resamples. The CI derives from the same distribution as the p-value, so the two always agree about whether 0 is excluded. New student_t_critical (bisection on the existing t CDF) and paired_t_ci; paired_bootstrap_diff_ci is gone. - Render the paired section in the HTML report too. Both reporters now render one shared reports_stats.paired_comparison, so they cannot disagree about the numbers. Also adds the non-finite harness deferred earlier: it enumerates the module rather than listing helpers by hand, so a helper added later is covered automatically. It immediately caught regularized_incomplete_beta returning NaN for NaN input, now an explicit ValueError. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/harness-candidates.md | 9 - src/coder_eval/reports_experiment.py | 109 +++-------- src/coder_eval/reports_html.py | 81 ++++++--- src/coder_eval/reports_stats.py | 171 ++++++++++++++++-- .../report_snapshots/experiment_2variant.md | 2 +- tests/test_experiment_reports.py | 123 ++++++++++++- tests/test_replicate_stats.py | 92 +++++----- tests/test_reports_stats_nonfinite.py | 73 ++++++++ 8 files changed, 480 insertions(+), 180 deletions(-) create mode 100644 tests/test_reports_stats_nonfinite.py diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 2ad5eb55..cbfb4970 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -37,12 +37,3 @@ Deferred lint/test guardrails surfaced during reviews. Promote to a `CExxx` rule caught them. The cleanup plan explicitly deferred this as YAGNI for the one-time purge, but any future doc rename/deletion re-opens the same blind spot — caught in the 2026-07-03 open-source-docs-cleanup implementation run. - -- [ ] Non-finite (NaN/inf) guards on numeric report helpers — `reports_stats.py`'s - t-test entry points now guard with `math.isfinite`, but nothing mechanically - enforces it for the next statistical helper added to that module. Review - criterion 15 states the rule; a CExxx rule would need to recognise "function - that feeds a value into `math.log`/`lgamma`/a division without a finiteness - check", which is a dataflow question rather than the syntactic pattern the - existing CE001–CE025 rules match — caught in the 2026-07-21 welch-t-test-exact - implementation run. diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py index e8d2986c..5bfc3cce 100644 --- a/src/coder_eval/reports_experiment.py +++ b/src/coder_eval/reports_experiment.py @@ -15,15 +15,14 @@ from coder_eval.path_utils import replicate_subdir_name from coder_eval.reports import resolve_agent_settings from coder_eval.reports_stats import ( + VariantSeries, bootstrap_mean_ci, - cohens_d, + collect_variant_series, describe_prompt_config, fmt_mean_sd, fmt_p, load_variant_eval_results, - mean, - paired_bootstrap_diff_ci, - paired_t_test, + paired_comparison, stddev, welch_t_test, wilson_interval, @@ -218,27 +217,6 @@ def _prompt_config_lines(result: ExperimentResult, experiment: ExperimentDefinit lines.append(f"- **{vid}**: {desc}") return lines - @staticmethod - def _collect_variant_series( - result: ExperimentResult, - ) -> tuple[dict[str, list[float]], dict[str, list[float]], dict[str, list[float]], dict[str, list[float]]]: - """Per-variant (scores, durations, tokens, assistant-turns) series collected - across all task summaries — the raw inputs to the Aggregate Metrics stat rows.""" - variant_scores: dict[str, list[float]] = {vid: [] for vid in result.variant_ids} - variant_durations: dict[str, list[float]] = {vid: [] for vid in result.variant_ids} - variant_tokens: dict[str, list[float]] = {vid: [] for vid in result.variant_ids} - variant_asst_turns: dict[str, list[float]] = {vid: [] for vid in result.variant_ids} - - for ts in result.task_summaries: - for vr in ts.variant_results: - variant_scores[vr.variant_id].append(vr.weighted_score) - variant_durations[vr.variant_id].append(vr.duration_seconds / vr.replicate_count) - if vr.total_tokens is not None: - variant_tokens[vr.variant_id].append(float(vr.total_tokens)) - if vr.total_assistant_turns is not None: - variant_asst_turns[vr.variant_id].append(float(vr.total_assistant_turns)) - return variant_scores, variant_durations, variant_tokens, variant_asst_turns - @staticmethod def _aggregate_count_rows(result: ExperimentResult, show_p_values: bool) -> list[str]: """The integer-aggregate rows of the Aggregate Metrics table: Tasks Run, @@ -314,10 +292,7 @@ def _aggregate_count_rows(result: ExperimentResult, show_p_values: bool) -> list @staticmethod def _aggregate_stat_rows( result: ExperimentResult, - variant_scores: dict[str, list[float]], - variant_durations: dict[str, list[float]], - variant_tokens: dict[str, list[float]], - variant_asst_turns: dict[str, list[float]], + series: dict[str, VariantSeries], show_p_values: bool, vid_a: str, vid_b: str, @@ -330,38 +305,38 @@ def _aggregate_stat_rows( # Row: Score (mean ± stddev, p-value) row = "| Score" for vid in result.variant_ids: - row += f" | {fmt_mean_sd(variant_scores[vid])}" + row += f" | {fmt_mean_sd(series[vid].scores)}" if show_p_values: - p = welch_t_test(variant_scores[vid_a], variant_scores[vid_b]) + p = welch_t_test(series[vid_a].scores, series[vid_b].scores) row += f" | {fmt_p(p)}" lines.append(row + " |") # Row: Duration row = "| Avg Duration (s)" for vid in result.variant_ids: - row += f" | {fmt_mean_sd(variant_durations[vid], '.1f')}" + row += f" | {fmt_mean_sd(series[vid].durations, '.1f')}" if show_p_values: - p = welch_t_test(variant_durations[vid_a], variant_durations[vid_b]) + p = welch_t_test(series[vid_a].durations, series[vid_b].durations) row += f" | {fmt_p(p)}" lines.append(row + " |") # Row: Assistant Turns (if data available) - if any(variant_asst_turns[vid] for vid in result.variant_ids): + if any(series[vid].asst_turns for vid in result.variant_ids): row = "| Assistant Turns" for vid in result.variant_ids: - row += f" | {fmt_mean_sd(variant_asst_turns[vid], '.1f')}" + row += f" | {fmt_mean_sd(series[vid].asst_turns, '.1f')}" if show_p_values: - p = welch_t_test(variant_asst_turns[vid_a], variant_asst_turns[vid_b]) + p = welch_t_test(series[vid_a].asst_turns, series[vid_b].asst_turns) row += f" | {fmt_p(p)}" lines.append(row + " |") # Row: Tokens (if data available) - if any(variant_tokens[vid] for vid in result.variant_ids): + if any(series[vid].tokens for vid in result.variant_ids): row = "| Tokens" for vid in result.variant_ids: - row += f" | {fmt_mean_sd(variant_tokens[vid], ',.0f')}" + row += f" | {fmt_mean_sd(series[vid].tokens, ',.0f')}" if show_p_values: - p = welch_t_test(variant_tokens[vid_a], variant_tokens[vid_b]) + p = welch_t_test(series[vid_a].tokens, series[vid_b].tokens) row += f" | {fmt_p(p)}" lines.append(row + " |") @@ -373,9 +348,7 @@ def _aggregate_metrics_lines(result: ExperimentResult) -> list[str]: columns). The p-value column + Welch t-tests appear only for exactly 2 variants; ``vid_a``/``vid_b`` stay local so the 3+-variant path never indexes them.""" # ── Aggregate Metrics (vertical: metrics as rows, variants as columns) ── - variant_scores, variant_durations, variant_tokens, variant_asst_turns = ( - ExperimentReportGenerator._collect_variant_series(result) - ) + series = collect_variant_series(result) show_p_values = len(result.variant_ids) == 2 vid_a, vid_b = (result.variant_ids[0], result.variant_ids[1]) if show_p_values else ("", "") @@ -391,9 +364,7 @@ def _aggregate_metrics_lines(result: ExperimentResult) -> list[str]: lines = ["", "## Aggregate Metrics", "", header, sep] lines += ExperimentReportGenerator._aggregate_count_rows(result, show_p_values) - lines += ExperimentReportGenerator._aggregate_stat_rows( - result, variant_scores, variant_durations, variant_tokens, variant_asst_turns, show_p_values, vid_a, vid_b - ) + lines += ExperimentReportGenerator._aggregate_stat_rows(result, series, show_p_values, vid_a, vid_b) # Row: Replicates/task (if any variant ran >1 replicate) if any(result.variant_aggregates[vid].replicate_count > 1 for vid in result.variant_ids): @@ -495,51 +466,31 @@ def _replicate_stats_lines(result: ExperimentResult) -> list[str]: def _paired_comparison_lines(result: ExperimentResult) -> list[str]: """The ``## Paired Comparison`` block for 2-variant experiments. - Pairs the two variants' *per-task mean* scores. The task is the unit of - analysis: replicate slots within a task share the task effect and are not - independent, so pairing them individually would understate the standard - error. Unlike ``## Replicate Statistics`` there is no replicate gate — - single-replicate experiments pair task-by-task. Returns ``[]`` only when - the two variants have no scored task in common; when they have exactly - one, the section explains why no paired result is shown. + Renders :func:`coder_eval.reports_stats.paired_comparison`, which the HTML + reporter renders too. Returns ``[]`` only when the two variants have no + scored task in common; when they have exactly one, the section explains why + no paired result is shown. """ - if len(result.variant_ids) != 2: - return [] - vid_a, vid_b = result.variant_ids[0], result.variant_ids[1] - per_rep_a = result.per_replicate_scores.get(vid_a, {}) - per_rep_b = result.per_replicate_scores.get(vid_b, {}) - # Replicate counts need not match: a task's mean score is a well-defined - # pair member either way, so no task is excluded for having fewer runs. - common_tasks = sorted(t for t in set(per_rep_a) & set(per_rep_b) if per_rep_a[t] and per_rep_b[t]) - if not common_tasks: - # Nothing to pair: no shared task, or per_replicate_scores absent (old results). + pc = paired_comparison(result) + if pc is None: return [] header = ["", "## Paired Comparison", ""] - if len(common_tasks) < 2: + if pc.mean_diff is None or pc.ci_low is None or pc.ci_high is None: return [ *header, - f"*A paired comparison needs at least 2 tasks common to {vid_a} and {vid_b}; found 1.*", + f"*A paired comparison needs at least 2 tasks common to {pc.vid_a} and {pc.vid_b};" + + f" found {pc.task_count}.*", ] - a_scores = [mean(per_rep_a[task_id]) for task_id in common_tasks] - b_scores = [mean(per_rep_b[task_id]) for task_id in common_tasks] - # The interval is a percentile bootstrap while the p-value is an exact paired - # t — separate inference models, so they can disagree at small task counts. - # Equal lengths and >=2 pairs are guaranteed above, so this never returns None. - diff = paired_bootstrap_diff_ci(a_scores, b_scores) - if diff is None: # pragma: no cover - defensive - return [] - mean_diff, ci_lo, ci_hi = diff - d_val = cohens_d(a_scores, b_scores) - d_str = f"{d_val:.2f}" if d_val is not None else "n/a" + d_str = f"{pc.effect_size:.2f}" if pc.effect_size is not None else "n/a" return [ *header, - f"*Paired over the per-task mean score of {len(common_tasks)} task(s) common to both variants" + f"*Paired over the per-task mean score of {pc.task_count} task(s) common to both variants" + " — pairing cancels between-task difficulty, which the pooled Welch test above cannot.*", - f"**Paired mean diff ({vid_a} - {vid_b})**: {mean_diff:+.3f}" - + f" [95% CI {ci_lo:+.3f}, {ci_hi:+.3f}], Cohen's d = {d_str}" - + f", p = {fmt_p(paired_t_test(a_scores, b_scores))}", + f"**Paired mean diff ({pc.vid_a} - {pc.vid_b})**: {pc.mean_diff:+.3f}" + + f" [95% CI {pc.ci_low:+.3f}, {pc.ci_high:+.3f}], Cohen's d = {d_str}" + + f", p = {fmt_p(pc.p_value)}", ] @staticmethod diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py index 57184eff..28ef07ea 100644 --- a/src/coder_eval/reports_html.py +++ b/src/coder_eval/reports_html.py @@ -1236,28 +1236,50 @@ def _experiment_prompt_config(experiment: ExperimentDefinition | None, variant_i """ -def _collect_variant_series(result: ExperimentResult) -> dict[str, dict[str, list[float]]]: - """Group per-variant numeric series (scores, durations, etc.).""" - series: dict[str, dict[str, list[float]]] = { - vid: {"scores": [], "durations": [], "asst_turns": [], "tokens": []} for vid in result.variant_ids - } - for ts in result.task_summaries: - for vr in ts.variant_results: - s = series.get(vr.variant_id) - if s is None: - continue - s["scores"].append(vr.weighted_score) - s["durations"].append(vr.duration_seconds) - if vr.total_tokens is not None: - s["tokens"].append(float(vr.total_tokens)) - if vr.total_assistant_turns is not None: - s["asst_turns"].append(float(vr.total_assistant_turns)) - return series +def _experiment_paired_comparison(result: ExperimentResult) -> str: + """Render the Paired Comparison section — the HTML twin of the markdown one. + + Both render the same ``reports_stats.paired_comparison`` result, so the two + reports can never disagree about the paired numbers. + """ + from .reports_stats import fmt_p, paired_comparison + + pc = paired_comparison(result) + if pc is None: + return "" + + if pc.mean_diff is None or pc.ci_low is None or pc.ci_high is None: + body = _esc( + f"A paired comparison needs at least 2 tasks common to {pc.vid_a} and {pc.vid_b}; found {pc.task_count}." + ) + return f""" +

Paired Comparison

+
+

{body}

+
+""" + + d_str = f"{pc.effect_size:.2f}" if pc.effect_size is not None else "n/a" + note = _esc( + f"Paired over the per-task mean score of {pc.task_count} task(s) common to both variants" + + " — pairing cancels between-task difficulty, which the pooled Welch test above cannot." + ) + headline = _esc( + f"Paired mean diff ({pc.vid_a} - {pc.vid_b}): {pc.mean_diff:+.3f}" + + f" [95% CI {pc.ci_low:+.3f}, {pc.ci_high:+.3f}], Cohen's d = {d_str}, p = {fmt_p(pc.p_value)}" + ) + return f""" +

Paired Comparison

+
+

{note}

+

{headline}

+
+""" def _experiment_aggregate_metrics(result: ExperimentResult) -> str: """Render the Aggregate Metrics table (with p-values when exactly 2 variants).""" - from .reports_stats import fmt_mean_sd, fmt_p, welch_t_test + from .reports_stats import collect_variant_series, fmt_mean_sd, fmt_p, welch_t_test show_p = len(result.variant_ids) == 2 vid_a, vid_b = (result.variant_ids[0], result.variant_ids[1]) if show_p else ("", "") @@ -1267,7 +1289,7 @@ def _experiment_aggregate_metrics(result: ExperimentResult) -> str: header_cells.append("p-value") header_html = "" + "".join(header_cells) + "" - series = _collect_variant_series(result) + series = collect_variant_series(result) def _row(label: str, values: list[str], p: str | None) -> str: cells = [f"{_esc(label)}"] + [f"{_esc(v)}" for v in values] @@ -1298,31 +1320,31 @@ def _success_rate(vid: str) -> str: rows.append( _row( "Score", - [fmt_mean_sd(series[vid]["scores"]) for vid in result.variant_ids], - fmt_p(welch_t_test(series[vid_a]["scores"], series[vid_b]["scores"])) if show_p else None, + [fmt_mean_sd(series[vid].scores) for vid in result.variant_ids], + fmt_p(welch_t_test(series[vid_a].scores, series[vid_b].scores)) if show_p else None, ) ) rows.append( _row( "Duration (s)", - [fmt_mean_sd(series[vid]["durations"], ".1f") for vid in result.variant_ids], - fmt_p(welch_t_test(series[vid_a]["durations"], series[vid_b]["durations"])) if show_p else None, + [fmt_mean_sd(series[vid].durations, ".1f") for vid in result.variant_ids], + fmt_p(welch_t_test(series[vid_a].durations, series[vid_b].durations)) if show_p else None, ) ) - if any(series[vid]["asst_turns"] for vid in result.variant_ids): + if any(series[vid].asst_turns for vid in result.variant_ids): rows.append( _row( "Assistant Turns", - [fmt_mean_sd(series[vid]["asst_turns"], ".1f") for vid in result.variant_ids], - fmt_p(welch_t_test(series[vid_a]["asst_turns"], series[vid_b]["asst_turns"])) if show_p else None, + [fmt_mean_sd(series[vid].asst_turns, ".1f") for vid in result.variant_ids], + fmt_p(welch_t_test(series[vid_a].asst_turns, series[vid_b].asst_turns)) if show_p else None, ) ) - if any(series[vid]["tokens"] for vid in result.variant_ids): + if any(series[vid].tokens for vid in result.variant_ids): rows.append( _row( "Tokens", - [fmt_mean_sd(series[vid]["tokens"], ",.0f") for vid in result.variant_ids], - fmt_p(welch_t_test(series[vid_a]["tokens"], series[vid_b]["tokens"])) if show_p else None, + [fmt_mean_sd(series[vid].tokens, ",.0f") for vid in result.variant_ids], + fmt_p(welch_t_test(series[vid_a].tokens, series[vid_b].tokens)) if show_p else None, ) ) @@ -1598,6 +1620,7 @@ def generate_experiment_html( """ + _experiment_prompt_config(experiment, result.variant_ids) + _experiment_aggregate_metrics(result) + + _experiment_paired_comparison(result) + _experiment_win_rates(result) + _experiment_per_task_comparison(result) + _experiment_most_divergent(result) diff --git a/src/coder_eval/reports_stats.py b/src/coder_eval/reports_stats.py index 16dbcf01..a5635def 100644 --- a/src/coder_eval/reports_stats.py +++ b/src/coder_eval/reports_stats.py @@ -12,8 +12,9 @@ import random import statistics as _stats from pathlib import Path +from typing import NamedTuple -from coder_eval.models import EvaluationResult, ExperimentVariant, TaskExperimentSummary +from coder_eval.models import EvaluationResult, ExperimentResult, ExperimentVariant, TaskExperimentSummary logger = logging.getLogger(__name__) @@ -76,7 +77,15 @@ def _betacf(a: float, b: float, x: float) -> float: def regularized_incomplete_beta(a: float, b: float, x: float) -> float: - """Regularized incomplete beta function I_x(a, b), for a, b > 0 and x in [0, 1].""" + """Regularized incomplete beta function I_x(a, b), for a, b > 0 and x in [0, 1]. + + Raises ValueError outside that domain — returning NaN would let a bad input + render as a real-looking statistic downstream. + """ + if not (math.isfinite(a) and math.isfinite(b) and math.isfinite(x)): + raise ValueError(f"a, b and x must be finite, got a={a!r}, b={b!r}, x={x!r}") + if a <= 0.0 or b <= 0.0: + raise ValueError(f"a and b must be positive, got a={a!r}, b={b!r}") if x <= 0.0: return 0.0 if x >= 1.0: @@ -203,30 +212,60 @@ def wilson_interval(successes: int, n: int, confidence: float = 0.95) -> tuple[f return (max(0.0, center - half), min(1.0, center + half)) -def paired_bootstrap_diff_ci( - a: list[float], - b: list[float], - n_resamples: int = 1000, - confidence: float = 0.95, - seed: int = 0, -) -> tuple[float, float, float] | None: - """Paired bootstrap of mean(a_i - b_i). - - Returns (mean_diff, ci_low, ci_high) or None if lengths differ or < 2. - """ +def cohens_d(a: list[float], b: list[float]) -> float | None: + """Paired Cohen's d = mean(a_i - b_i) / stddev(a_i - b_i).""" if len(a) != len(b) or len(a) < 2: return None diffs = [ai - bi for ai, bi in zip(a, b, strict=True)] - return bootstrap_mean_ci(diffs, n_resamples=n_resamples, confidence=confidence, seed=seed) + s = stddev(diffs) + return (sum(diffs) / len(diffs)) / s if s > 0 else None -def cohens_d(a: list[float], b: list[float]) -> float | None: - """Paired Cohen's d = mean(a_i - b_i) / stddev(a_i - b_i).""" +def student_t_critical(confidence: float, df: float) -> float: + """Two-tailed critical value t* with P(|T| >= t*) = 1 - confidence. + + Inverts :func:`student_t_two_tailed_p` by bisection — that p is continuous and + strictly decreasing in |t|, so a plain bracket-and-halve is exact to ~1e-12 and + needs no separate quantile expansion. + """ + if not 0.0 < confidence < 1.0: + raise ValueError(f"confidence must be in (0, 1), got {confidence!r}") + if df <= 0 or not math.isfinite(df): + return math.inf + alpha = 1.0 - confidence + lo, hi = 0.0, 1.0 + while student_t_two_tailed_p(hi, df) > alpha: + lo = hi + hi *= 2.0 + if hi > 1e12: + return hi + for _ in range(200): + mid = (lo + hi) / 2.0 + if mid in (lo, hi): + break + if student_t_two_tailed_p(mid, df) > alpha: + lo = mid + else: + hi = mid + return (lo + hi) / 2.0 + + +def paired_t_ci(a: list[float], b: list[float], confidence: float = 0.95) -> tuple[float, float, float] | None: + """Student-t confidence interval for mean(a_i - b_i): mean ± t* · sd/√n. + + Returns (mean_diff, ci_low, ci_high), or None if lengths differ, n < 2, or any + value is non-finite. Shares its distribution with :func:`paired_t_test`, so the + interval and the p-value always agree about whether 0 is excluded. + """ if len(a) != len(b) or len(a) < 2: return None + if not all(math.isfinite(v) for v in (*a, *b)): + return None diffs = [ai - bi for ai, bi in zip(a, b, strict=True)] - s = stddev(diffs) - return (sum(diffs) / len(diffs)) / s if s > 0 else None + n = len(diffs) + mean_diff = sum(diffs) / n + half_width = student_t_critical(confidence, n - 1) * stddev(diffs) / math.sqrt(n) + return (mean_diff, mean_diff - half_width, mean_diff + half_width) def paired_t_test(a: list[float], b: list[float]) -> float | None: @@ -249,6 +288,102 @@ def paired_t_test(a: list[float], b: list[float]) -> float | None: return student_t_two_tailed_p(t_stat, len(diffs) - 1) +# --------------------------------------------------------------------------- +# Aggregate-metric series +# --------------------------------------------------------------------------- + + +class VariantSeries(NamedTuple): + """One variant's numeric series across all tasks — the raw inputs to the + Aggregate Metrics rows and their p-values.""" + + scores: list[float] + durations: list[float] + tokens: list[float] + asst_turns: list[float] + + +def collect_variant_series(result: ExperimentResult) -> dict[str, VariantSeries]: + """Per-variant (scores, durations, tokens, assistant-turns) series, keyed by variant id. + + Shared by the markdown and HTML reporters so both render the same numbers. + ``VariantResult.duration_seconds`` is *summed* across replicates, so it is + divided by ``replicate_count`` to give a per-run duration comparable across + variants that ran different replicate counts. + """ + series = {vid: VariantSeries([], [], [], []) for vid in result.variant_ids} + for ts in result.task_summaries: + for vr in ts.variant_results: + s = series.get(vr.variant_id) + if s is None: # a task result for a variant not in variant_ids + continue + s.scores.append(vr.weighted_score) + s.durations.append(vr.duration_seconds / vr.replicate_count) + if vr.total_tokens is not None: + s.tokens.append(float(vr.total_tokens)) + if vr.total_assistant_turns is not None: + s.asst_turns.append(float(vr.total_assistant_turns)) + return series + + +class PairedComparison(NamedTuple): + """A 2-variant paired comparison over per-task mean scores. + + ``task_count`` is the number of tasks both variants scored. When it is < 2 + the statistics are all ``None`` — there is nothing to compare, and the + reporters say so rather than rendering an empty section. + """ + + vid_a: str + vid_b: str + task_count: int + mean_diff: float | None + ci_low: float | None + ci_high: float | None + effect_size: float | None + p_value: float | None + + +def paired_comparison(result: ExperimentResult, confidence: float = 0.95) -> PairedComparison | None: + """Pair the two variants' per-task mean scores. Returns None unless the + experiment has exactly 2 variants with at least one commonly-scored task. + + The task is the unit of analysis: replicate slots within a task share the task + effect and are not independent, so pairing them individually would understate + the standard error. Replicate counts need not match — a task's mean score is a + well-defined pair member either way. + """ + if len(result.variant_ids) != 2: + return None + vid_a, vid_b = result.variant_ids[0], result.variant_ids[1] + per_rep_a = result.per_replicate_scores.get(vid_a, {}) + per_rep_b = result.per_replicate_scores.get(vid_b, {}) + common_tasks = sorted(t for t in set(per_rep_a) & set(per_rep_b) if per_rep_a[t] and per_rep_b[t]) + if not common_tasks: + # No shared task, or per_replicate_scores absent (results from before it existed). + return None + + if len(common_tasks) < 2: + return PairedComparison(vid_a, vid_b, len(common_tasks), None, None, None, None, None) + + a_scores = [mean(per_rep_a[task_id]) for task_id in common_tasks] + b_scores = [mean(per_rep_b[task_id]) for task_id in common_tasks] + ci = paired_t_ci(a_scores, b_scores, confidence=confidence) + if ci is None: # non-finite scores + return PairedComparison(vid_a, vid_b, len(common_tasks), None, None, None, None, None) + mean_diff, ci_low, ci_high = ci + return PairedComparison( + vid_a, + vid_b, + len(common_tasks), + mean_diff, + ci_low, + ci_high, + cohens_d(a_scores, b_scores), + paired_t_test(a_scores, b_scores), + ) + + # --------------------------------------------------------------------------- # Prompt config + variant-result loaders # --------------------------------------------------------------------------- diff --git a/tests/_fixtures/report_snapshots/experiment_2variant.md b/tests/_fixtures/report_snapshots/experiment_2variant.md index 18830bac..e19649fd 100644 --- a/tests/_fixtures/report_snapshots/experiment_2variant.md +++ b/tests/_fixtures/report_snapshots/experiment_2variant.md @@ -45,4 +45,4 @@ ## Paired Comparison *Paired over the per-task mean score of 2 task(s) common to both variants — pairing cancels between-task difficulty, which the pooled Welch test above cannot.* -**Paired mean diff (baseline - mutated)**: -0.125 [95% CI -0.450, +0.200], Cohen's d = -0.27, p = 0.766 +**Paired mean diff (baseline - mutated)**: -0.125 [95% CI -4.255, +4.005], Cohen's d = -0.27, p = 0.766 diff --git a/tests/test_experiment_reports.py b/tests/test_experiment_reports.py index 0f7da38e..8e55103a 100644 --- a/tests/test_experiment_reports.py +++ b/tests/test_experiment_reports.py @@ -1215,6 +1215,84 @@ def test_variant_report_no_ci_when_replicate_count_is_one(self): assert "Score 95% CI" not in md +class TestCollectVariantSeries: + """The one series collector both reporters share.""" + + @staticmethod + def _result_with_replicates() -> ExperimentResult: + """One task per variant, 3 replicates, 90s summed ⇒ 30s per run.""" + return ExperimentResult( + experiment_id="exp", + description="d", + variant_ids=["a"], + task_summaries=[ + TaskExperimentSummary( + task_id="t1", + variant_results=[ + VariantResult( + variant_id="a", + task_id="t1", + weighted_score=0.8, + final_status="SUCCESS", + duration_seconds=90.0, + replicate_count=3, + ) + ], + best_variant="a", + score_spread=0.0, + ) + ], + variant_aggregates={ + "a": VariantAggregate( + variant_id="a", + tasks_run=1, + tasks_succeeded=1, + tasks_failed=0, + tasks_error=0, + average_score=0.8, + average_duration=30.0, + replicate_count=3, + ) + }, + total_duration_seconds=90.0, + ) + + def test_duration_is_per_run_not_replicate_inflated(self): + """duration_seconds is summed across replicates, so it must be divided out.""" + from coder_eval.reports_stats import collect_variant_series + + series = collect_variant_series(self._result_with_replicates()) + assert series["a"].durations == [30.0] + + def test_html_and_markdown_report_the_same_duration(self): + """Regression: the HTML copy of this collector used the raw summed duration.""" + from coder_eval.reports_html import _experiment_aggregate_metrics + + result = self._result_with_replicates() + md = ExperimentReportGenerator.generate_experiment_report(result) + html = _experiment_aggregate_metrics(result) + assert "| Avg Duration (s) | 30.0 |" in md + assert "30.0" in html + assert "90.0" not in html + + def test_result_for_unknown_variant_is_ignored(self): + """A task result naming a variant outside variant_ids must not raise.""" + from coder_eval.reports_stats import collect_variant_series + + result = self._result_with_replicates() + result.task_summaries[0].variant_results.append( + VariantResult( + variant_id="ghost", + task_id="t1", + weighted_score=0.1, + final_status="SUCCESS", + duration_seconds=1.0, + ) + ) + series = collect_variant_series(result) + assert set(series) == {"a"} + + class TestPairedComparisonSection: """Tests for the ``## Paired Comparison`` markdown section.""" @@ -1347,6 +1425,45 @@ def test_paired_comparison_keeps_tasks_with_unequal_replicate_counts(self): # Task means: a = [0.6, 0.4, 0.8], b = [0.5, 0.45, 0.7] ⇒ diffs [+0.1, -0.05, +0.1]. assert "**Paired mean diff (a - b)**: +0.050" in md + def test_html_renders_the_same_paired_numbers_as_markdown(self): + """Both reporters render one shared computation, so they cannot disagree.""" + from coder_eval.reports_html import _experiment_paired_comparison + + result = self._make_result( + ["a", "b"], + {"a": {"t1": [0.9], "t2": [0.5], "t3": [0.8]}, "b": {"t1": [0.7], "t2": [0.55], "t3": [0.6]}}, + ) + md = ExperimentReportGenerator.generate_experiment_report(result) + html = _experiment_paired_comparison(result) + assert "

Paired Comparison

" in html + # Cohen's apostrophe is HTML-escaped, so match the numbers either side of it. + for fragment in ("Paired mean diff (a - b): +0.117", "d = 0.81", "p = 0.296"): + assert fragment in html + assert "**Paired mean diff (a - b)**: +0.117 [95% CI -0.242, +0.475], Cohen's d = 0.81, p = 0.296" in md + + def test_html_paired_section_absent_for_three_variants(self): + from coder_eval.reports_html import _experiment_paired_comparison + + result = self._make_result( + ["a", "b", "c"], + {"a": {"t1": [0.9]}, "b": {"t1": [0.7]}, "c": {"t1": [0.6]}}, + ) + assert _experiment_paired_comparison(result) == "" + + def test_paired_ci_agrees_with_p_value(self): + """The interval and the p-value share a distribution, so they agree about 0.""" + result = self._make_result( + ["a", "b"], + { + "a": {"t1": [0.9], "t2": [0.85], "t3": [0.95], "t4": [0.9]}, + "b": {"t1": [0.2], "t2": [0.25], "t3": [0.15], "t4": [0.3]}, + }, + ) + md = ExperimentReportGenerator.generate_experiment_report(result) + # A large, consistent gap ⇒ p well below 0.05 and a CI clear of zero. + assert "p = <0.001" in md + assert "[95% CI +0." in md + def test_paired_comparison_ignores_tasks_with_no_scores(self): """A task with an empty replicate list on either side is not a pair.""" result = self._make_result( @@ -1359,9 +1476,9 @@ def test_paired_comparison_ignores_tasks_with_no_scores(self): class TestExperimentReportSnapshots: """Byte-identical characterization snapshots for generate_experiment_report — the - safety net for its decomposition. Output is deterministic: bootstrap_mean_ci / - paired_bootstrap_diff_ci in reports_stats use a fixed default seed, so no scrubbing - is needed (do not pass a varying seed).""" + safety net for its decomposition. Output is deterministic: bootstrap_mean_ci in + reports_stats uses a fixed default seed, so no scrubbing is needed (do not pass a + varying seed); the paired section is closed-form Student-t with no randomness.""" def test_experiment_report_snapshot_2variant(self): """2 variants → p-value column shown; plus prompt config, budget sub-rows, diff --git a/tests/test_replicate_stats.py b/tests/test_replicate_stats.py index 850996e8..e0c99012 100644 --- a/tests/test_replicate_stats.py +++ b/tests/test_replicate_stats.py @@ -5,7 +5,9 @@ from coder_eval.reports_stats import ( bootstrap_mean_ci, cohens_d, - paired_bootstrap_diff_ci, + paired_t_ci, + paired_t_test, + student_t_critical, wilson_interval, ) @@ -94,54 +96,62 @@ def test_bounds_clamped_to_unit_interval(self): assert 0.0 <= hi <= 1.0 -class TestPairedBootstrapDiffCi: - def test_none_on_length_mismatch(self): - assert paired_bootstrap_diff_ci([1.0, 2.0], [1.0]) is None - - def test_none_on_single_pair(self): - assert paired_bootstrap_diff_ci([1.0], [0.5]) is None +class TestPairedTCi: + """The Student-t paired interval that replaced the percentile bootstrap — it + shares a distribution with paired_t_test, so the two always agree about 0.""" - def test_none_on_empty(self): - assert paired_bootstrap_diff_ci([], []) is None + def test_none_on_invalid_input(self): + assert paired_t_ci([1.0, 2.0], [1.0]) is None + assert paired_t_ci([1.0], [0.5]) is None + assert paired_t_ci([], []) is None + assert paired_t_ci([1.0, float("nan")], [0.5, 0.5]) is None - def test_detects_positive_shift(self): - a = [1.0] * 20 - b = [0.5] * 20 - result = paired_bootstrap_diff_ci(a, b) + def test_zero_variance_shift_is_a_point_interval(self): + result = paired_t_ci([1.0] * 20, [0.5] * 20) assert result is not None mean_diff, lo, hi = result - assert abs(mean_diff - 0.5) < 0.01 - # CI should be very tight around 0.5 (zero variance) - assert lo > 0.45 - assert hi < 0.55 - - def test_detects_negative_shift(self): - # Use non-constant inputs so CI is not degenerate - import random - - rng = random.Random(7) - a = [0.3 + rng.gauss(0, 0.05) for _ in range(20)] - b = [0.8 + rng.gauss(0, 0.05) for _ in range(20)] - result = paired_bootstrap_diff_ci(a, b) - assert result is not None - mean_diff, lo, hi = result - assert mean_diff < 0 - assert lo < hi - - def test_ci_contains_diff(self): - a = [float(i) / 9 for i in range(10)] - b = [float(i) / 9 + 0.1 for i in range(10)] - result = paired_bootstrap_diff_ci(a, b) + assert abs(mean_diff - 0.5) < 1e-12 + # Every diff identical ⇒ sd = 0 ⇒ the interval collapses onto the mean. + assert (lo, hi) == (0.5, 0.5) + + def test_ci_contains_diff_and_is_symmetric(self): + a = [0.3, 0.5, 0.9, 0.2, 0.7] + b = [0.2, 0.55, 0.6, 0.3, 0.5] + result = paired_t_ci(a, b) assert result is not None mean_diff, lo, hi = result assert lo <= mean_diff <= hi + assert abs((mean_diff - lo) - (hi - mean_diff)) < 1e-12 + + def test_agrees_with_paired_t_test_about_zero(self): + """The interval excludes 0 exactly when the p-value is below alpha.""" + for a, b in ( + ([0.9, 0.5, 0.8, 0.7, 0.95], [0.7, 0.55, 0.6, 0.72, 0.8]), # p = 0.153, not significant + ([0.9, 0.85, 0.95, 0.8, 0.9], [0.2, 0.25, 0.15, 0.3, 0.2]), # strongly significant + ): + ci = paired_t_ci(a, b, confidence=0.95) + p = paired_t_test(a, b) + assert ci is not None and p is not None + _, lo, hi = ci + excludes_zero = lo > 0 or hi < 0 + assert excludes_zero == (p < 0.05) + + +class TestStudentTCritical: + def test_matches_t_table(self): + assert abs(student_t_critical(0.95, 4) - 2.776) < 1e-3 + assert abs(student_t_critical(0.95, 10) - 2.228) < 1e-3 + assert abs(student_t_critical(0.99, 10) - 3.169) < 1e-3 + + def test_converges_to_normal_at_large_df(self): + assert abs(student_t_critical(0.95, 1e7) - 1.959964) < 1e-4 + + def test_rejects_confidence_outside_unit_interval(self): + with pytest.raises(ValueError, match="confidence must be in"): + student_t_critical(1.0, 5) - def test_is_deterministic(self): - a = [0.1 * i for i in range(1, 11)] - b = [0.1 * i + 0.05 for i in range(1, 11)] - r1 = paired_bootstrap_diff_ci(a, b) - r2 = paired_bootstrap_diff_ci(a, b) - assert r1 == r2 + def test_degenerate_df_is_infinite(self): + assert student_t_critical(0.95, 0) == float("inf") class TestCohensD: diff --git a/tests/test_reports_stats_nonfinite.py b/tests/test_reports_stats_nonfinite.py new file mode 100644 index 00000000..fe9f8ce6 --- /dev/null +++ b/tests/test_reports_stats_nonfinite.py @@ -0,0 +1,73 @@ +"""Harness: no public numeric helper in reports_stats may launder NaN/inf into a report. + +This enumerates the module rather than listing functions by hand, so a helper added +later is covered automatically — the failure mode this guards (a new statistic that +silently returns NaN, which then renders as a real-looking number) is exactly the one +that motivated the exact-t work. See review-rubric.md criterion 15. + +To add a helper that legitimately propagates non-finite input, add it to +``_PASSTHROUGH_HELPERS`` with a reason — that edit is the point, it forces the call +to be deliberate. +""" + +import inspect +import math + +import pytest + +from coder_eval import reports_stats + + +# Thin wrappers over the stdlib that intentionally propagate whatever they are given; +# they are inputs to the guarded functions below, not report-facing statistics. +_PASSTHROUGH_HELPERS = {"mean", "stddev"} + +_NAN = float("nan") +_INF = float("inf") + + +def _numeric_helpers(): + """Public reports_stats functions whose parameters are all floats / float lists.""" + for name, fn in vars(reports_stats).items(): + if name.startswith("_") or name in _PASSTHROUGH_HELPERS or not inspect.isfunction(fn): + continue + if fn.__module__ != reports_stats.__name__: + continue + hints = inspect.get_annotations(fn, eval_str=True) + params = inspect.signature(fn).parameters + annotations = [hints.get(p) for p in params] + if annotations and all(ann is float or ann == list[float] for ann in annotations): + yield name, fn, params, hints + + +def _bad_args(params, hints, bad_value): + args = [] + for name in params: + args.append([1.0, bad_value] if hints.get(name) == list[float] else bad_value) + return args + + +def _floats_in(value): + if isinstance(value, float): + yield value + elif isinstance(value, tuple): + for item in value: + yield from _floats_in(item) + + +@pytest.mark.parametrize("bad_value", [_NAN, _INF, -_INF], ids=["nan", "inf", "-inf"]) +def test_no_public_numeric_helper_returns_nan(bad_value): + """Non-finite input must yield None, a finite number, or an explicit ValueError.""" + checked = [] + for name, fn, params, hints in _numeric_helpers(): + checked.append(name) + try: + result = fn(*_bad_args(params, hints, bad_value)) + except ValueError: + continue # refusing explicitly is a valid response + for f in _floats_in(result): + assert not math.isnan(f), f"{name} returned NaN for {bad_value} input" + + # Guard the guard: if the discovery filter silently matches nothing, the test + # above would pass vacuously. + assert {"welch_t_test", "paired_t_test", "paired_t_ci", "student_t_two_tailed_p"} <= set(checked)