From 8c02dc6b7116527de012f3d46407efad6e6826ae Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 16:31:42 -0700 Subject: [PATCH 1/2] fix: weight:0 un-gates criteria (informational criteria) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - weight: 0 now makes a criterion purely informational — excluded from both the weighted score AND the pass/fail gate (previously it could still flip a task to FAILURE). New BaseSuccessCriterion.is_gating drives the single-source gate (all_criteria_passed) and the `coder-eval evaluate` exit code. A validator rejects weight:0 combined with stop_when or suite_thresholds (arming a non-gating criterion for a pass/fail gate is incoherent). - reports.py failure-reason sampling uses pass_threshold, not score < 1.0. - Doc drift: dropped the non-existent `rephrase` mutation; corrected the stale "max_memory_mb NOT enforced" claim (docker --memory/--cpus/--pids-limit). Split out per review: the configurable retry policy moved to #44, and the evalboard OSS-edition change is dropped entirely (see @bai-uipath's review comment — the panels need core-produced fields that do not exist yet). Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 4 +- docs/AB_EXPERIMENTS.md | 4 +- docs/TASK_DEFINITION_GUIDE.md | 21 ++++++-- src/coder_eval/cli/evaluate_command.py | 19 +++++-- src/coder_eval/models/criteria.py | 44 ++++++++++++++-- src/coder_eval/models/results.py | 30 +++++++---- src/coder_eval/models/sandbox.py | 5 +- src/coder_eval/reports.py | 2 +- .../test_task_informational_criterion.yaml | 17 +++++++ tests/test_evaluate_command.py | 21 ++++++++ tests/test_threshold_enforcement.py | 51 +++++++++++++++++++ 11 files changed, 187 insertions(+), 31 deletions(-) create mode 100644 tests/fixtures/tasks/test_task_informational_criterion.yaml diff --git a/CLAUDE.md b/CLAUDE.md index 5c5cf392..c20bde4f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,7 +35,7 @@ coder_eval/ │ ├── criteria.py # 14 success criterion types + base + union │ ├── experiment.py # ExperimentDefinition, ExperimentVariant, ResolvedTask, result models │ ├── judge_defaults.py # DEFAULT_JUDGE_MODEL constant (cycle-free leaf) -│ ├── mutations.py # PromptMutation variants (prefix/suffix/replace/template/rephrase) +│ ├── mutations.py # PromptMutation variants (prefix/suffix/replace/template) │ ├── results.py # CriterionResult (+ ClassificationCriterionResult), TurnRecord, EvaluationResult, EarlyStopInfo/EarlyStopReason, CriterionAggregate, ThresholdCheck, SuiteRollup │ ├── routing.py # ApiRoute (DirectRoute/BedrockRoute) │ ├── sandbox.py # SandboxConfig, ResourceLimits @@ -294,7 +294,7 @@ Tasks are YAML files. See [docs/TASK_DEFINITION_GUIDE.md](docs/TASK_DEFINITION_G **Runtime (always)**: pydantic, pydantic-settings, pyyaml, typer, rich, python-dotenv, anthropic, claude-agent-sdk, anyio, radon, tqdm, jmespath, jsonschema -**Runtime (optional, `[uipath]` extra)**: uipath — the in-host `uipath` SDK (handy for local sandbox parity with tasks that invoke `uv run uipath eval ...`). Base installs without this extra still run end-to-end; UiPath-dependent paths fail at dispatch with a clear `pip install 'coder-eval[uipath]'` hint. The LLM judge and the `rephrase` mutation no longer use the LLM Gateway client — they route through the run's backend (Bedrock / Anthropic), so `uipath-llmgw-client` is no longer a dependency. +**Runtime (optional, `[uipath]` extra)**: uipath — the in-host `uipath` SDK (handy for local sandbox parity with tasks that invoke `uv run uipath eval ...`). Base installs without this extra still run end-to-end; UiPath-dependent paths fail at dispatch with a clear `pip install 'coder-eval[uipath]'` hint. The LLM judge no longer uses the LLM Gateway client — it routes through the run's backend (Bedrock / Anthropic), so `uipath-llmgw-client` is no longer a dependency. **Dev**: pytest, pytest-asyncio, pytest-mock, pytest-cov, ruff, pyright, pip-audit, bandit, pre-commit, mcp diff --git a/docs/AB_EXPERIMENTS.md b/docs/AB_EXPERIMENTS.md index 52c56269..0f437bc3 100644 --- a/docs/AB_EXPERIMENTS.md +++ b/docs/AB_EXPERIMENTS.md @@ -227,8 +227,8 @@ variants: text: "\n\nThink step by step and validate your work before finishing." ``` -The full mutation catalog (prefix / suffix / replace / template / rephrase) is -defined in `coder_eval/models/mutations.py`. +The full mutation catalog (prefix / suffix / replace / template) is defined in +`coder_eval/models/mutations.py`. ## Recipe: Smoke vs. e2e Flavors (Early Stop) diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 3de0c196..2fe88b53 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -275,11 +275,17 @@ sandbox: - "!node_modules" - "*.bak" # bare entry adds an extra pattern limits: # Optional: resource limits - timeout: 300 # Enforced via subprocess timeout - max_memory_mb: 512 # NOT enforced (reserved for future use) - max_disk_mb: 1024 # NOT enforced (reserved for future use) + timeout: 300 # Enforced via subprocess timeout (both drivers) + max_memory_mb: 512 # driver:docker -> `--memory`; ignored under tempdir + max_cpus: 2 # driver:docker -> `--cpus`; ignored under tempdir + max_pids: 512 # driver:docker -> `--pids-limit`; ignored under tempdir + max_disk_mb: 1024 # NOT enforced (reserved: no portable docker knob) ``` +Under `driver: tempdir` only `timeout` is enforced — the agent can consume +arbitrary host memory, CPU, and PIDs. Use `driver: docker` when you need the +container limits above to actually bind. + ## Template Sources Tasks can start with preset files instead of an empty sandbox. Multiple sources are applied sequentially (last wins for conflicts). @@ -383,7 +389,7 @@ All criteria share these fields: | Field | Default | Description | |-------|---------|-------------| | `description` | — | Human-readable description (required) | -| `weight` | 1.0 | Relative importance for weighted score | +| `weight` | 1.0 | Relative importance for weighted score. `0` = **informational**: excluded from both the score and the pass/fail gate | | `pass_threshold` | 0.9 | Minimum score (0.0–1.0) to pass | | `stop_when` | `null` | Arms this criterion for early stop (`pass`/`fail`/`decided`); requires `run_limits.stop_early: true` and an observable criterion type (`skill_triggered`, `command_executed`). See [`stop_early`](#stop_early-opt-in-early-stop). | @@ -392,7 +398,12 @@ All criteria share these fields: - **Fractional** (0.0–1.0): `file_contains`, `file_check`, `json_check`, `command_executed`, `uipath_eval` - **Continuous** (0.0–1.0): `reference_comparison`, `llm_judge`, `agent_judge` -**Task success:** ALL criteria must score >= their `pass_threshold`. +**Task success:** all *gating* criteria must score >= their `pass_threshold`. A +criterion with `weight: 0` is informational — it is still checked, stored, and +rendered in reports, but it neither contributes to the score nor fails the task. +(A `weight: 0` criterion may not set `stop_when` or `suite_thresholds`: arming a +non-gating criterion for the early-stop or suite gate would let an +"informational" check flip a run to failure.) **Weighted score:** `weighted_score = sum(score * weight) / sum(weight)` — calculated regardless for quality assessment. diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py index bad1d9c3..1e62d886 100644 --- a/src/coder_eval/cli/evaluate_command.py +++ b/src/coder_eval/cli/evaluate_command.py @@ -130,7 +130,12 @@ async def _setup_and_run() -> EvaluationResult: raise typer.Exit(1) for criterion, cr in zip(task.success_criteria, criteria_results, strict=True): - status = "[green]✓[/green]" if cr.score >= criterion.pass_threshold else "[red]✗[/red]" + if not criterion.is_gating: + # weight=0 is informational: it cannot pass/fail the task, so don't + # render it as ✓/✗ (that would contradict the gate and the exit code). + status = "[dim]○[/dim]" + else: + status = "[green]✓[/green]" if cr.score >= criterion.pass_threshold else "[red]✗[/red]" console.print(f"{status} {cr.criterion_type}") console.print(f" [dim]{cr.description}[/dim]") console.print(f" [dim]Score: {cr.score:.2f}[/dim]") @@ -140,15 +145,19 @@ async def _setup_and_run() -> EvaluationResult: console.print(f" [red]Error: {cr.error}[/red]") console.print() - passed = sum( - 1 for cr, c in zip(criteria_results, task.success_criteria, strict=True) if cr.score >= c.pass_threshold - ) - total = len(task.success_criteria) + # Gate over gating criteria only (weight=0 is informational and cannot fail + # the task) so this summary + the exit code below match final_status. + gating = [(cr, c) for cr, c in zip(criteria_results, task.success_criteria, strict=True) if c.is_gating] + passed = sum(1 for cr, c in gating if cr.score >= c.pass_threshold) + total = len(gating) failed = total - passed + informational = len(task.success_criteria) - total console.print("[bold]Summary:[/bold]") console.print(f" Passed: {passed}/{total}") console.print(f" Failed: {failed}/{total}") + if informational: + console.print(f" [dim]Informational (weight=0, not gated): {informational}[/dim]") console.print(f"\n[dim]Run directory: {prepared_run_dir}[/dim]") if result.sandbox_path: console.print(f"[dim]Artifacts: {result.sandbox_path}[/dim]") diff --git a/src/coder_eval/models/criteria.py b/src/coder_eval/models/criteria.py index 1bb59c8f..8786ac86 100644 --- a/src/coder_eval/models/criteria.py +++ b/src/coder_eval/models/criteria.py @@ -9,7 +9,7 @@ from __future__ import annotations from abc import ABC -from typing import Annotated, Any, ClassVar, Literal +from typing import Annotated, Any, ClassVar, Literal, Self from pydantic import BaseModel, ConfigDict, Field, model_validator @@ -101,10 +101,11 @@ class BaseSuccessCriterion(BaseModel, ABC): ge=0.0, description=( "Relative importance of this criterion in the weighted score (default: 1.0). Set to 0 to " - "exclude it from the weighted SCORE -- useful for informational or side-effect checks " - "(e.g. a setup command). NOTE: weight=0 excludes from the score but NOT from the pass/fail " - "gate -- a criterion scoring below its pass_threshold still flips the task to FAILURE " - "regardless of weight. To make a criterion truly non-gating, also set pass_threshold=0." + "make the criterion purely INFORMATIONAL -- useful for side-effect checks (e.g. a setup " + "command): it is excluded from the weighted score AND from the pass/fail gate, so scoring " + "below its pass_threshold no longer flips the task to FAILURE. The result is still " + "computed, stored, and rendered in reports. A weight=0 criterion may not set stop_when " + "or suite_thresholds (arming a non-gating criterion for a pass/fail gate is incoherent)." ), ) @@ -145,6 +146,39 @@ def model_post_init(self, context: Any, /) -> None: # the discriminated union rejects the dump with union_tag_not_found. self.__pydantic_fields_set__.add("type") + @model_validator(mode="after") + def check_weight_zero_is_not_gating(self) -> Self: + """Reject ``weight: 0`` combined with any gate-arming field. + + ``weight: 0`` makes a criterion informational — excluded from the score + and from the pass/fail gate (see ``is_gating``). Both ``stop_when`` (arms + the per-row early-stop gate) and ``suite_thresholds`` (arms the + across-row suite gate, which drives the run's exit code) would let an + "informational" criterion flip a run to failure — directly contradicting + the field's contract. So both combinations are authoring errors, caught + at load time rather than surfacing as a confusing exit code later. + """ + if self.weight == 0.0: + for field, name in (("stop_when", "stop_when"), ("suite_thresholds", "suite_thresholds")): + if getattr(self, field) is not None: + raise ValueError( + f"criterion {self.type!r}: weight=0 makes the criterion informational (non-gating), " + + f"so it cannot also set {name} (which arms it for a pass/fail gate). " + + f"Give it a non-zero weight, or drop {name}." + ) + return self + + @property + def is_gating(self) -> bool: + """True when this criterion participates in the task's pass/fail gate. + + Single source of truth for "does a low score here fail the task". A + ``weight: 0`` criterion is informational: it is excluded from the + weighted score (it contributes 0 to both numerator and denominator) and, + by the same token, from the gate — so the two never disagree. + """ + return self.weight > 0.0 + # Business logic (check operations) moved to SuccessChecker in evaluator.py diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 05dbbb1d..dbc1e33a 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -593,20 +593,30 @@ def calculate_weighted_score(self, criteria: list[SuccessCriterion]) -> None: self.weighted_score = total_weighted_score / total_weight if total_weight > 0 else 0.0 def all_criteria_passed(self, criteria: list[SuccessCriterion]) -> bool: - """True iff every criterion result meets its pass_threshold. - - Single source of truth for the success gate. A results/criteria length - mismatch raises ``ValueError`` rather than silently truncating — the - ``len()`` pre-check is required because ``all()`` short-circuits on the - first failing pair, so ``zip(strict=True)`` alone would not reliably - reach the length check. + """True iff every GATING criterion result meets its pass_threshold. + + Single source of truth for the success gate. ``weight: 0`` criteria are + informational (``BaseSuccessCriterion.is_gating`` is False): they are + excluded from the weighted score, so they are excluded from the gate + too — otherwise a criterion that contributes nothing to the score could + still single-handedly flip the task to FAILURE. A task whose criteria + are ALL weight-0 has an empty gate and therefore passes. + + A results/criteria length mismatch raises ``ValueError`` rather than + silently truncating — the ``len()`` pre-check is required because + ``all()`` short-circuits on the first failing pair, so + ``zip(strict=True)`` alone would not reliably reach the length check. """ if len(self.success_criteria_results) != len(criteria): raise ValueError( f"Results/criteria length mismatch for task {self.task_id}: " + f"{len(self.success_criteria_results)} results vs {len(criteria)} criteria." ) - return all(r.score >= c.pass_threshold for r, c in zip(self.success_criteria_results, criteria, strict=True)) + return all( + r.score >= c.pass_threshold + for r, c in zip(self.success_criteria_results, criteria, strict=True) + if c.is_gating + ) def armed_criteria_passed(self, criteria: list[SuccessCriterion]) -> bool: """True iff every ARMED criterion (``stop_when`` set) meets its pass_threshold. @@ -618,7 +628,9 @@ def armed_criteria_passed(self, criteria: list[SuccessCriterion]) -> bool: pre-check as ``all_criteria_passed`` so the gate logic stays single-sourced. Raises ``ValueError`` on an empty armed set — unreachable when a stop actually fired (a stop requires an armed criterion), so this - is a defensive guard against misuse. + is a defensive guard against misuse. No ``is_gating`` filter is needed + here: ``BaseSuccessCriterion`` rejects ``weight: 0`` together with + ``stop_when``, so every armed criterion is gating by construction. """ if len(self.success_criteria_results) != len(criteria): raise ValueError( diff --git a/src/coder_eval/models/sandbox.py b/src/coder_eval/models/sandbox.py index bf86106e..282cdbdc 100644 --- a/src/coder_eval/models/sandbox.py +++ b/src/coder_eval/models/sandbox.py @@ -300,8 +300,9 @@ class SandboxConfig(BaseModel): network, process table, and filesystem outside the temp dir. ``driver: docker`` runs each task inside its own container; see :class:`DockerDriverConfig` for knobs. ``ResourceLimits.timeout`` is the - only limit enforced in tempdir mode; under docker, ``max_memory_mb`` also - maps to ``--memory`` when set. + only limit enforced in tempdir mode; under docker, ``max_memory_mb`` / + ``max_cpus`` / ``max_pids`` also map to ``--memory`` / ``--cpus`` / + ``--pids-limit`` when set. """ model_config = ConfigDict(populate_by_name=True, extra="forbid") diff --git a/src/coder_eval/reports.py b/src/coder_eval/reports.py index 6e5b9deb..3dac1371 100644 --- a/src/coder_eval/reports.py +++ b/src/coder_eval/reports.py @@ -815,7 +815,7 @@ def _compute_suite_rollup( break reasons: list[str] = [] for cr in row.result.success_criteria_results: - if cr.error is not None or cr.score < 1.0: + if cr.error is not None or cr.score < cr.pass_threshold: reason = cr.error or cr.details or f"{cr.criterion_type}: score={cr.score:.2f}" reasons.append(reason[:_FAILURE_REASON_MAX_LEN]) if len(reasons) >= _FAILURE_REASONS_PER_ROW: diff --git a/tests/fixtures/tasks/test_task_informational_criterion.yaml b/tests/fixtures/tasks/test_task_informational_criterion.yaml new file mode 100644 index 00000000..98a0f0df --- /dev/null +++ b/tests/fixtures/tasks/test_task_informational_criterion.yaml @@ -0,0 +1,17 @@ +task_id: test_task_informational +description: Gating criterion passes; a weight=0 informational criterion fails. +initial_prompt: Test +# agent block required by TaskDefinition schema but unused in evaluate-only mode +agent: + type: claude-code +sandbox: + driver: tempdir + python: null +success_criteria: + - type: file_exists + path: app.py + description: app.py must exist (gating) + - type: file_exists + path: missing.py + description: informational side-effect check (weight=0, must not gate) + weight: 0.0 diff --git a/tests/test_evaluate_command.py b/tests/test_evaluate_command.py index 24fd16ea..67a98046 100644 --- a/tests/test_evaluate_command.py +++ b/tests/test_evaluate_command.py @@ -132,6 +132,27 @@ def test_evaluate_command_failure(tmp_path): assert exc_info.value.exit_code == 1 +def test_evaluate_command_informational_criterion_does_not_fail_exit(tmp_path): + """A failing weight=0 criterion must not flip the exit code (it is non-gating).""" + task_file = FIXTURES_DIR / "tasks" / "test_task_informational_criterion.yaml" + + work_dir = tmp_path / "work" + work_dir.mkdir() + (work_dir / "app.py").write_text("print('hello')") # gating criterion passes; missing.py absent + + from coder_eval.cli.evaluate_command import evaluate_command + + run_dir = tmp_path / "run" + run_dir.mkdir() + + with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): + with pytest.raises(typer.Exit) as exc_info: + evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + + # The only gating criterion passed → exit 0, despite the weight=0 miss. + assert exc_info.value.exit_code == 0 + + def test_evaluate_command_invalid_task_file(tmp_path): """Test evaluate command with invalid task file.""" # Use non-existent task file diff --git a/tests/test_threshold_enforcement.py b/tests/test_threshold_enforcement.py index ec789805..fa7a1c39 100644 --- a/tests/test_threshold_enforcement.py +++ b/tests/test_threshold_enforcement.py @@ -9,8 +9,10 @@ from datetime import datetime import pytest +from pydantic import ValidationError from coder_eval.models import ( + CommandExecutedCriterion, CriterionResult, EvaluationResult, FileExistsCriterion, @@ -103,6 +105,55 @@ def test_all_criteria_passed_raises_on_length_mismatch(self): with pytest.raises(ValueError, match="length mismatch"): result.all_criteria_passed(criteria) + def test_zero_weight_criterion_is_informational_and_does_not_gate(self): + """weight=0 excludes a criterion from the score AND from the pass/fail gate.""" + criteria = [ + FileExistsCriterion(path="f1.txt", description="crit-0", weight=1.0, pass_threshold=0.9), + FileExistsCriterion(path="f2.txt", description="crit-1", weight=0.0, pass_threshold=0.9), + ] + result = _make_result([1.0, 0.0]) # the informational criterion scores zero + + result.calculate_weighted_score(criteria) + assert result.weighted_score == 1.0 # weight-0 contributes to neither term + assert result.all_criteria_passed(criteria) # ...and cannot flip the task to FAILURE + + def test_zero_weight_does_not_rescue_a_failing_gating_criterion(self): + """Only the weight-0 criterion is exempt; real criteria still gate.""" + criteria = [ + FileExistsCriterion(path="f1.txt", description="crit-0", weight=1.0, pass_threshold=0.9), + FileExistsCriterion(path="f2.txt", description="crit-1", weight=0.0, pass_threshold=0.9), + ] + assert not _make_result([0.5, 1.0]).all_criteria_passed(criteria) + + def test_all_zero_weight_criteria_leave_an_empty_gate(self): + """A task of purely informational criteria has nothing to fail on.""" + criteria = [ + FileExistsCriterion(path="f1.txt", description="crit-0", weight=0.0, pass_threshold=0.9), + FileExistsCriterion(path="f2.txt", description="crit-1", weight=0.0, pass_threshold=0.9), + ] + assert _make_result([0.0, 0.0]).all_criteria_passed(criteria) + + def test_zero_weight_cannot_be_armed_for_early_stop(self): + """weight=0 + stop_when is incoherent: it would leave the early-stop gate empty.""" + with pytest.raises(ValidationError, match="weight=0"): + CommandExecutedCriterion( + description="informational + armed", + weight=0.0, + stop_when="pass", + tool_name="Bash", + command_pattern="pytest", + ) + + def test_zero_weight_cannot_declare_suite_thresholds(self): + """weight=0 + suite_thresholds is incoherent: the suite gate drives the run exit code.""" + with pytest.raises(ValidationError, match="weight=0"): + FileExistsCriterion( + path="f1.txt", + description="informational + suite-gated", + weight=0.0, + suite_thresholds={"mean": 0.8}, + ) + def test_empty_inputs_score_zero_without_raising(self): """Empty results/criteria still yield 0.0 (not a raise) and an empty gate passes.""" result = _make_result([]) From b2688756ddac8bbe55bb2469c0442c2557188097 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 22 Jul 2026 14:51:03 -0700 Subject: [PATCH 2/2] fix: render weight:0 criteria as informational on every display surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the follow-up deferred in #34. `final_status` and both exit codes already ignored weight:0 criteria, but the display sites still labeled a below-threshold informational criterion as FAILED — a header-vs-list contradiction where the task reads SUCCESS while a row reads FAIL. The fix is one carried marker rather than four independent re-derivations: - `CriterionResult.gating` (default True) mirrors `BaseSuccessCriterion. is_gating`, stamped by `SuccessChecker` alongside `pass_threshold` on all three construction paths. Defaulting True means task.json files written before this field read back as gating. - checker log line says "below threshold (informational)" / "errored (informational)" instead of FAILED. - reports_html: rows tagged "informational — not gated (weight: 0)"; the "(n/m passed)" header counts gating criteria only and appends "+ k informational". - reports: suite failure-reason sampling skips non-gating criteria — an informational miss must not explain why a row failed. - evalboard: DTO carries `gating` + `passThreshold`; the pill renders INFO (gray) for non-gating criteria, and the criteria header counts them separately. Drive-by: the pill compared `score === 1`, so a fractional criterion passing at 0.95 with threshold 0.9 rendered FAIL — it now compares against the criterion's own threshold. Tests: gating is stamped from the criterion; results default to gating on read-back; the HTML header excludes informational criteria and labels the row; suite sampling omits an informational miss (verified failing without the fix). Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/TASK_DEFINITION_GUIDE.md | 8 +++ evalboard/app/runs/[id]/[...task]/_chips.tsx | 20 ++++++-- .../app/runs/[id]/[...task]/_sections.tsx | 15 +++++- evalboard/lib/runs.ts | 10 ++++ src/coder_eval/evaluation/checker.py | 11 +++- src/coder_eval/models/results.py | 11 ++++ src/coder_eval/reports.py | 4 ++ src/coder_eval/reports_html.py | 17 +++++-- tests/test_evaluator.py | 2 + tests/test_suite_rollup.py | 23 +++++++++ tests/test_threshold_enforcement.py | 51 +++++++++++++++++++ 11 files changed, 159 insertions(+), 13 deletions(-) diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 2fe88b53..d0baa887 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -405,6 +405,14 @@ rendered in reports, but it neither contributes to the score nor fails the task. non-gating criterion for the early-stop or suite gate would let an "informational" check flip a run to failure.) +Every surface labels it as such rather than as a failure: the terminal shows `○` +instead of `✓`/`✗`, the HTML report tags the row "informational — not gated" and +excludes it from the *n*/*m* passed header, the evalboard renders an `INFO` pill, +and a below-threshold informational criterion is never sampled as the reason a +suite row failed. The persisted `CriterionResult.gating` field carries this to +every consumer, so no reader needs the original criterion to know whether a low +score mattered. + **Weighted score:** `weighted_score = sum(score * weight) / sum(weight)` — calculated regardless for quality assessment. ### `file_exists` diff --git a/evalboard/app/runs/[id]/[...task]/_chips.tsx b/evalboard/app/runs/[id]/[...task]/_chips.tsx index cfaa5c4f..09e1c44c 100644 --- a/evalboard/app/runs/[id]/[...task]/_chips.tsx +++ b/evalboard/app/runs/[id]/[...task]/_chips.tsx @@ -26,15 +26,25 @@ export function Expandable({ ); } -export function ResultPill({ passed }: { passed: boolean }) { - const cls = passed - ? "bg-green-50 text-green-700 border-green-200" - : "bg-red-50 text-red-700 border-red-200"; +// `gating: false` (weight: 0) criteria are informational — they cannot fail the +// task, so rendering PASS/FAIL for them would contradict the task's own status. +export function ResultPill({ + passed, + gating = true, +}: { + passed: boolean; + gating?: boolean; +}) { + const cls = !gating + ? "bg-gray-50 text-gray-600 border-gray-200" + : passed + ? "bg-green-50 text-green-700 border-green-200" + : "bg-red-50 text-red-700 border-red-200"; return ( - {passed ? "PASS" : "FAIL"} + {!gating ? "INFO" : passed ? "PASS" : "FAIL"} ); } diff --git a/evalboard/app/runs/[id]/[...task]/_sections.tsx b/evalboard/app/runs/[id]/[...task]/_sections.tsx index 6e7f4407..62a5bae0 100644 --- a/evalboard/app/runs/[id]/[...task]/_sections.tsx +++ b/evalboard/app/runs/[id]/[...task]/_sections.tsx @@ -108,16 +108,27 @@ export function CriteriaSection({ criteria }: { criteria: CriterionResult[] }) {

Success criteria ({criteria.length}) + {criteria.some((c) => !c.gating) && ( + + {criteria.filter((c) => !c.gating).length}{" "} + informational + + )}

{criteria.map((c, i) => { - const passed = c.score === 1; + // Compare against the criterion's own threshold, not === 1: + // fractional criteria pass below 1.0 (default threshold 0.9). + const passed = (c.score ?? 0) >= c.passThreshold; return ( - + {c.description ?? c.criterionType ?? diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index c8538f0d..c1d3f824 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -103,6 +103,12 @@ export interface CriterionResult { score: number | null; details: string | null; error: string | null; + // Mirrors the Python CriterionResult fields. `gating: false` (weight: 0) means + // the criterion is informational — measured, but excluded from the score and + // the pass/fail gate, so it must not render as PASS/FAIL. Both default the way + // pre-existing task.json files behave: gating, threshold 0.9. + passThreshold: number; + gating: boolean; } export interface ElementExecution { @@ -1823,6 +1829,8 @@ export async function readTaskDetail( score?: number; details?: string; error?: string | null; + pass_threshold?: number; + gating?: boolean; }>; iterations?: TurnEntry[]; environment_info?: RawRunJson["environment_info"]; @@ -1836,6 +1844,8 @@ export async function readTaskDetail( score: c.score ?? null, details: c.details ?? null, error: c.error ?? null, + passThreshold: c.pass_threshold ?? 0.9, + gating: c.gating ?? true, })); const artifactRoot = path.join(contentDir, "artifacts"); diff --git a/src/coder_eval/evaluation/checker.py b/src/coder_eval/evaluation/checker.py index 91f3b5da..60515ca0 100644 --- a/src/coder_eval/evaluation/checker.py +++ b/src/coder_eval/evaluation/checker.py @@ -214,12 +214,16 @@ def _check_single( context=context, ) result.pass_threshold = criterion.pass_threshold + result.gating = criterion.is_gating logger.debug(f"Criterion '{criterion_type}' score: {result.score:.2f}") if result.score < criterion.pass_threshold: + # An informational (weight: 0) criterion cannot fail the task, so + # logging it as FAILED contradicts the final status. Say what it is. logger.info( - "Criterion '%s' FAILED (score=%.2f, threshold=%.2f): %s", + "Criterion '%s' %s (score=%.2f, threshold=%.2f): %s", criterion_type, + "FAILED" if criterion.is_gating else "below threshold (informational)", result.score, criterion.pass_threshold, _short_failure_reason(result), @@ -237,6 +241,7 @@ def _check_single( details=f"No checker registered for criterion type '{criterion_type}'", error=f"Unsupported criterion type: '{criterion_type}'", pass_threshold=criterion.pass_threshold, + gating=criterion.is_gating, ) except JudgeInfrastructureError: raise # judge infra failure escalates to FinalStatus.ERROR; do not score it @@ -250,10 +255,12 @@ def _check_single( details="Error running checker", error=str(e), pass_threshold=criterion.pass_threshold, + gating=criterion.is_gating, ) logger.info( - "Criterion '%s' FAILED (score=0.00, threshold=%.2f): %s", + "Criterion '%s' %s (score=0.00, threshold=%.2f): %s", criterion_type, + "FAILED" if criterion.is_gating else "errored (informational)", criterion.pass_threshold, _short_failure_reason(failed), ) diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index dbc1e33a..9a424914 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -76,6 +76,17 @@ class CriterionResult(BaseModel): le=1.0, description="Score required to pass this criterion (mirrors BaseSuccessCriterion.pass_threshold).", ) + gating: bool = Field( + default=True, + description=( + "Whether a below-threshold score here fails the task (mirrors " + "BaseSuccessCriterion.is_gating, i.e. weight > 0). False marks an informational " + "criterion: it is measured and reported but excluded from the score and the " + "pass/fail gate, so every display surface must render it as informational rather " + "than failed. Defaults True so results persisted before this field existed — and " + "any result built without a source criterion — read back as gating." + ), + ) class ClassificationCriterionResult(CriterionResult): diff --git a/src/coder_eval/reports.py b/src/coder_eval/reports.py index 3dac1371..d5cce11f 100644 --- a/src/coder_eval/reports.py +++ b/src/coder_eval/reports.py @@ -815,6 +815,10 @@ def _compute_suite_rollup( break reasons: list[str] = [] for cr in row.result.success_criteria_results: + # Informational (weight: 0) criteria cannot fail a row, so they must + # not supply the reason the row is reported as failed. + if not cr.gating: + continue if cr.error is not None or cr.score < cr.pass_threshold: reason = cr.error or cr.details or f"{cr.criterion_type}: score={cr.score:.2f}" reasons.append(reason[:_FAILURE_REASON_MAX_LEN]) diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py index 57184eff..7f0e9cad 100644 --- a/src/coder_eval/reports_html.py +++ b/src/coder_eval/reports_html.py @@ -575,7 +575,11 @@ def _render_criteria(results: list[CriterionResult], early_stop: EarlyStopInfo | rows: list[str] = [] for cr in results: advisory = "" - if early_stop is not None and f"{cr.criterion_type}: {cr.description}" not in armed: + if not cr.gating: + # weight: 0 — measured but excluded from the gate. Saying so here keeps + # the row from reading as a failure that the header/status contradicts. + advisory = '
informational — not gated (weight: 0)
' + elif early_stop is not None and f"{cr.criterion_type}: {cr.description}" not in armed: advisory = '
advisory — not gated (run stopped early)
' rows.append( f""" @@ -587,10 +591,15 @@ def _render_criteria(results: list[CriterionResult], early_stop: EarlyStopInfo | """ ) - passed = sum(1 for r in results if r.score >= r.pass_threshold) - total = len(results) + # Count over gating criteria only, so this header agrees with final_status + # and the CLI exit code (both of which ignore weight: 0 criteria). + gating_results = [r for r in results if r.gating] + passed = sum(1 for r in gating_results if r.score >= r.pass_threshold) + total = len(gating_results) + informational = len(results) - total + info_note = f' + {informational} informational' if informational else "" return f""" -

Success Criteria ({passed}/{total} passed)

+

Success Criteria ({passed}/{total} passed{info_note})

diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index adf4c23e..8d61cc0d 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -59,6 +59,7 @@ class _Fake: type = "unsupported_type" description = "fake" pass_threshold = 0.42 + is_gating = True result = checker._check_single(_Fake(), reference_code=None) # type: ignore[arg-type] assert result.pass_threshold == 0.42 @@ -206,6 +207,7 @@ def test_success_checker_unsupported_type(): bad_criterion.type = "invalid_type_that_does_not_exist" bad_criterion.description = "Test criterion with unsupported type" bad_criterion.pass_threshold = 0.9 + bad_criterion.is_gating = True # a bare Mock() yields a Mock here, not a bool # Should return failed result instead of raising result = checker.check(bad_criterion) diff --git a/tests/test_suite_rollup.py b/tests/test_suite_rollup.py index 20041251..425ee7d7 100644 --- a/tests/test_suite_rollup.py +++ b/tests/test_suite_rollup.py @@ -139,6 +139,29 @@ def test_mixed_outcomes(self, tmp_path: Path) -> None: ids = sorted(s.row_id for s in rollup.failed_samples if s.row_id is not None) assert ids == ["r2", "r3"] + def test_informational_criterion_is_not_a_failure_reason(self, tmp_path: Path) -> None: + """A weight-0 criterion cannot fail a row, so it must not explain one. + + The row here fails on its gating criterion; the informational one also + scored 0. Only the gating miss may appear in failure_reasons — otherwise + the sample tells a reviewer to go fix a criterion that never gated. + """ + row = _make_row( + suite_id="s", + row_id="r1", + final_status=FinalStatus.FAILURE, + weighted_score=0.0, + criteria=[("file_exists", 0.0, None), ("skill_triggered", 0.0, None)], + ) + row.result.success_criteria_results[1].gating = False + row.result.success_criteria_results[0].details = "gating miss" + row.result.success_criteria_results[1].details = "informational miss" + + rollup = _compute_suite_rollup("s", "v1", [row], tmp_path) + + reasons = rollup.failed_samples[0].failure_reasons + assert reasons == ["gating miss"] + def test_per_criterion_stats_averaging(self, tmp_path: Path) -> None: rows = [ _make_row( diff --git a/tests/test_threshold_enforcement.py b/tests/test_threshold_enforcement.py index fa7a1c39..3f59ffa4 100644 --- a/tests/test_threshold_enforcement.py +++ b/tests/test_threshold_enforcement.py @@ -160,3 +160,54 @@ def test_empty_inputs_score_zero_without_raising(self): result.calculate_weighted_score([]) assert result.weighted_score == 0.0 assert result.all_criteria_passed([]) # all([]) is True + + +class TestInformationalDisplayParity: + """A weight-0 criterion must render as informational everywhere, not as failed. + + ``final_status`` and both exit codes already ignore weight-0 criteria. These + pin the *display* surfaces to the same story, so a report header can't + contradict its own row list. + """ + + def test_checker_stamps_gating_from_the_criterion(self, tmp_path): + """The result mirrors ``is_gating`` so downstream renderers need no criterion.""" + from coder_eval.evaluation.checker import SuccessChecker + from coder_eval.models import SandboxConfig + from coder_eval.sandbox import Sandbox + + sandbox = Sandbox(SandboxConfig(driver="tempdir"), task_id="gating-display-test") + sandbox.sandbox_dir = tmp_path + checker = SuccessChecker(sandbox) + + gating = checker.check(FileExistsCriterion(path="missing.txt", description="gating", weight=1.0)) + informational = checker.check(FileExistsCriterion(path="missing.txt", description="informational", weight=0.0)) + + assert gating.gating is True + assert informational.gating is False + # Both genuinely scored zero — un-gating changes the label, never the measurement. + assert gating.score == 0.0 + assert informational.score == 0.0 + + def test_criterion_result_defaults_to_gating(self): + """Results persisted before the field existed must read back as gating.""" + assert CriterionResult(criterion_type="file_exists", description="d", score=0.0).gating is True + restored = CriterionResult.model_validate_json( + '{"criterion_type": "file_exists", "description": "d", "score": 0.0}' + ) + assert restored.gating is True + + def test_html_report_labels_informational_and_excludes_it_from_the_count(self): + """The HTML header counts gating criteria only, and the row says why.""" + from coder_eval.reports_html import _render_criteria + + html = _render_criteria( + [ + CriterionResult(criterion_type="file_exists", description="real", score=1.0, gating=True), + CriterionResult(criterion_type="file_exists", description="info", score=0.0, gating=False), + ] + ) + + assert "(1/1 passed" in html # NOT 1/2 — the informational miss isn't a failure + assert "informational — not gated (weight: 0)" in html + assert "1 informational" in html