{criteria.map((c, i) => {
- const passed = c.score === 1;
+ // Compare against the criterion's own threshold, not === 1:
+ // fractional criteria pass below 1.0 (default threshold 0.9).
+ const passed = (c.score ?? 0) >= c.passThreshold;
return (
-
+
{c.description ??
c.criterionType ??
diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts
index c8538f0d..c1d3f824 100644
--- a/evalboard/lib/runs.ts
+++ b/evalboard/lib/runs.ts
@@ -103,6 +103,12 @@ export interface CriterionResult {
score: number | null;
details: string | null;
error: string | null;
+ // Mirrors the Python CriterionResult fields. `gating: false` (weight: 0) means
+ // the criterion is informational — measured, but excluded from the score and
+ // the pass/fail gate, so it must not render as PASS/FAIL. Both default the way
+ // pre-existing task.json files behave: gating, threshold 0.9.
+ passThreshold: number;
+ gating: boolean;
}
export interface ElementExecution {
@@ -1823,6 +1829,8 @@ export async function readTaskDetail(
score?: number;
details?: string;
error?: string | null;
+ pass_threshold?: number;
+ gating?: boolean;
}>;
iterations?: TurnEntry[];
environment_info?: RawRunJson["environment_info"];
@@ -1836,6 +1844,8 @@ export async function readTaskDetail(
score: c.score ?? null,
details: c.details ?? null,
error: c.error ?? null,
+ passThreshold: c.pass_threshold ?? 0.9,
+ gating: c.gating ?? true,
}));
const artifactRoot = path.join(contentDir, "artifacts");
diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py
index bad1d9c3..1e62d886 100644
--- a/src/coder_eval/cli/evaluate_command.py
+++ b/src/coder_eval/cli/evaluate_command.py
@@ -130,7 +130,12 @@ async def _setup_and_run() -> EvaluationResult:
raise typer.Exit(1)
for criterion, cr in zip(task.success_criteria, criteria_results, strict=True):
- status = "[green]✓[/green]" if cr.score >= criterion.pass_threshold else "[red]✗[/red]"
+ if not criterion.is_gating:
+ # weight=0 is informational: it cannot pass/fail the task, so don't
+ # render it as ✓/✗ (that would contradict the gate and the exit code).
+ status = "[dim]○[/dim]"
+ else:
+ status = "[green]✓[/green]" if cr.score >= criterion.pass_threshold else "[red]✗[/red]"
console.print(f"{status} {cr.criterion_type}")
console.print(f" [dim]{cr.description}[/dim]")
console.print(f" [dim]Score: {cr.score:.2f}[/dim]")
@@ -140,15 +145,19 @@ async def _setup_and_run() -> EvaluationResult:
console.print(f" [red]Error: {cr.error}[/red]")
console.print()
- passed = sum(
- 1 for cr, c in zip(criteria_results, task.success_criteria, strict=True) if cr.score >= c.pass_threshold
- )
- total = len(task.success_criteria)
+ # Gate over gating criteria only (weight=0 is informational and cannot fail
+ # the task) so this summary + the exit code below match final_status.
+ gating = [(cr, c) for cr, c in zip(criteria_results, task.success_criteria, strict=True) if c.is_gating]
+ passed = sum(1 for cr, c in gating if cr.score >= c.pass_threshold)
+ total = len(gating)
failed = total - passed
+ informational = len(task.success_criteria) - total
console.print("[bold]Summary:[/bold]")
console.print(f" Passed: {passed}/{total}")
console.print(f" Failed: {failed}/{total}")
+ if informational:
+ console.print(f" [dim]Informational (weight=0, not gated): {informational}[/dim]")
console.print(f"\n[dim]Run directory: {prepared_run_dir}[/dim]")
if result.sandbox_path:
console.print(f"[dim]Artifacts: {result.sandbox_path}[/dim]")
diff --git a/src/coder_eval/evaluation/checker.py b/src/coder_eval/evaluation/checker.py
index 91f3b5da..60515ca0 100644
--- a/src/coder_eval/evaluation/checker.py
+++ b/src/coder_eval/evaluation/checker.py
@@ -214,12 +214,16 @@ def _check_single(
context=context,
)
result.pass_threshold = criterion.pass_threshold
+ result.gating = criterion.is_gating
logger.debug(f"Criterion '{criterion_type}' score: {result.score:.2f}")
if result.score < criterion.pass_threshold:
+ # An informational (weight: 0) criterion cannot fail the task, so
+ # logging it as FAILED contradicts the final status. Say what it is.
logger.info(
- "Criterion '%s' FAILED (score=%.2f, threshold=%.2f): %s",
+ "Criterion '%s' %s (score=%.2f, threshold=%.2f): %s",
criterion_type,
+ "FAILED" if criterion.is_gating else "below threshold (informational)",
result.score,
criterion.pass_threshold,
_short_failure_reason(result),
@@ -237,6 +241,7 @@ def _check_single(
details=f"No checker registered for criterion type '{criterion_type}'",
error=f"Unsupported criterion type: '{criterion_type}'",
pass_threshold=criterion.pass_threshold,
+ gating=criterion.is_gating,
)
except JudgeInfrastructureError:
raise # judge infra failure escalates to FinalStatus.ERROR; do not score it
@@ -250,10 +255,12 @@ def _check_single(
details="Error running checker",
error=str(e),
pass_threshold=criterion.pass_threshold,
+ gating=criterion.is_gating,
)
logger.info(
- "Criterion '%s' FAILED (score=0.00, threshold=%.2f): %s",
+ "Criterion '%s' %s (score=0.00, threshold=%.2f): %s",
criterion_type,
+ "FAILED" if criterion.is_gating else "errored (informational)",
criterion.pass_threshold,
_short_failure_reason(failed),
)
diff --git a/src/coder_eval/models/criteria.py b/src/coder_eval/models/criteria.py
index 1bb59c8f..8786ac86 100644
--- a/src/coder_eval/models/criteria.py
+++ b/src/coder_eval/models/criteria.py
@@ -9,7 +9,7 @@
from __future__ import annotations
from abc import ABC
-from typing import Annotated, Any, ClassVar, Literal
+from typing import Annotated, Any, ClassVar, Literal, Self
from pydantic import BaseModel, ConfigDict, Field, model_validator
@@ -101,10 +101,11 @@ class BaseSuccessCriterion(BaseModel, ABC):
ge=0.0,
description=(
"Relative importance of this criterion in the weighted score (default: 1.0). Set to 0 to "
- "exclude it from the weighted SCORE -- useful for informational or side-effect checks "
- "(e.g. a setup command). NOTE: weight=0 excludes from the score but NOT from the pass/fail "
- "gate -- a criterion scoring below its pass_threshold still flips the task to FAILURE "
- "regardless of weight. To make a criterion truly non-gating, also set pass_threshold=0."
+ "make the criterion purely INFORMATIONAL -- useful for side-effect checks (e.g. a setup "
+ "command): it is excluded from the weighted score AND from the pass/fail gate, so scoring "
+ "below its pass_threshold no longer flips the task to FAILURE. The result is still "
+ "computed, stored, and rendered in reports. A weight=0 criterion may not set stop_when "
+ "or suite_thresholds (arming a non-gating criterion for a pass/fail gate is incoherent)."
),
)
@@ -145,6 +146,39 @@ def model_post_init(self, context: Any, /) -> None:
# the discriminated union rejects the dump with union_tag_not_found.
self.__pydantic_fields_set__.add("type")
+ @model_validator(mode="after")
+ def check_weight_zero_is_not_gating(self) -> Self:
+ """Reject ``weight: 0`` combined with any gate-arming field.
+
+ ``weight: 0`` makes a criterion informational — excluded from the score
+ and from the pass/fail gate (see ``is_gating``). Both ``stop_when`` (arms
+ the per-row early-stop gate) and ``suite_thresholds`` (arms the
+ across-row suite gate, which drives the run's exit code) would let an
+ "informational" criterion flip a run to failure — directly contradicting
+ the field's contract. So both combinations are authoring errors, caught
+ at load time rather than surfacing as a confusing exit code later.
+ """
+ if self.weight == 0.0:
+ for field, name in (("stop_when", "stop_when"), ("suite_thresholds", "suite_thresholds")):
+ if getattr(self, field) is not None:
+ raise ValueError(
+ f"criterion {self.type!r}: weight=0 makes the criterion informational (non-gating), "
+ + f"so it cannot also set {name} (which arms it for a pass/fail gate). "
+ + f"Give it a non-zero weight, or drop {name}."
+ )
+ return self
+
+ @property
+ def is_gating(self) -> bool:
+ """True when this criterion participates in the task's pass/fail gate.
+
+ Single source of truth for "does a low score here fail the task". A
+ ``weight: 0`` criterion is informational: it is excluded from the
+ weighted score (it contributes 0 to both numerator and denominator) and,
+ by the same token, from the gate — so the two never disagree.
+ """
+ return self.weight > 0.0
+
# Business logic (check operations) moved to SuccessChecker in evaluator.py
diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py
index 05dbbb1d..9a424914 100644
--- a/src/coder_eval/models/results.py
+++ b/src/coder_eval/models/results.py
@@ -76,6 +76,17 @@ class CriterionResult(BaseModel):
le=1.0,
description="Score required to pass this criterion (mirrors BaseSuccessCriterion.pass_threshold).",
)
+ gating: bool = Field(
+ default=True,
+ description=(
+ "Whether a below-threshold score here fails the task (mirrors "
+ "BaseSuccessCriterion.is_gating, i.e. weight > 0). False marks an informational "
+ "criterion: it is measured and reported but excluded from the score and the "
+ "pass/fail gate, so every display surface must render it as informational rather "
+ "than failed. Defaults True so results persisted before this field existed — and "
+ "any result built without a source criterion — read back as gating."
+ ),
+ )
class ClassificationCriterionResult(CriterionResult):
@@ -593,20 +604,30 @@ def calculate_weighted_score(self, criteria: list[SuccessCriterion]) -> None:
self.weighted_score = total_weighted_score / total_weight if total_weight > 0 else 0.0
def all_criteria_passed(self, criteria: list[SuccessCriterion]) -> bool:
- """True iff every criterion result meets its pass_threshold.
-
- Single source of truth for the success gate. A results/criteria length
- mismatch raises ``ValueError`` rather than silently truncating — the
- ``len()`` pre-check is required because ``all()`` short-circuits on the
- first failing pair, so ``zip(strict=True)`` alone would not reliably
- reach the length check.
+ """True iff every GATING criterion result meets its pass_threshold.
+
+ Single source of truth for the success gate. ``weight: 0`` criteria are
+ informational (``BaseSuccessCriterion.is_gating`` is False): they are
+ excluded from the weighted score, so they are excluded from the gate
+ too — otherwise a criterion that contributes nothing to the score could
+ still single-handedly flip the task to FAILURE. A task whose criteria
+ are ALL weight-0 has an empty gate and therefore passes.
+
+ A results/criteria length mismatch raises ``ValueError`` rather than
+ silently truncating — the ``len()`` pre-check is required because
+ ``all()`` short-circuits on the first failing pair, so
+ ``zip(strict=True)`` alone would not reliably reach the length check.
"""
if len(self.success_criteria_results) != len(criteria):
raise ValueError(
f"Results/criteria length mismatch for task {self.task_id}: "
+ f"{len(self.success_criteria_results)} results vs {len(criteria)} criteria."
)
- return all(r.score >= c.pass_threshold for r, c in zip(self.success_criteria_results, criteria, strict=True))
+ return all(
+ r.score >= c.pass_threshold
+ for r, c in zip(self.success_criteria_results, criteria, strict=True)
+ if c.is_gating
+ )
def armed_criteria_passed(self, criteria: list[SuccessCriterion]) -> bool:
"""True iff every ARMED criterion (``stop_when`` set) meets its pass_threshold.
@@ -618,7 +639,9 @@ def armed_criteria_passed(self, criteria: list[SuccessCriterion]) -> bool:
pre-check as ``all_criteria_passed`` so the gate logic stays
single-sourced. Raises ``ValueError`` on an empty armed set — unreachable
when a stop actually fired (a stop requires an armed criterion), so this
- is a defensive guard against misuse.
+ is a defensive guard against misuse. No ``is_gating`` filter is needed
+ here: ``BaseSuccessCriterion`` rejects ``weight: 0`` together with
+ ``stop_when``, so every armed criterion is gating by construction.
"""
if len(self.success_criteria_results) != len(criteria):
raise ValueError(
diff --git a/src/coder_eval/models/sandbox.py b/src/coder_eval/models/sandbox.py
index bf86106e..282cdbdc 100644
--- a/src/coder_eval/models/sandbox.py
+++ b/src/coder_eval/models/sandbox.py
@@ -300,8 +300,9 @@ class SandboxConfig(BaseModel):
network, process table, and filesystem outside the temp dir.
``driver: docker`` runs each task inside its own container; see
:class:`DockerDriverConfig` for knobs. ``ResourceLimits.timeout`` is the
- only limit enforced in tempdir mode; under docker, ``max_memory_mb`` also
- maps to ``--memory`` when set.
+ only limit enforced in tempdir mode; under docker, ``max_memory_mb`` /
+ ``max_cpus`` / ``max_pids`` also map to ``--memory`` / ``--cpus`` /
+ ``--pids-limit`` when set.
"""
model_config = ConfigDict(populate_by_name=True, extra="forbid")
diff --git a/src/coder_eval/reports.py b/src/coder_eval/reports.py
index 6e5b9deb..d5cce11f 100644
--- a/src/coder_eval/reports.py
+++ b/src/coder_eval/reports.py
@@ -815,7 +815,11 @@ def _compute_suite_rollup(
break
reasons: list[str] = []
for cr in row.result.success_criteria_results:
- if cr.error is not None or cr.score < 1.0:
+ # Informational (weight: 0) criteria cannot fail a row, so they must
+ # not supply the reason the row is reported as failed.
+ if not cr.gating:
+ continue
+ if cr.error is not None or cr.score < cr.pass_threshold:
reason = cr.error or cr.details or f"{cr.criterion_type}: score={cr.score:.2f}"
reasons.append(reason[:_FAILURE_REASON_MAX_LEN])
if len(reasons) >= _FAILURE_REASONS_PER_ROW:
diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py
index 57184eff..7f0e9cad 100644
--- a/src/coder_eval/reports_html.py
+++ b/src/coder_eval/reports_html.py
@@ -575,7 +575,11 @@ def _render_criteria(results: list[CriterionResult], early_stop: EarlyStopInfo |
rows: list[str] = []
for cr in results:
advisory = ""
- if early_stop is not None and f"{cr.criterion_type}: {cr.description}" not in armed:
+ if not cr.gating:
+ # weight: 0 — measured but excluded from the gate. Saying so here keeps
+ # the row from reading as a failure that the header/status contradicts.
+ advisory = 'informational — not gated (weight: 0)
'
+ elif early_stop is not None and f"{cr.criterion_type}: {cr.description}" not in armed:
advisory = 'advisory — not gated (run stopped early)
'
rows.append(
f"""
@@ -587,10 +591,15 @@ def _render_criteria(results: list[CriterionResult], early_stop: EarlyStopInfo |
"""
)
- passed = sum(1 for r in results if r.score >= r.pass_threshold)
- total = len(results)
+ # Count over gating criteria only, so this header agrees with final_status
+ # and the CLI exit code (both of which ignore weight: 0 criteria).
+ gating_results = [r for r in results if r.gating]
+ passed = sum(1 for r in gating_results if r.score >= r.pass_threshold)
+ total = len(gating_results)
+ informational = len(results) - total
+ info_note = f' + {informational} informational' if informational else ""
return f"""
-Success Criteria ({passed}/{total} passed)
+Success Criteria ({passed}/{total} passed{info_note})
diff --git a/tests/fixtures/tasks/test_task_informational_criterion.yaml b/tests/fixtures/tasks/test_task_informational_criterion.yaml
new file mode 100644
index 00000000..98a0f0df
--- /dev/null
+++ b/tests/fixtures/tasks/test_task_informational_criterion.yaml
@@ -0,0 +1,17 @@
+task_id: test_task_informational
+description: Gating criterion passes; a weight=0 informational criterion fails.
+initial_prompt: Test
+# agent block required by TaskDefinition schema but unused in evaluate-only mode
+agent:
+ type: claude-code
+sandbox:
+ driver: tempdir
+ python: null
+success_criteria:
+ - type: file_exists
+ path: app.py
+ description: app.py must exist (gating)
+ - type: file_exists
+ path: missing.py
+ description: informational side-effect check (weight=0, must not gate)
+ weight: 0.0
diff --git a/tests/test_evaluate_command.py b/tests/test_evaluate_command.py
index 24fd16ea..67a98046 100644
--- a/tests/test_evaluate_command.py
+++ b/tests/test_evaluate_command.py
@@ -132,6 +132,27 @@ def test_evaluate_command_failure(tmp_path):
assert exc_info.value.exit_code == 1
+def test_evaluate_command_informational_criterion_does_not_fail_exit(tmp_path):
+ """A failing weight=0 criterion must not flip the exit code (it is non-gating)."""
+ task_file = FIXTURES_DIR / "tasks" / "test_task_informational_criterion.yaml"
+
+ work_dir = tmp_path / "work"
+ work_dir.mkdir()
+ (work_dir / "app.py").write_text("print('hello')") # gating criterion passes; missing.py absent
+
+ from coder_eval.cli.evaluate_command import evaluate_command
+
+ run_dir = tmp_path / "run"
+ run_dir.mkdir()
+
+ with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"):
+ with pytest.raises(typer.Exit) as exc_info:
+ evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir)
+
+ # The only gating criterion passed → exit 0, despite the weight=0 miss.
+ assert exc_info.value.exit_code == 0
+
+
def test_evaluate_command_invalid_task_file(tmp_path):
"""Test evaluate command with invalid task file."""
# Use non-existent task file
diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py
index adf4c23e..8d61cc0d 100644
--- a/tests/test_evaluator.py
+++ b/tests/test_evaluator.py
@@ -59,6 +59,7 @@ class _Fake:
type = "unsupported_type"
description = "fake"
pass_threshold = 0.42
+ is_gating = True
result = checker._check_single(_Fake(), reference_code=None) # type: ignore[arg-type]
assert result.pass_threshold == 0.42
@@ -206,6 +207,7 @@ def test_success_checker_unsupported_type():
bad_criterion.type = "invalid_type_that_does_not_exist"
bad_criterion.description = "Test criterion with unsupported type"
bad_criterion.pass_threshold = 0.9
+ bad_criterion.is_gating = True # a bare Mock() yields a Mock here, not a bool
# Should return failed result instead of raising
result = checker.check(bad_criterion)
diff --git a/tests/test_suite_rollup.py b/tests/test_suite_rollup.py
index 20041251..425ee7d7 100644
--- a/tests/test_suite_rollup.py
+++ b/tests/test_suite_rollup.py
@@ -139,6 +139,29 @@ def test_mixed_outcomes(self, tmp_path: Path) -> None:
ids = sorted(s.row_id for s in rollup.failed_samples if s.row_id is not None)
assert ids == ["r2", "r3"]
+ def test_informational_criterion_is_not_a_failure_reason(self, tmp_path: Path) -> None:
+ """A weight-0 criterion cannot fail a row, so it must not explain one.
+
+ The row here fails on its gating criterion; the informational one also
+ scored 0. Only the gating miss may appear in failure_reasons — otherwise
+ the sample tells a reviewer to go fix a criterion that never gated.
+ """
+ row = _make_row(
+ suite_id="s",
+ row_id="r1",
+ final_status=FinalStatus.FAILURE,
+ weighted_score=0.0,
+ criteria=[("file_exists", 0.0, None), ("skill_triggered", 0.0, None)],
+ )
+ row.result.success_criteria_results[1].gating = False
+ row.result.success_criteria_results[0].details = "gating miss"
+ row.result.success_criteria_results[1].details = "informational miss"
+
+ rollup = _compute_suite_rollup("s", "v1", [row], tmp_path)
+
+ reasons = rollup.failed_samples[0].failure_reasons
+ assert reasons == ["gating miss"]
+
def test_per_criterion_stats_averaging(self, tmp_path: Path) -> None:
rows = [
_make_row(
diff --git a/tests/test_threshold_enforcement.py b/tests/test_threshold_enforcement.py
index ec789805..3f59ffa4 100644
--- a/tests/test_threshold_enforcement.py
+++ b/tests/test_threshold_enforcement.py
@@ -9,8 +9,10 @@
from datetime import datetime
import pytest
+from pydantic import ValidationError
from coder_eval.models import (
+ CommandExecutedCriterion,
CriterionResult,
EvaluationResult,
FileExistsCriterion,
@@ -103,9 +105,109 @@ def test_all_criteria_passed_raises_on_length_mismatch(self):
with pytest.raises(ValueError, match="length mismatch"):
result.all_criteria_passed(criteria)
+ def test_zero_weight_criterion_is_informational_and_does_not_gate(self):
+ """weight=0 excludes a criterion from the score AND from the pass/fail gate."""
+ criteria = [
+ FileExistsCriterion(path="f1.txt", description="crit-0", weight=1.0, pass_threshold=0.9),
+ FileExistsCriterion(path="f2.txt", description="crit-1", weight=0.0, pass_threshold=0.9),
+ ]
+ result = _make_result([1.0, 0.0]) # the informational criterion scores zero
+
+ result.calculate_weighted_score(criteria)
+ assert result.weighted_score == 1.0 # weight-0 contributes to neither term
+ assert result.all_criteria_passed(criteria) # ...and cannot flip the task to FAILURE
+
+ def test_zero_weight_does_not_rescue_a_failing_gating_criterion(self):
+ """Only the weight-0 criterion is exempt; real criteria still gate."""
+ criteria = [
+ FileExistsCriterion(path="f1.txt", description="crit-0", weight=1.0, pass_threshold=0.9),
+ FileExistsCriterion(path="f2.txt", description="crit-1", weight=0.0, pass_threshold=0.9),
+ ]
+ assert not _make_result([0.5, 1.0]).all_criteria_passed(criteria)
+
+ def test_all_zero_weight_criteria_leave_an_empty_gate(self):
+ """A task of purely informational criteria has nothing to fail on."""
+ criteria = [
+ FileExistsCriterion(path="f1.txt", description="crit-0", weight=0.0, pass_threshold=0.9),
+ FileExistsCriterion(path="f2.txt", description="crit-1", weight=0.0, pass_threshold=0.9),
+ ]
+ assert _make_result([0.0, 0.0]).all_criteria_passed(criteria)
+
+ def test_zero_weight_cannot_be_armed_for_early_stop(self):
+ """weight=0 + stop_when is incoherent: it would leave the early-stop gate empty."""
+ with pytest.raises(ValidationError, match="weight=0"):
+ CommandExecutedCriterion(
+ description="informational + armed",
+ weight=0.0,
+ stop_when="pass",
+ tool_name="Bash",
+ command_pattern="pytest",
+ )
+
+ def test_zero_weight_cannot_declare_suite_thresholds(self):
+ """weight=0 + suite_thresholds is incoherent: the suite gate drives the run exit code."""
+ with pytest.raises(ValidationError, match="weight=0"):
+ FileExistsCriterion(
+ path="f1.txt",
+ description="informational + suite-gated",
+ weight=0.0,
+ suite_thresholds={"mean": 0.8},
+ )
+
def test_empty_inputs_score_zero_without_raising(self):
"""Empty results/criteria still yield 0.0 (not a raise) and an empty gate passes."""
result = _make_result([])
result.calculate_weighted_score([])
assert result.weighted_score == 0.0
assert result.all_criteria_passed([]) # all([]) is True
+
+
+class TestInformationalDisplayParity:
+ """A weight-0 criterion must render as informational everywhere, not as failed.
+
+ ``final_status`` and both exit codes already ignore weight-0 criteria. These
+ pin the *display* surfaces to the same story, so a report header can't
+ contradict its own row list.
+ """
+
+ def test_checker_stamps_gating_from_the_criterion(self, tmp_path):
+ """The result mirrors ``is_gating`` so downstream renderers need no criterion."""
+ from coder_eval.evaluation.checker import SuccessChecker
+ from coder_eval.models import SandboxConfig
+ from coder_eval.sandbox import Sandbox
+
+ sandbox = Sandbox(SandboxConfig(driver="tempdir"), task_id="gating-display-test")
+ sandbox.sandbox_dir = tmp_path
+ checker = SuccessChecker(sandbox)
+
+ gating = checker.check(FileExistsCriterion(path="missing.txt", description="gating", weight=1.0))
+ informational = checker.check(FileExistsCriterion(path="missing.txt", description="informational", weight=0.0))
+
+ assert gating.gating is True
+ assert informational.gating is False
+ # Both genuinely scored zero — un-gating changes the label, never the measurement.
+ assert gating.score == 0.0
+ assert informational.score == 0.0
+
+ def test_criterion_result_defaults_to_gating(self):
+ """Results persisted before the field existed must read back as gating."""
+ assert CriterionResult(criterion_type="file_exists", description="d", score=0.0).gating is True
+ restored = CriterionResult.model_validate_json(
+ '{"criterion_type": "file_exists", "description": "d", "score": 0.0}'
+ )
+ assert restored.gating is True
+
+ def test_html_report_labels_informational_and_excludes_it_from_the_count(self):
+ """The HTML header counts gating criteria only, and the row says why."""
+ from coder_eval.reports_html import _render_criteria
+
+ html = _render_criteria(
+ [
+ CriterionResult(criterion_type="file_exists", description="real", score=1.0, gating=True),
+ CriterionResult(criterion_type="file_exists", description="info", score=0.0, gating=False),
+ ]
+ )
+
+ assert "(1/1 passed" in html # NOT 1/2 — the informational miss isn't a failure
+ assert "informational — not gated (weight: 0)" in html
+ assert "1 informational" in html