feat(criteria): pass_context for run_command + node_modules/.bin PATH hardening

CLAUDE.md

@@ -149,7 +149,7 @@ templates/                         # Sandbox template directories
 | `file_contains` | Fractional | String presence/absence |
 | `file_check` | Fractional | Unified file existence + content + regex check |
 | `json_check` | Fractional | JSON validation + JSON Schema + JMESPath assertions |
-| `run_command` | Binary / Continuous | Command exit code + optional stdout matching or float scoring |
+| `run_command` | Binary / Continuous | Command exit code + optional stdout matching or float scoring; opt-in `pass_context: true` exposes the in-flight `EvaluationResult` (task.json schema) at `$CODER_EVAL_CONTEXT` for the scoring script |
 | `file_matches_regex` | Binary | Regex match on file |
 | `reference_comparison` | Continuous | AST/token/complexity similarity |
 | `command_executed` | Fractional | Agent tool usage verification |

docs/TASK_DEFINITION_GUIDE.md

@@ -539,6 +539,13 @@ Runs a command and checks the exit code, with optional stdout matching. **Binary
   expected_stdout: "Hello, World!"    # Optional: check stdout content
   stdout_match: "exact"               # "exact" (default), "contains", or "regex"
   description: "Script must output the correct text"
+
+# Give the scoring script the agent trajectory + resolved config
+- type: "run_command"
+  command: "python score.py"          # reads json.load(open(os.environ["CODER_EVAL_CONTEXT"]))
+  score_from_stdout: true
+  pass_context: true                  # expose the in-flight record at $CODER_EVAL_CONTEXT
+  description: "Score derived from the trajectory"
 ```
 
 | Field | Default | Description |
@@ -548,6 +555,45 @@ Runs a command and checks the exit code, with optional stdout matching. **Binary
 | `expected_exit_code` | 0 | Expected exit code |
 | `expected_stdout` | `null` | When set, stdout is also checked |
 | `stdout_match` | `"exact"` | Match mode: `exact` (stripped), `contains` (substring), `regex` (pattern) |
+| `pass_context` | `false` | Expose the in-flight evaluation record as JSON at `$CODER_EVAL_CONTEXT` (see below) |
+
+#### `pass_context` — the evaluation record as `$CODER_EVAL_CONTEXT`
+
+With `pass_context: true`, the command runs with an env var `CODER_EVAL_CONTEXT`
+pointing at a JSON file. Its schema is **exactly** `task.json` (an
+`EvaluationResult`): `iterations` (the full agent trajectory, including
+`commands` and `messages`), `task_config` (the resolved `TaskDefinition` plus
+per-key merge lineage), `command_stats`, `total_token_usage`, `agent_config`,
+`environment_info`, and `early_stop`. This lets a scoring script see the
+trajectory and resolved config, not just the filesystem.
+
+```python
+# score.py
+import json, os
+
+ctx = json.load(open(os.environ["CODER_EVAL_CONTEXT"]))
+print(min(1.0, len(ctx["iterations"]) / 3))   # first line = the score
+```
+
+Because the payload is the `task.json` format, you can **develop the script
+offline** against a real `task.json` from a previous run — zero agent runs, zero
+API cost — then wire the same script in unchanged.
+
+Two rules to respect:
+
+- **`success_criteria_results` is always `[]`** in this payload. Criteria are
+  checked in list order, so exposing already-computed verdicts would make a
+  script silently depend on its position in the YAML. Do not read it.
+- Fields finalized only *after* the criteria phase are provisional: don't read
+  them. `weighted_score`, `completed_at`, and `duration_seconds` are `null`, and
+  `final_status` is still a placeholder (typically `"FAILURE"`), not the real
+  outcome. Everything else is fully populated. On the evaluate-only path (no
+  agent), `iterations` is `[]` but `task_config` is present, so tolerate an
+  empty `iterations`.
+
+The file is written to a private temp dir (never inside the sandbox work dir, so
+it is not archived with preserved sandboxes) and deleted immediately after the
+command completes.
 
 ### `file_matches_regex`
 

src/coder_eval/criteria/base.py

@@ -17,7 +17,7 @@
 if TYPE_CHECKING:
     from pathlib import Path
 
-    from coder_eval.models.results import TurnRecord
+    from coder_eval.models.results import EvaluationResult, TurnRecord
     from coder_eval.models.routing import ApiRoute
     from coder_eval.sandbox import Sandbox
 
@@ -39,10 +39,19 @@ class CheckContext:
 
     Non-judge checkers receive it too (uniform ``_check_impl`` signature) and
     ignore it.
+
+    ``run_result`` carries the *in-flight* :class:`EvaluationResult` — the same
+    object the orchestrator is populating for the current attempt, not a finished
+    record. It is present for the criteria phase (its ``iterations`` /
+    ``task_config`` are set), but fields computed after checking
+    (``weighted_score`` / ``final_status`` / ``completed_at``) are still
+    provisional. ``run_command`` with ``pass_context=true`` serializes it for the
+    scoring script; other checkers ignore it.
     """
 
     route: "ApiRoute | None" = None
     reference_dir: "Path | None" = None
+    run_result: "EvaluationResult | None" = None
 
 
 def handle_criterion_errors(func: Callable[..., CriterionResult]) -> Callable[..., CriterionResult]:

src/coder_eval/criteria/run_command.py

@@ -3,18 +3,37 @@
 import logging
 import math
 import re
+import tempfile
+from pathlib import Path
 from typing import TYPE_CHECKING
 
 from coder_eval.criteria.base import BaseCriterion, CheckContext, register_criterion
 from coder_eval.models import CriterionResult, RunCommandCriterion
 
 
 if TYPE_CHECKING:
-    from coder_eval.models.results import TurnRecord
+    from coder_eval.models.results import EvaluationResult, TurnRecord
     from coder_eval.sandbox import Sandbox
 
 logger = logging.getLogger(__name__)
 
+# Env var name pointing the scoring command at the serialized context file.
+# Mirrors the framework-owned TASK_DIR convention (sandbox._build_run_command_env).
+_CONTEXT_ENV_VAR = "CODER_EVAL_CONTEXT"
+
+
+def _serialize_context(run_result: "EvaluationResult") -> str:
+    """Render the in-flight evaluation record as task.json-shaped JSON.
+
+    ``success_criteria_results`` is force-cleared: criteria are checked in list
+    order, so leaking already-computed verdicts would make scripts silently
+    depend on their position in the task YAML — and in simulation every_turn
+    mode the field still holds the *previous* turn's results. ``model_copy`` is a
+    shallow copy, so the large ``iterations`` list is shared, not duplicated, and
+    the caller's ``run_result`` is never mutated.
+    """
+    return run_result.model_copy(update={"success_criteria_results": []}).model_dump_json()
+
 
 # Per-stream output budget included in criterion details. Large enough for
 # typical tool diagnostics (e.g. uip --output json, pytest tracebacks) while
@@ -65,7 +84,29 @@
     ) -> CriterionResult:
         """Execute command and dispatch to the appropriate scoring method."""
         logger.debug(f"Running command for criterion '{criterion.description}': {criterion.command}")
-        exit_code, stdout, stderr = sandbox.run_command(criterion.command, timeout=criterion.timeout)
+
+        if not criterion.pass_context:
+            exit_code, stdout, stderr = sandbox.run_command(criterion.command, timeout=criterion.timeout)
+        else:
+            run_result = context.run_result if context is not None else None
+            if run_result is None:
+                raise ValueError(
+                    "pass_context=true requires the in-flight EvaluationResult, but none was "
+                    + "supplied. It is threaded from the Orchestrator via CheckContext.run_result; "
+                    + "a SuccessChecker constructed outside a run must pass run_result= explicitly."
+                )
+            # tempfile (not the sandbox work dir): preserved sandboxes are archived
+            # to blob storage, and a context file dropped in the work dir would ride
+            # along. The with-block deletes the dir on every exit path, including the
+            # timeout return (run_command returns (-1, "", msg), it does not raise).
+            with tempfile.TemporaryDirectory(prefix="coder-eval-ctx-") as tmpdir:
+                context_path = Path(tmpdir) / "context.json"
+                context_path.write_text(_serialize_context(run_result), encoding="utf-8")
+                exit_code, stdout, stderr = sandbox.run_command(
+                    criterion.command,
+                    timeout=criterion.timeout,
+                    extra_env={_CONTEXT_ENV_VAR: str(context_path)},
+                )
 
         if criterion.score_from_stdout:
             return self._score_from_stdout(criterion, exit_code, stdout, stderr)

src/coder_eval/evaluation/checker.py

@@ -12,7 +12,14 @@
 from ..criteria import BaseCriterion, CriterionRegistry, init_criteria
 from ..criteria.base import CheckContext
 from ..errors import JudgeInfrastructureError
-from ..models import CriteriaResults, CriterionResult, SuccessCriteria, SuccessCriterion, TurnRecords
+from ..models import (
+    CriteriaResults,
+    CriterionResult,
+    EvaluationResult,
+    SuccessCriteria,
+    SuccessCriterion,
+    TurnRecords,
+)
 from ..sandbox import Sandbox
 
 
@@ -93,6 +100,10 @@ def __init__(
         self._reference_dir: Path | None = None
         # Cached turn records - set by check()/check_all() when provided
         self._turn_records: TurnRecords | None = None
+        # Cached in-flight EvaluationResult - set by check()/check_all() when provided.
+        # Forwarded to run_command's pass_context=true so a scoring script can read the
+        # trajectory + resolved config; ignored by every other criterion.
+        self._run_result: EvaluationResult | None = None
         self.route = route
 
         # V3: Lazy initialization - registry loaded here, not at import
@@ -105,6 +116,7 @@ def check(
         reference_code: str | None = None,
         turn_records: TurnRecords | None = None,
         reference_dir: Path | None = None,
+        run_result: EvaluationResult | None = None,
     ) -> CriterionResult:
         """Check a single criterion (backward compatibility wrapper).
 
@@ -116,6 +128,9 @@ def check(
             reference_dir: Optional resolved path to a reference directory
                 (from ``task.reference.directory``). Only consumed by
                 ``agent_judge``; non-judge criteria ignore it.
+            run_result: Optional in-flight :class:`EvaluationResult`. Only
+                consumed by ``run_command`` with ``pass_context=true``; other
+                criteria ignore it.
 
         Returns:
             CriterionResult with score
@@ -127,17 +142,21 @@ def check(
             self._reference_dir = reference_dir
         if turn_records is not None:
             self._turn_records = turn_records
+        if run_result is not None:
+            self._run_result = run_result
         ref_code = reference_code if reference_code is not None else self._reference_code
         ref_dir = reference_dir if reference_dir is not None else self._reference_dir
         records = turn_records if turn_records is not None else self._turn_records
-        return self._check_single(criterion, ref_code, records, ref_dir)
+        result = run_result if run_result is not None else self._run_result
+        return self._check_single(criterion, ref_code, records, ref_dir, result)
 
     def check_all(
         self,
         criteria: SuccessCriteria,
         reference_code: str | None = None,
         turn_records: TurnRecords | None = None,
         reference_dir: Path | None = None,
+        run_result: EvaluationResult | None = None,
     ) -> CriteriaResults:
         """Check all success criteria.
 
@@ -147,6 +166,9 @@ def check_all(
             turn_records: Optional turn records for command inspection
             reference_dir: Optional resolved path to a reference directory.
                 Only consumed by ``agent_judge``; non-judge criteria ignore it.
+            run_result: Optional in-flight :class:`EvaluationResult`. Only
+                consumed by ``run_command`` with ``pass_context=true``; other
+                criteria ignore it.
 
         Returns:
             List of criterion results with scores
@@ -158,13 +180,16 @@ def check_all(
             self._reference_dir = reference_dir
         if turn_records is not None:
             self._turn_records = turn_records
+        if run_result is not None:
+            self._run_result = run_result
         ref_code = reference_code if reference_code is not None else self._reference_code
         ref_dir = reference_dir if reference_dir is not None else self._reference_dir
         records = turn_records if turn_records is not None else self._turn_records
+        result = run_result if run_result is not None else self._run_result
         results = []
         for criterion in criteria:
-            result = self._check_single(criterion, ref_code, records, ref_dir)
-            results.append(result)
+            r = self._check_single(criterion, ref_code, records, ref_dir, result)
+            results.append(r)
         return results
 
     def _get_checker_instance(self, criterion_type: str) -> BaseCriterion[Any]:
@@ -187,6 +212,7 @@ def _check_single(
         reference_code: str | None,
         turn_records: TurnRecords | None = None,
         reference_dir: Path | None = None,
+        run_result: EvaluationResult | None = None,
     ) -> CriterionResult:
         """Check a single criterion using registered checker.
 
@@ -195,6 +221,8 @@ def _check_single(
             reference_code: Optional reference code (string form)
             turn_records: Optional turn records for command inspection
             reference_dir: Optional resolved path to a reference directory.
+            run_result: Optional in-flight EvaluationResult (run_command's
+                pass_context=true consumes it; other criteria ignore it).
 
         Returns:
             CriterionResult with score
@@ -205,7 +233,7 @@ def _check_single(
         try:
             # Get cached instance
             checker = self._get_checker_instance(criterion_type)
-            context = CheckContext(route=self.route, reference_dir=reference_dir)
+            context = CheckContext(route=self.route, reference_dir=reference_dir, run_result=run_result)
             result = checker.check(
                 criterion,
                 self.sandbox,

src/coder_eval/models/criteria.py

@@ -198,6 +198,13 @@ class RunCommandCriterion(BaseSuccessCriterion):
             command: "python score.py"
             score_from_stdout: true
             description: "Similarity score from scoring script"
+
+          # Expose the in-flight evaluation record to the scoring script
+          - type: "run_command"
+            command: "python score.py"  # reads json.load(open(os.environ["CODER_EVAL_CONTEXT"]))
+            score_from_stdout: true
+            pass_context: true
+            description: "Score derived from the agent trajectory + resolved config"
     """
 
     type: Literal["run_command"] = "run_command"
@@ -219,6 +226,17 @@ class RunCommandCriterion(BaseSuccessCriterion):
             "Mutually exclusive with expected_stdout."
         ),
     )
+    pass_context: bool = Field(
+        default=False,
+        description=(
+            "Expose the in-flight evaluation record to the command as a JSON file at "
+            "$CODER_EVAL_CONTEXT. Same schema as task.json (EvaluationResult), so a scoring "
+            "script can be developed offline against a task.json from a previous run. "
+            "success_criteria_results is always empty; weighted_score / completed_at / "
+            "duration_seconds are null and final_status is still a placeholder (e.g. FAILURE) "
+            "because these are only finalized after the criteria phase — do not read them."
+        ),
+    )
 
     @model_validator(mode="after")
     def check_score_from_stdout_exclusivity(self) -> RunCommandCriterion:

src/coder_eval/orchestrator.py

@@ -1352,6 +1352,7 @@ async def _evaluation_loop(self) -> bool:
                 reference_code=reference_code,
                 reference_dir=reference_dir,
                 turn_records=self.result.iterations,
+                run_result=self.result,
             )
             self.result.success_criteria_results = criteria_results
             return self.result.all_criteria_passed(self.task.success_criteria)
@@ -1411,6 +1412,7 @@ async def _evaluation_loop(self) -> bool:
             reference_code=reference_code,
             reference_dir=reference_dir,
             turn_records=self.result.iterations,
+            run_result=self.result,
         )
         self.result.success_criteria_results = criteria_results
 
@@ -1504,6 +1506,7 @@ async def _run_dialog_criteria_check(
             reference_code=reference_code,
             reference_dir=reference_dir,
             turn_records=self.result.iterations,
+            run_result=self.result,
         )
         self._accumulate_judge_usage(criteria_results, judge_usage_accum)
         self.result.success_criteria_results = criteria_results