From b5bd41eb33fb1e5c543b23475d595251bc71548c Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 17:47:01 -0700 Subject: [PATCH 1/8] =?UTF-8?q?feat(reports):=201/3=20=E2=80=94=20add=20re?= =?UTF-8?q?ports=5Fjunit.py=20disk-driven=20JUnit=20XML=20writer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New src/coder_eval/reports_junit.py: generate_junit_xml(run_dir) reads the run.json spine (RunSummary), optional */*/suite.json gates (SuiteRollup), and per-failed-row task.json (best-effort plain-dict) to build a JUnit XML string; write_junit_xml is the thin persist wrapper. Counts always equal emitted children; status classification goes through FinalStatus.category; all agent-derived text is scrubbed of illegal XML 1.0 chars. Production code never parses XML. Adds defusedxml (dev-only) for test-side round-trip parsing and a shared write_run_json conftest fixture reused by later phases. Co-Authored-By: Claude Opus 4.8 (1M context) --- pyproject.toml | 1 + src/coder_eval/reports_junit.py | 310 +++++++++++++++++++++ tests/conftest.py | 52 ++++ tests/test_reports_junit.py | 458 ++++++++++++++++++++++++++++++++ 4 files changed, 821 insertions(+) create mode 100644 src/coder_eval/reports_junit.py create mode 100644 tests/test_reports_junit.py diff --git a/pyproject.toml b/pyproject.toml index dc127a66..36d6d083 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,7 @@ dev = [ "pip-audit>=2.10.0", "bandit[toml]>=1.9.4", "pre-commit>=4.5.1", + "defusedxml>=0.7.1", # test-side JUnit XML round-trip parsing (defense-in-depth; production code never parses XML) ] # Optional extra that enables UiPath-specific features: # - the `uipath` Python SDK on the host (handy for local sandbox parity with diff --git a/src/coder_eval/reports_junit.py b/src/coder_eval/reports_junit.py new file mode 100644 index 00000000..194a6f5e --- /dev/null +++ b/src/coder_eval/reports_junit.py @@ -0,0 +1,310 @@ +"""Disk-driven JUnit XML report generation from a finalized run directory. + +Reads a run directory's on-disk contracts — ``run.json`` (required, the +``RunSummary`` spine), ``*/*/suite.json`` (optional suite gates), and per-failed +row ``task.json`` (optional, best-effort failure detail) — and produces a JUnit +XML string that CI test-report ingesters (GitHub ``mikepenz/action-junit-report``, +Azure DevOps ``PublishTestResults@2``, …) understand. + +Single code path: ``coder-eval run --junit-xml`` and ``coder-eval report -f +junit`` both call :func:`generate_junit_xml`, so the two entry points can never +drift. + +SECURITY: this module only *builds* an element tree from JSON-derived data and +serializes it — it never *parses* XML, so stdlib ``xml.etree.ElementTree`` is +safe here (XXE / billion-laughs are parser-side attacks on untrusted XML input, +which has no surface). Every string entering the tree is scrubbed of characters +outside the XML 1.0 legal set via :func:`_xml_safe`. +""" + +from __future__ import annotations + +import json +import logging +import re +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Any, Literal + +from .evaluation.judge_context import truncate +from .models import FinalStatus, RunSummary, SuiteRollup + + +logger = logging.getLogger(__name__) + +# Characters outside XML 1.0's legal set. Kept as a plain (non-raw) ASCII-only +# string with doubled backslashes so this source file carries no literal astral +# or control characters. ``ElementTree`` will happily serialize control chars +# into *invalid* XML, so every agent-derived string is scrubbed before it enters +# the tree. +_ILLEGAL_XML = re.compile("[^\\x09\\x0A\\x0D\\x20-\\uD7FF\\uE000-\\uFFFD\\U00010000-\\U0010FFFF]") + +# Per-testcase failure/error body cap (chars). Agent detail dumps can be huge. +_BODY_LIMIT = 10_000 + + +def _xml_safe(text: str) -> str: + """Strip characters outside XML 1.0's legal set (ET handles ``<>&`` itself).""" + return _ILLEGAL_XML.sub("", text) + + +def _category_of(status: str) -> Literal["succeeded", "failed", "error"]: + """Map a serialized status string to a reporting category via the SSOT. + + Goes through ``FinalStatus(value).category`` (an explicit allowlist, CE018); + an unknown status value (older/newer schema) falls to the error bucket + explicitly rather than via a denylist. + """ + try: + return FinalStatus(status).category + except ValueError: + return "error" + + +def _set_counts(elem: ET.Element, cases: list[ET.Element]) -> None: + """Set ``tests``/``failures``/``errors``/``skipped`` from the actual children. + + The invariant the whole writer is built around: count attributes always + equal the emitted child elements (no separately-tracked counters that can + drift). + """ + elem.set("tests", str(len(cases))) + elem.set("failures", str(sum(1 for c in cases if c.find("failure") is not None))) + elem.set("errors", str(sum(1 for c in cases if c.find("error") is not None))) + elem.set("skipped", str(sum(1 for c in cases if c.find("skipped") is not None))) + + +def _load_task_json(run_dir: Path, row: dict[str, Any], variant: str) -> dict[str, Any] | None: + """Best-effort load of a failed row's ``task.json`` as a plain dict. + + Plain-dict access (not ``EvaluationResult.model_validate``) keeps the writer + tolerant of schema skew in blob-pulled/older runs and avoids materializing + the large ``turns`` array. Any error (missing dir/file, bad JSON) → ``None``. + """ + task_id = str(row.get("task_id", "")) + replicate_index = row.get("replicate_index") + task_dir = run_dir / variant / task_id + if isinstance(replicate_index, int): + candidate = task_dir / f"{replicate_index:02d}" / "task.json" + else: + matches = sorted(task_dir.glob("*/task.json")) + candidate = matches[0] if matches else task_dir / "task.json" + try: + return json.loads(candidate.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + + +def _criteria_body(row: dict[str, Any], run_dir: Path, variant: str) -> str: + """Build the failure/error body for a non-succeeded row. + + Prefers per-criterion lines from the row's ``task.json``; falls back to a + status + weighted-score line when task.json is missing/corrupt. Capped via + the shared ``truncate``. + """ + status = str(row.get("status")) + data = _load_task_json(run_dir, row, variant) + criteria = data.get("success_criteria_results") if isinstance(data, dict) else None + + lines: list[str] = [] + if isinstance(criteria, list) and criteria: + for crit in criteria: + if not isinstance(crit, dict): + continue + ctype = str(crit.get("criterion_type", "unknown")) + description = str(crit.get("description", "")) + score = crit.get("score") + threshold = crit.get("pass_threshold") + passed = isinstance(score, int | float) and isinstance(threshold, int | float) and score >= threshold + if passed: + lines.append(f"[PASS] {ctype}: {description}") + continue + score_str = f"{score:.2f}" if isinstance(score, int | float) else str(score) + thr_str = f"{threshold:.2f}" if isinstance(threshold, int | float) else str(threshold) + lines.append(f"[FAIL] {ctype}: score {score_str} < threshold {thr_str} — {description}") + detail = crit.get("details") or crit.get("error") + if detail: + lines.append(str(detail)) + else: + weighted = row.get("weighted_score") + weighted_str = f"{weighted:.2f}" if isinstance(weighted, int | float) else str(weighted) + lines.append(f"status={status} weighted_score={weighted_str}") + + return _xml_safe(truncate("\n".join(lines), _BODY_LIMIT)) + + +def _task_case(row: dict[str, Any], run_dir: Path) -> ET.Element: + """Build one ```` for a task row.""" + variant = row.get("variant_id") or "default" + task_id = str(row.get("task_id", "")) + replicate_index = row.get("replicate_index") + name = f"{task_id}[{replicate_index:02d}]" if isinstance(replicate_index, int) else task_id + + task_path = row.get("task_path") + classname = Path(task_path).stem if task_path else variant + + duration = row.get("duration") + time_str = f"{duration:.3f}" if isinstance(duration, int | float) else "0.000" + + case = ET.Element( + "testcase", + {"name": _xml_safe(name), "classname": _xml_safe(classname), "time": time_str}, + ) + + # Properties — emit only non-None values (no "None" strings in the tree). + prop_specs = [ + ("model_used", row.get("model_used")), + ("weighted_score", row.get("weighted_score")), + ("total_cost_usd", row.get("total_cost_usd")), + ("total_tokens", row.get("total_tokens")), + ("visible_turns", row.get("visible_turns")), + ] + props = [(k, v) for k, v in prop_specs if v is not None] + if props: + properties = ET.SubElement(case, "properties") + for key, value in props: + ET.SubElement(properties, "property", {"name": key, "value": _xml_safe(str(value))}) + + category = _category_of(str(row.get("status"))) + if category == "succeeded": + return case + + status = str(row.get("status")) + try: + FinalStatus(status) + message = status + except ValueError: + message = f"unknown status: {status}" + tag = "failure" if category == "failed" else "error" + child = ET.SubElement(case, tag, {"message": _xml_safe(message)}) + child.text = _criteria_body(row, run_dir, variant) + return case + + +def _skipped_suite(summary: RunSummary) -> ET.Element | None: + """Build the synthetic ``skipped`` testsuite from ``RunSummary.skipped_tasks``.""" + if not summary.skipped_tasks: + return None + suite = ET.Element("testsuite", {"name": "skipped"}) + cases: list[ET.Element] = [] + for entry in summary.skipped_tasks: + case = ET.SubElement( + suite, + "testcase", + {"name": _xml_safe(Path(entry.path).stem), "classname": "skipped"}, + ) + ET.SubElement(case, "skipped", {"message": _xml_safe(entry.reason)}) + cases.append(case) + _set_counts(suite, cases) + return suite + + +def _suite_gate_suite(run_dir: Path) -> ET.Element | None: + """Build the synthetic ``suite-gates`` testsuite from ``*/*/suite.json``. + + A suite.json that fails ``SuiteRollup`` validation is skipped with a warning + (schema skew must not kill report generation). Returns ``None`` when no valid + rollup is found. + """ + rollups: list[SuiteRollup] = [] + for path in sorted(run_dir.glob("*/*/suite.json")): + try: + rollups.append(SuiteRollup.model_validate_json(path.read_text(encoding="utf-8"))) + except (OSError, ValueError) as e: + logger.warning("Skipping unparseable suite rollup %s: %s", path, e) + if not rollups: + return None + + suite = ET.Element("testsuite", {"name": "suite-gates"}) + cases: list[ET.Element] = [] + for rollup in rollups: + case = ET.SubElement( + suite, + "testcase", + {"name": _xml_safe(f"{rollup.variant_id}/{rollup.suite_id}"), "classname": "suite-gates"}, + ) + cases.append(case) + if rollup.passed: + continue + lines: list[str] = [] + for agg in rollup.criterion_aggregates: + for check in agg.threshold_checks: + if check.passed: + continue + if check.actual_value is None: + lines.append(f"{check.metric}: metric not emitted (min {check.min_value})") + else: + lines.append(f"{check.metric}: {check.actual_value} < {check.min_value}") + if agg.error: + lines.append(f"{agg.criterion_type}: {agg.error}") + failure = ET.SubElement(case, "failure", {"message": "suite thresholds not met"}) + failure.text = _xml_safe(truncate("\n".join(lines), _BODY_LIMIT)) + _set_counts(suite, cases) + return suite + + +def generate_junit_xml(run_dir: Path) -> str: + """Read ``run_dir`` and return a JUnit XML string. + + Args: + run_dir: A finalized run directory containing ``run.json``. + + Returns: + JUnit XML with an ```` declaration. + + Raises: + FileNotFoundError: When ``run.json`` is absent (not a finalized run dir). + pydantic.ValidationError: When ``run.json`` fails ``RunSummary`` validation + (the spine contract is broken; a fabricated report would be worse). + """ + run_json = run_dir / "run.json" + if not run_json.is_file(): + raise FileNotFoundError(f"No run.json in {run_dir} — not a finalized run directory (see 'coder-eval run')") + summary = RunSummary.model_validate_json(run_json.read_text(encoding="utf-8")) + + root = ET.Element("testsuites", {"name": _xml_safe(summary.run_id)}) + root.set("time", f"{summary.total_duration_seconds:.3f}") + + # Group task rows by variant, preserving first-seen order. + grouped: dict[str, list[dict[str, Any]]] = {} + for row in summary.task_results: + variant = row.get("variant_id") or "default" + grouped.setdefault(variant, []).append(row) + + all_cases: list[ET.Element] = [] + for variant, rows in grouped.items(): + suite = ET.SubElement(root, "testsuite", {"name": _xml_safe(variant)}) + cases = [_task_case(row, run_dir) for row in rows] + for case in cases: + suite.append(case) + _set_counts(suite, cases) + all_cases.extend(cases) + + skipped_suite = _skipped_suite(summary) + if skipped_suite is not None: + root.append(skipped_suite) + all_cases.extend(skipped_suite.findall("testcase")) + + gate_suite = _suite_gate_suite(run_dir) + if gate_suite is not None: + root.append(gate_suite) + all_cases.extend(gate_suite.findall("testcase")) + + _set_counts(root, all_cases) + + return ET.tostring(root, encoding="utf-8", xml_declaration=True).decode("utf-8") + + +def write_junit_xml(run_dir: Path, output_path: Path) -> Path: + """Generate JUnit XML for ``run_dir`` and write it to ``output_path``. + + Thin persist wrapper mirroring the ``build_run_summary``/``write_run_summary`` + seam. Creates parent directories as needed. + + Returns: + The path written. + """ + xml = generate_junit_xml(run_dir) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(xml, encoding="utf-8") + return output_path diff --git a/tests/conftest.py b/tests/conftest.py index 61f9b118..023745d0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,10 @@ """Pytest configuration and shared fixtures.""" import logging +from collections.abc import Callable +from datetime import datetime +from pathlib import Path +from typing import Any import pytest @@ -54,3 +58,51 @@ def reset_logging_after_test(): # Reset to default level app_logger.setLevel(logging.NOTSET) + + +@pytest.fixture +def write_run_json() -> Callable[..., Path]: + """Factory that writes a valid ``run.json`` (a real ``RunSummary``) into a run dir. + + Shared by the JUnit writer tests (Phase 1) and the CLI report/run tests + (Phase 2) so both track the ``RunSummary`` model instead of hand-typed JSON. + Task-status counts (``tasks_succeeded``/``failed``/``error``) are computed + from each row's ``status`` via ``FinalStatus.category`` so the model's + task-count invariant always holds. Callers pass row dicts shaped like + ``eval_result_to_task_dict`` output; only ``task_id`` and ``status`` are + required per row. + """ + from coder_eval.models import FinalStatus, RunSummary, SkippedTask + + def _build( + run_dir: Path, + rows: list[dict[str, Any]], + *, + skipped: list[tuple[str, str]] | None = None, + run_id: str = "2026-07-21_12-00-00", + ) -> Path: + counts = {"succeeded": 0, "failed": 0, "error": 0} + for row in rows: + try: + category = FinalStatus(row["status"]).category + except ValueError: + category = "error" # unknown status → error bucket (mirrors the writer) + counts[category] += 1 + summary = RunSummary( + run_id=run_id, + start_time=datetime(2026, 7, 21, 12, 0, 0), + end_time=datetime(2026, 7, 21, 12, 5, 0), + total_duration_seconds=300.0, + tasks_run=len(rows), + tasks_succeeded=counts["succeeded"], + tasks_failed=counts["failed"], + tasks_error=counts["error"], + skipped_tasks=[SkippedTask(path=p, reason=r) for p, r in (skipped or [])], + task_results=rows, + framework_version="test", + ) + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "run.json").write_text(summary.model_dump_json(indent=2), encoding="utf-8") + return run_dir / "run.json" + + return _build diff --git a/tests/test_reports_junit.py b/tests/test_reports_junit.py new file mode 100644 index 00000000..a84c42e0 --- /dev/null +++ b/tests/test_reports_junit.py @@ -0,0 +1,458 @@ +"""Unit tests for the disk-driven JUnit XML writer (``reports_junit``). + +All tests are hermetic: the run directory is built by hand under ``tmp_path`` +(no agents, no API). Test-side XML parsing uses ``defusedxml`` as +defense-in-depth even though the parsed strings are self-generated. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest +from defusedxml.ElementTree import fromstring + +from coder_eval.models import SuiteRollup, ThresholdCheck +from coder_eval.reports_junit import generate_junit_xml, write_junit_xml + + +def _write_task_json( + run_dir: Path, + variant: str, + task_id: str, + replicate_index: int, + criteria: list[dict[str, Any]], +) -> None: + """Write a minimal task.json (plain dict) at the run-layout location.""" + task_dir = run_dir / variant / task_id / f"{replicate_index:02d}" + task_dir.mkdir(parents=True, exist_ok=True) + (task_dir / "task.json").write_text(json.dumps({"success_criteria_results": criteria}), encoding="utf-8") + + +def _row( + task_id: str, + status: str, + *, + variant_id: str | None = "default", + replicate_index: int | None = 0, + duration: float | None = 1.5, + task_path: str | None = "tasks/sample.yaml", + weighted_score: float | None = 0.5, + model_used: str | None = "claude-haiku-4-5-20251001", + total_tokens: int | None = 1234, + total_cost_usd: float | None = 0.01, + visible_turns: int | None = 3, +) -> dict[str, Any]: + return { + "task_id": task_id, + "status": status, + "variant_id": variant_id, + "replicate_index": replicate_index, + "duration": duration, + "task_path": task_path, + "weighted_score": weighted_score, + "model_used": model_used, + "total_tokens": total_tokens, + "total_cost_usd": total_cost_usd, + "visible_turns": visible_turns, + } + + +def _find_testsuite(root: Any, name: str) -> Any: + for ts in root.findall("testsuite"): + if ts.get("name") == name: + return ts + return None + + +def test_happy_path_grouping_and_counts(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [ + _row("t_pass", "SUCCESS", variant_id="v1"), + _row("t_fail", "FAILURE", variant_id="v1"), + _row("t_err", "ERROR", variant_id="v1"), + _row("t_pass", "SUCCESS", variant_id="v2"), + _row("t_fail", "FAILURE", variant_id="v2"), + _row("t_err", "ERROR", variant_id="v2"), + ] + write_run_json(run_dir, rows) + + root = fromstring(generate_junit_xml(run_dir)) + assert root.tag == "testsuites" + # Root counts match the emitted children. + assert int(root.get("tests")) == 6 + assert int(root.get("failures")) == 2 + assert int(root.get("errors")) == 2 + assert int(root.get("skipped")) == 0 + + for variant in ("v1", "v2"): + ts = _find_testsuite(root, variant) + assert ts is not None + cases = ts.findall("testcase") + assert int(ts.get("tests")) == len(cases) == 3 + assert int(ts.get("failures")) == 1 + assert int(ts.get("errors")) == 1 + # classname derives from task_path stem. + assert all(c.get("classname") == "sample" for c in cases) + # time is a float string. + assert all(float(c.get("time")) == 1.5 for c in cases) + + +def test_root_counts_equal_summed_children(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("a", "SUCCESS"), _row("b", "FAILURE"), _row("c", "MAX_TURNS_EXHAUSTED")] + write_run_json(run_dir, rows) + root = fromstring(generate_junit_xml(run_dir)) + + total_cases = 0 + total_failures = 0 + total_errors = 0 + for ts in root.findall("testsuite"): + cases = ts.findall("testcase") + total_cases += len(cases) + total_failures += sum(1 for c in cases if c.find("failure") is not None) + total_errors += sum(1 for c in cases if c.find("error") is not None) + assert int(root.get("tests")) == total_cases + assert int(root.get("failures")) == total_failures + assert int(root.get("errors")) == total_errors + # MAX_TURNS_EXHAUSTED is category "failed". + assert total_failures == 2 + assert total_errors == 0 + + +def test_failure_body_from_task_json(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t_fail", "FAILURE", variant_id="v1", replicate_index=0)] + write_run_json(run_dir, rows) + _write_task_json( + run_dir, + "v1", + "t_fail", + 0, + [ + { + "criterion_type": "file_exists", + "description": "output.txt must exist", + "score": 0.0, + "pass_threshold": 0.9, + "details": "file not found on disk", + "error": None, + }, + { + "criterion_type": "file_contains", + "description": "greeting present", + "score": 1.0, + "pass_threshold": 0.9, + "details": None, + "error": None, + }, + ], + ) + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "v1") + failure = ts.find("testcase").find("failure") + body = failure.text or "" + assert "file_exists" in body + assert "0.00" in body and "0.90" in body # score-vs-threshold line + assert "file not found on disk" in body + assert "output.txt must exist" in body + # passed criterion appears as a one-liner. + assert "file_contains" in body + + +def test_fallback_body_without_task_json(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t_fail", "FAILURE", variant_id="v1", weighted_score=0.42)] + write_run_json(run_dir, rows) + # No task.json written. + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "v1") + body = ts.find("testcase").find("failure").text or "" + assert "FAILURE" in body + assert "0.42" in body + + +def test_replicates_naming_and_per_replicate_task_json(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [ + _row("task", "FAILURE", variant_id="v1", replicate_index=0), + _row("task", "FAILURE", variant_id="v1", replicate_index=1), + ] + write_run_json(run_dir, rows) + _write_task_json( + run_dir, + "v1", + "task", + 0, + [ + { + "criterion_type": "c", + "description": "rep0", + "score": 0.0, + "pass_threshold": 0.9, + "details": "REP0DETAIL", + "error": None, + } + ], + ) + _write_task_json( + run_dir, + "v1", + "task", + 1, + [ + { + "criterion_type": "c", + "description": "rep1", + "score": 0.0, + "pass_threshold": 0.9, + "details": "REP1DETAIL", + "error": None, + } + ], + ) + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "v1") + names = {c.get("name") for c in ts.findall("testcase")} + assert names == {"task[00]", "task[01]"} + bodies = {c.get("name"): (c.find("failure").text or "") for c in ts.findall("testcase")} + assert "REP0DETAIL" in bodies["task[00]"] + assert "REP1DETAIL" in bodies["task[01]"] + + +def test_replicate_index_none_globs_task_json(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("task", "FAILURE", variant_id="v1", replicate_index=None)] + write_run_json(run_dir, rows) + _write_task_json( + run_dir, + "v1", + "task", + 0, + [ + { + "criterion_type": "c", + "description": "d", + "score": 0.0, + "pass_threshold": 0.9, + "details": "GLOBBED", + "error": None, + } + ], + ) + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "v1") + case = ts.find("testcase") + assert case.get("name") == "task" # no [NN] suffix + assert "GLOBBED" in (case.find("failure").text or "") + + +def test_variant_none_grouped_as_default(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t", "SUCCESS", variant_id=None, task_path=None)] + write_run_json(run_dir, rows) + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "default") + assert ts is not None + # task_path None → classname falls back to variant. + assert ts.find("testcase").get("classname") == "default" + + +def test_duration_none_renders_zero(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t", "SUCCESS", duration=None)] + write_run_json(run_dir, rows) + root = fromstring(generate_junit_xml(run_dir)) + case = root.find("testsuite").find("testcase") + assert float(case.get("time")) == 0.0 + + +def test_properties_omit_none_values(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [ + _row("t", "SUCCESS", total_cost_usd=None, total_tokens=None, weighted_score=None), + ] + write_run_json(run_dir, rows) + xml = generate_junit_xml(run_dir) + root = fromstring(xml) + props = root.find("testsuite").find("testcase").find("properties") + names = {p.get("name") for p in props.findall("property")} if props is not None else set() + assert "total_cost_usd" not in names + assert "total_tokens" not in names + assert "weighted_score" not in names + assert "model_used" in names + # No literal "None" strings leaked into the tree. + assert "None" not in xml + + +def test_skipped_tasks_suite(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t", "SUCCESS")] + write_run_json( + run_dir, + rows, + skipped=[("tasks/broken.yaml", "ValueError: bad schema"), ("tasks/opt.yaml", "skip: true")], + ) + root = fromstring(generate_junit_xml(run_dir)) + assert int(root.get("skipped")) == 2 + ts = _find_testsuite(root, "skipped") + assert ts is not None + skipped_cases = ts.findall("testcase") + assert len(skipped_cases) == 2 + names = {c.get("name") for c in skipped_cases} + assert names == {"broken", "opt"} + assert all(c.find("skipped") is not None for c in skipped_cases) + + +def test_suite_gates_failing_and_none_metric(tmp_path: Path, write_run_json: Callable[..., Path]) -> None: + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("t", "SUCCESS", variant_id="v1")]) + rollup = SuiteRollup( + suite_id="s1", + variant_id="v1", + rows_total=10, + rows_passed=5, + rows_failed=5, + rows_error=0, + pass_rate=0.5, + passed=False, + ) + # Inject two failed threshold checks via a criterion aggregate. + from coder_eval.models import CriterionAggregate + + rollup.criterion_aggregates = [ + CriterionAggregate( + criterion_type="classification_match", + passed=False, + threshold_checks=[ + ThresholdCheck(metric="accuracy", min_value=0.9, actual_value=0.5, passed=False), + ThresholdCheck(metric="f1.macro", min_value=0.8, actual_value=None, passed=False), + ], + ), + # A criterion that declared suite_thresholds but aggregate() returned nothing. + CriterionAggregate( + criterion_type="skill_triggered", + passed=False, + error="aggregate() returned no metrics", + ), + ] + suite_dir = run_dir / "v1" / "s1" + suite_dir.mkdir(parents=True, exist_ok=True) + (suite_dir / "suite.json").write_text(rollup.model_dump_json(indent=2), encoding="utf-8") + + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "suite-gates") + assert ts is not None + case = ts.find("testcase") + assert case.get("name") == "v1/s1" + body = case.find("failure").text or "" + assert "accuracy" in body and "0.5" in body and "0.9" in body + assert "f1.macro" in body + assert "metric not emitted" in body + # CriterionAggregate.error is surfaced in the gate body. + assert "skill_triggered" in body and "aggregate() returned no metrics" in body + + +def test_suite_gates_passing_is_plain_case(tmp_path: Path, write_run_json: Callable[..., Path]) -> None: + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("t", "SUCCESS", variant_id="v1")]) + rollup = SuiteRollup( + suite_id="s1", + variant_id="v1", + rows_total=2, + rows_passed=2, + rows_failed=0, + rows_error=0, + pass_rate=1.0, + passed=True, + ) + suite_dir = run_dir / "v1" / "s1" + suite_dir.mkdir(parents=True, exist_ok=True) + (suite_dir / "suite.json").write_text(rollup.model_dump_json(indent=2), encoding="utf-8") + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "suite-gates") + case = ts.find("testcase") + assert case.find("failure") is None + + +def test_corrupt_suite_json_skipped_with_report_still_produced( + tmp_path: Path, write_run_json: Callable[..., Path], caplog: pytest.LogCaptureFixture +) -> None: + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("t", "SUCCESS", variant_id="v1")]) + suite_dir = run_dir / "v1" / "s1" + suite_dir.mkdir(parents=True, exist_ok=True) + (suite_dir / "suite.json").write_text("{not valid json", encoding="utf-8") + # Report still generated; no suite-gates suite (the only rollup was corrupt). + root = fromstring(generate_junit_xml(run_dir)) + assert _find_testsuite(root, "suite-gates") is None + + +def test_xml_safety_control_chars_and_markup(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t_fail", "FAILURE", variant_id="v1", replicate_index=0)] + write_run_json(run_dir, rows) + nasty = "\x1b[31mboom\x00 & 'quote'" + _write_task_json( + run_dir, + "v1", + "t_fail", + 0, + [ + { + "criterion_type": "c", + "description": "d", + "score": 0.0, + "pass_threshold": 0.9, + "details": nasty, + "error": None, + } + ], + ) + xml = generate_junit_xml(run_dir) + root = fromstring(xml) # must re-parse + body = _find_testsuite(root, "v1").find("testcase").find("failure").text or "" + assert "boom" in body + assert "\x00" not in body # scrubbed + assert "\x1b" not in body + + +def test_unknown_status_lands_in_errors(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t", "SOME_FUTURE_STATUS", variant_id="v1")] + write_run_json(run_dir, rows) + root = fromstring(generate_junit_xml(run_dir)) + assert int(root.get("errors")) == 1 + err = _find_testsuite(root, "v1").find("testcase").find("error") + assert err is not None + assert "unknown status" in (err.get("message") or "") + + +def test_empty_run_all_skipped(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + write_run_json(run_dir, [], skipped=[("tasks/a.yaml", "skip: true")]) + root = fromstring(generate_junit_xml(run_dir)) + assert int(root.get("tests")) == 1 # only the skipped case + assert int(root.get("skipped")) == 1 + assert int(root.get("failures")) == 0 + assert int(root.get("errors")) == 0 + + +def test_missing_run_json_raises(tmp_path: Path) -> None: + run_dir = tmp_path / "empty" + run_dir.mkdir() + with pytest.raises(FileNotFoundError, match=r"run\.json"): + generate_junit_xml(run_dir) + + +def test_write_junit_xml_creates_parents_and_returns_path(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("t", "SUCCESS")]) + out = tmp_path / "nested" / "deeper" / "junit.xml" + written = write_junit_xml(run_dir, out) + assert written == out + assert out.is_file() + fromstring(out.read_text(encoding="utf-8")) # parses From 352dcf33962bebaf8f5d65886c780e7050b5fc23 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 17:55:56 -0700 Subject: [PATCH 2/8] =?UTF-8?q?feat(cli):=202/3=20=E2=80=94=20wire=20run?= =?UTF-8?q?=20--junit-xml=20and=20report=20-f=20junit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add --junit-xml to `coder-eval run` (writes the JUnit report after the run summary is persisted and before the failure exit-code gate, so a red run still produces a report; write errors propagate). Add 'junit' to `coder-eval report --format` (regenerates from any run dir; defaults to /junit.xml, -o overrides; missing run.json → clean red error). Output-only — does not enter the config merge. Docs: CI-pipeline tutorial JUnit section + CLAUDE.md tree entry. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 1 + docs/tutorials/02-ci-pipeline.md | 46 +++++++++++++++++ src/coder_eval/cli/report_command.py | 24 +++++++-- src/coder_eval/cli/run_command.py | 19 +++++++ tests/test_report_command.py | 52 +++++++++++++++++++ tests/test_run_command_junit.py | 77 ++++++++++++++++++++++++++++ 6 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 tests/test_run_command_junit.py diff --git a/CLAUDE.md b/CLAUDE.md index c20bde4f..947ad4f2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,6 +20,7 @@ coder_eval/ ├── orchestrator.py # Main evaluation loop ├── reports.py # Markdown/JSON report generation (run-level + per-suite rollup via write_suite_rollups) ├── reports_experiment.py # Experiment/cross-variant report generation +├── reports_junit.py # JUnit XML report from a finalized run dir (run.json spine; for CI test-report ingestion) ├── analysis.py # Command statistics aggregation ├── logging_config.py # Structured logging setup ├── path_utils.py # Run ID generation, path utilities diff --git a/docs/tutorials/02-ci-pipeline.md b/docs/tutorials/02-ci-pipeline.md index 7dc04eea..63143087 100644 --- a/docs/tutorials/02-ci-pipeline.md +++ b/docs/tutorials/02-ci-pipeline.md @@ -158,6 +158,52 @@ jobs: - **`if: always()`** on the verdict and upload steps means you still get results even when a task fails. +## Publishing test results (JUnit) + +The verdict step above parses `task.json` by hand. If your CI already ingests +JUnit XML — for the per-test annotations, history, and flake tracking most CI +systems render from it — let `coder-eval` emit it directly instead. Add +`--junit-xml` to the run: + +```yaml + - name: Run evaluations + run: coder-eval run $TASK_GLOBS -j 4 --junit-xml coder-eval-junit.xml +``` + +The file is written **before** the exit-code decision, so a failing run still +produces a report. You can also regenerate it from any existing run directory +without re-running the suite: + +```bash +coder-eval report runs/latest -f junit # writes runs/latest/junit.xml +coder-eval report runs/latest -f junit -o out.xml # custom path +``` + +The mapping: one `` per task row (grouped into a `` per +variant), failed/errored rows carry a ``/`` with the per-criterion +breakdown, skipped tasks and failing suite gates get their own synthetic suites. + +Feed the file to whatever your platform uses to render test results. On GitHub +Actions, [`mikepenz/action-junit-report`](https://github.com/mikepenz/action-junit-report): + +```yaml + - name: Publish test report + if: always() + uses: mikepenz/action-junit-report@v5 + with: + report_paths: coder-eval-junit.xml +``` + +On Azure DevOps, `PublishTestResults@2`: + +```yaml + - task: PublishTestResults@2 + condition: always() + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: 'coder-eval-junit.xml' +``` + ## Secrets Add these under **Settings → Secrets and variables → Actions**: diff --git a/src/coder_eval/cli/report_command.py b/src/coder_eval/cli/report_command.py index 4d801c99..6c5fcfc4 100644 --- a/src/coder_eval/cli/report_command.py +++ b/src/coder_eval/cli/report_command.py @@ -27,7 +27,10 @@ def report_command( "md", "--format", "-f", - help="Output format: 'md' (default markdown), 'html' (render task.json files as HTML).", + help=( + "Output format: 'md' (default markdown), 'html' (render task.json files as HTML), " + "'junit' (JUnit XML from run.json)." + ), ), ) -> None: """Display or export a run report. @@ -44,16 +47,31 @@ def report_command( # Re-generate HTML reports for every task.json under a run dir coder-eval report runs/latest --format html + + # Write a JUnit XML report (defaults to /junit.xml) + coder-eval report runs/latest --format junit """ fmt = report_format.lower() - if fmt not in ("md", "html"): - console.print(f"[red]Error: unknown --format '{report_format}' (expected 'md' or 'html')[/red]") + if fmt not in ("md", "html", "junit"): + console.print(f"[red]Error: unknown --format '{report_format}' (expected 'md', 'html', or 'junit')[/red]") raise typer.Exit(1) if fmt == "html": _regenerate_html_reports(run_dir, output_file) return + if fmt == "junit": + from ..reports_junit import write_junit_xml + + target = output_file if output_file is not None else run_dir / "junit.xml" + try: + written = write_junit_xml(run_dir, target) + except FileNotFoundError as e: + console.print(f"[red]Error: {e}[/red]") + raise typer.Exit(1) from e + console.print(f"[green][OK]JUnit report written to {written}[/green]") + return + try: report_md, source_path = ReportGenerator.load_from_run_dir(run_dir) console.print(f"[dim]Reading report from {source_path}[/dim]\n") diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index b35366ca..dff081da 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -173,6 +173,11 @@ def run_command( "--log-file", help="Log to file in addition to console", ), + junit_xml: Path | None = typer.Option( # noqa: B008 + None, + "--junit-xml", + help="Write a JUnit XML report of task results to this path (for CI test-report ingestion).", + ), tags: str | None = typer.Option( None, "--tags", @@ -365,6 +370,7 @@ def run_command( verbose=verbose, resume=resume, include_skipped=include_skipped, + junit_xml=junit_xml, ) ) except KeyboardInterrupt: @@ -389,6 +395,7 @@ async def _run_all_tasks( verbose: bool = False, resume: bool = False, include_skipped: bool = False, + junit_xml: Path | None = None, ) -> None: """Async entry point for running all tasks (optionally in parallel). @@ -407,6 +414,8 @@ async def _run_all_tasks( from -D/--set and the bespoke flag aliases stream_mode: Optional stream mode ('full' or 'minimal') for real-time output experiment_path: Optional path to experiment YAML (default: experiments/default.yaml) + junit_xml: Optional path to write a JUnit XML report to, after the run + summary is persisted and before the failure exit-code gate. """ # Prepare run directory run_dir = prepare_run_directory(run_dir) @@ -464,6 +473,16 @@ async def _run_all_tasks( # Print execution summary print_execution_summary(run_dir, summary) + + # Write the JUnit report (if requested) BEFORE the exit-code gate below, + # so a failing run still produces the report. suite.json + run.json are + # already on disk (written inside _run_with_experiment). A write error + # propagates (loud failure, exit != 0) rather than being swallowed. + if junit_xml is not None: + from ..reports_junit import write_junit_xml + + written = write_junit_xml(run_dir, junit_xml) + console.print(f"[green][OK]JUnit report written to {written}[/green]") finally: # Explicit flush before process exit (belt-and-suspenders with atexit). # In a `finally` so it runs on the success path and on any raised diff --git a/tests/test_report_command.py b/tests/test_report_command.py index 3c7a7fe9..060e7f12 100644 --- a/tests/test_report_command.py +++ b/tests/test_report_command.py @@ -144,3 +144,55 @@ def test_report_html_malformed_task_json_skipped_and_fails(tmp_path: Path) -> No assert "Skipping" in res.stdout assert "1 task(s) failed" in res.stdout assert (run_dir / "task-good" / "task.html").exists() + + +# -------------------------------------------------------------------------- +# JUnit path (-f junit) +# -------------------------------------------------------------------------- + + +def test_report_junit_default_output(write_run_json, tmp_path: Path) -> None: + from defusedxml.ElementTree import fromstring + + run_dir = tmp_path / "run" + write_run_json(run_dir, [{"task_id": "t", "status": "SUCCESS"}]) + + res = runner.invoke(app, ["report", str(run_dir), "-f", "junit"]) + assert res.exit_code == 0 + out = run_dir / "junit.xml" + assert out.is_file() + root = fromstring(out.read_text(encoding="utf-8")) + assert root.tag == "testsuites" + + +def test_report_junit_custom_output(write_run_json, tmp_path: Path) -> None: + from defusedxml.ElementTree import fromstring + + run_dir = tmp_path / "run" + write_run_json(run_dir, [{"task_id": "t", "status": "SUCCESS"}]) + out = tmp_path / "out" / "custom.xml" + + res = runner.invoke(app, ["report", str(run_dir), "-f", "junit", "-o", str(out)]) + assert res.exit_code == 0 + assert out.is_file() + fromstring(out.read_text(encoding="utf-8")) + assert not (run_dir / "junit.xml").exists() + + +def test_report_junit_missing_run_json_exits_1(tmp_path: Path) -> None: + run_dir = tmp_path / "run" + run_dir.mkdir() # no run.json + + res = runner.invoke(app, ["report", str(run_dir), "-f", "junit"]) + assert res.exit_code == 1 + assert "run.json" in res.stdout + + +def test_report_unknown_format_message_lists_junit(tmp_path: Path) -> None: + run_dir = tmp_path / "run" + run_dir.mkdir() + + res = runner.invoke(app, ["report", str(run_dir), "--format", "bogus"]) + assert res.exit_code == 1 + assert "unknown --format" in res.stdout + assert "junit" in res.stdout diff --git a/tests/test_run_command_junit.py b/tests/test_run_command_junit.py new file mode 100644 index 00000000..b5ac1de0 --- /dev/null +++ b/tests/test_run_command_junit.py @@ -0,0 +1,77 @@ +"""`coder-eval run --junit-xml` wiring at the ``_run_all_tasks`` seam. + +Fully hermetic: every side-effecting collaborator is patched (see the pattern in +``test_cli_telemetry.py``). The ``_run_with_experiment`` mock writes a minimal +``run.json`` to disk, which is exactly what the JUnit writer reads. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from unittest.mock import AsyncMock, Mock, patch + +import pytest +import typer +from defusedxml.ElementTree import fromstring + +from coder_eval.cli.run_command import _run_all_tasks + + +async def _invoke( + run_dir: Path, + junit_xml: Path | None, + write_run_json: Callable[..., Path], + *, + status: str, + failed: bool, +) -> None: + summary = Mock(tasks_failed=1 if failed else 0, tasks_error=0) + + async def _fake(*_args, **_kwargs): + # Mirror production: run.json is persisted inside _run_with_experiment. + write_run_json(run_dir, [{"task_id": "t", "status": status}]) + return summary, 0 + + with ( + patch("coder_eval.cli.run_command.prepare_run_directory", return_value=run_dir), + patch("coder_eval.cli.run_command.expand_task_files", return_value=[Path("a.yaml")]), + patch("coder_eval.cli.run_command._run_with_experiment", new=AsyncMock(side_effect=_fake)), + patch("coder_eval.logging_config.aggregate_task_logs"), + patch("coder_eval.cli.run_command.print_execution_summary"), + patch("coder_eval.telemetry.track_event"), + patch("coder_eval.telemetry.flush_telemetry"), + ): + await _run_all_tasks( + task_files=[Path("a.yaml")], + preservation_mode=None, + run_dir=run_dir, + max_parallel=1, + junit_xml=junit_xml, + ) + + +async def test_junit_written_on_success(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + junit = tmp_path / "j.xml" + await _invoke(run_dir, junit, write_run_json, status="SUCCESS", failed=False) + assert junit.is_file() + fromstring(junit.read_text(encoding="utf-8")) + + +async def test_junit_written_even_when_run_fails(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + # Ordering guarantee: the report is written BEFORE the failure exit-code gate. + run_dir = tmp_path / "run" + junit = tmp_path / "j.xml" + with pytest.raises(typer.Exit) as exc: + await _invoke(run_dir, junit, write_run_json, status="FAILURE", failed=True) + assert exc.value.exit_code == 1 + assert junit.is_file() + fromstring(junit.read_text(encoding="utf-8")) + + +async def test_no_junit_when_flag_absent(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + junit = tmp_path / "j.xml" + await _invoke(run_dir, None, write_run_json, status="SUCCESS", failed=False) + assert not junit.exists() From 6e17edc7d9a2474957d21219aebbc09644e4a56e Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 18:09:27 -0700 Subject: [PATCH 3/8] =?UTF-8?q?feat(ci):=203/3=20=E2=80=94=20publish=20com?= =?UTF-8?q?posite=20action,=20release=20automation,=20PR=20dogfood?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add action.yml at the repo root: a composite action that installs a pinned coder-eval (or the local checkout via `version: local`), runs the suite, always emits run-dir/junit-path outputs and a run.md job summary, then exits with coder-eval's real exit code. Inputs cross into bash via env: only. release.yml now bumps the action's `version:` default inside the amended release commit (anchored on a `# <-- kept in sync` comment, with a loud grep guard) and force-moves the `v` tag after pushing the release tag. pr-checks.yml gains a fork-gated `action-dogfood` job that exercises the action via `uses: ./` on one Haiku task and verifies the JUnit + run.json output — the sensor for this surface, since it isn't covered by pytest. The action is deliberately agent-agnostic: it installs no coding-agent runtime, so callers provide the Claude CLI (as the dogfood job does). Documented in the README and the CI tutorial along with the pull_request_target security caveat. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/pr-checks.yml | 50 ++++++++++++++ .github/workflows/release.yml | 39 +++++++++-- CLAUDE.md | 1 + README.md | 47 +++++++++++++ action.yml | 110 +++++++++++++++++++++++++++++++ docs/tutorials/02-ci-pipeline.md | 25 +++++++ 6 files changed, 266 insertions(+), 6 deletions(-) create mode 100644 action.yml diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 28578b09..f985560f 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -831,3 +831,53 @@ jobs: name: byoa-live-output path: tmp/ retention-days: 7 + + action-dogfood: + name: Action Dogfood (composite action, real API) + runs-on: ubuntu-latest + timeout-minutes: 15 + # Skip on fork PRs where secrets aren't available (matches e2e-smoke). + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # The composite action is agent-agnostic and does NOT install a coding-agent + # runtime. The dogfood task uses the default claude-code agent, so provide + # Node + the Claude CLI here (as e2e-smoke does), before invoking the action. + - name: Set up Node.js 20 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "20" + - name: Install Claude CLI + run: npm install -g @anthropic-ai/claude-code + + - name: Run coder-eval via local action + id: dogfood + uses: ./ + with: + version: local + tasks: tasks/hello_date.yaml + model: claude-haiku-4-5-20251001 + run-dir: runs/ci-action-dogfood + junit-path: runs/ci-action-dogfood/junit.xml + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + + - name: Verify outputs and JUnit file + env: + JUNIT: ${{ steps.dogfood.outputs.junit-path }} + RUNDIR: ${{ steps.dogfood.outputs.run-dir }} + run: | + set -euo pipefail + test -n "$JUNIT" && test -f "$JUNIT" || { echo "junit output missing"; exit 1; } + # Well-formedness check on a file this job just generated (trusted input; + # our writer emits no DTDs/entities) — stdlib ET is fine here. + python3 -c "import sys, xml.etree.ElementTree as ET; ET.parse(sys.argv[1])" "$JUNIT" + test -f "$RUNDIR/run.json" || { echo "run.json missing"; exit 1; } + + - name: Upload dogfood run on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: action-dogfood-runs + path: runs/ci-action-dogfood/ + retention-days: 7 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ff2a7ebb..23094069 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -182,23 +182,50 @@ jobs: echo "version=$V" >> "$GITHUB_OUTPUT" echo "Publishing version: $V" - - name: Regenerate uv.lock and amend release commit + - name: Regenerate uv.lock, bump action.yml pin, and amend release commit if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != '' + env: + # Passed via env (not interpolated into the script) per GitHub's + # injection guidance. + VERSION: ${{ steps.release.outputs.version }} run: | + set -euo pipefail + git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.name "github-actions[bot]" + # Bump the composite action's default `version:` pin to the just-released + # version so `UiPath/coder_eval@vX.Y.Z` installs `coder-eval==X.Y.Z`. The + # anchor is indentation-tolerant and keyed on the unique trailing + # "# <-- kept in sync" comment; the grep guard fails the release loudly + # if a reformat ever detaches it (rather than shipping a stale pin). + sed -i -E 's/^([[:space:]]*default: ")[0-9]+\.[0-9]+\.[0-9]+(" # <-- kept in sync)/\1'"${VERSION}"'\2/' action.yml + grep -q "default: \"${VERSION}\"" action.yml || { echo "action.yml version bump failed"; exit 1; } + git add action.yml + # Regenerate the lock too; stage it (a no-op if unchanged). uv lock - if ! git diff --quiet uv.lock; then - git config user.email "github-actions[bot]@users.noreply.github.com" - git config user.name "github-actions[bot]" - git add uv.lock + git add uv.lock + # Amend only if action.yml/uv.lock actually changed the tree. + if ! git diff --cached --quiet; then git commit --amend --no-edit # Amend replaced the commit the tag points at; re-point it before pushing. - git tag -f "v${{ steps.release.outputs.version }}" + git tag -f "v${VERSION}" fi - name: Push release commit and tags if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != '' run: git push origin main "v${{ steps.release.outputs.version }}" + - name: Move major action tag (vN -> this release) + if: steps.release.outputs.version != '' + env: + VERSION: ${{ steps.release.outputs.version }} + run: | + set -euo pipefail + # Consumers pin `UiPath/coder_eval@v0` (becomes `@v1` at 1.0.0). Force-move + # the moving major tag to this release. Force on a missing tag creates it. + MAJOR="v${VERSION%%.*}" + git tag -f "$MAJOR" "v${VERSION}" + git push -f origin "$MAJOR" + - name: Build wheel + sdist if: steps.ver.outputs.version != '' run: uv build diff --git a/CLAUDE.md b/CLAUDE.md index 947ad4f2..87b531a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -121,6 +121,7 @@ tasks/ # Task definition YAML files tests/ # Test suite docs/ # Documentation templates/ # Sandbox template directories +action.yml # Published composite GitHub Action (coder-eval as a CI gate). release.yml maintains its `version:` default + the moving `v` tag. ``` ## Key Architectural Patterns diff --git a/README.md b/README.md index 733d1053..ccb79aff 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,53 @@ live in this repo — clone it or point the CLI at your own task files.) See [Tutorial 02 — Running coder_eval in CI](docs/tutorials/02-ci-pipeline.md) for the full setup. +## Use as a GitHub Action + +A composite action at the repo root runs `coder-eval` as a CI gate — it installs +the pinned CLI, runs your tasks, writes a JUnit XML report, appends `run.md` to +the job summary, and fails the step on any task/gate failure: + +```yaml +- uses: UiPath/coder_eval@v0 # becomes @v1 once 1.0.0 ships; @vX.Y.Z pins exactly + with: + tasks: tests/tasks/**/*.yaml + model: claude-haiku-4-5-20251001 + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} +``` + +| Input | Default | Purpose | +| --- | --- | --- | +| `tasks` | *(all `tasks/`)* | Task YAML path(s)/glob | +| `tags` | — | `--tags` filter | +| `model` | — | `--model` override | +| `extra-args` | — | Verbatim extra args (`--experiment`, `-D …`, …) | +| `version` | pinned release | PyPI version, or `local` to install from the checkout | +| `run-dir` | `runs/ci` | Run directory | +| `junit-path` | `coder-eval-junit.xml` | Where to write the JUnit report | +| `step-summary` | `true` | Append `run.md` to the job summary | +| `anthropic-api-key` | — | Exported as `ANTHROPIC_API_KEY` for the run | + +Outputs: `run-dir` and `junit-path`. Feed the JUnit file to your platform's +test-report renderer — e.g. on GitHub Actions with +[`mikepenz/action-junit-report`](https://github.com/mikepenz/action-junit-report): + +```yaml +- uses: mikepenz/action-junit-report@v5 + if: always() + with: + report_paths: coder-eval-junit.xml +``` + +> **Agent runtime is the caller's responsibility.** The action is agent-agnostic — +> it installs `coder-eval` but no coding-agent runtime. Tasks using the default +> `claude-code` agent need the `claude` CLI on `PATH` (`actions/setup-node` + +> `npm install -g @anthropic-ai/claude-code`) in the job before the action runs. + +> **Security.** Evaluated tasks execute agent-generated code. Do **not** run this +> action under `pull_request_target` with secrets exposed to untrusted fork PRs — +> use `pull_request` and gate on the same-repo condition, as this repo's own +> dogfood job does. + ## Telemetry > 📊 **Usage telemetry is on by default.** `coder-eval` sends **anonymous** usage diff --git a/action.yml b/action.yml new file mode 100644 index 00000000..c9006644 --- /dev/null +++ b/action.yml @@ -0,0 +1,110 @@ +name: coder-eval +description: Run coder-eval evaluation tasks as a CI gate, with JUnit XML output and a job-summary report. +branding: + icon: check-circle + color: orange + +# This action installs and runs the `coder-eval` CLI. It is agent-agnostic: it +# does NOT install any coding-agent runtime. Tasks that use the default +# `claude-code` agent need the `claude` CLI on PATH (Node + the +# `@anthropic-ai/claude-code` npm package) provided by the calling job before +# this action runs. See the README "Use as a GitHub Action" section. +# +# SECURITY: evaluated tasks execute agent-generated code. Do NOT run this action +# under `pull_request_target` with secrets exposed to untrusted fork PRs. + +inputs: + tasks: + description: Task YAML path(s)/glob passed to `coder-eval run` (empty = all tasks/ recursively) + required: false + default: "" + tags: + description: Only run tasks matching any of these comma-separated tags (--tags) + required: false + default: "" + model: + description: Override agent model for all tasks (--model) + required: false + default: "" + extra-args: + description: Extra arguments appended verbatim to `coder-eval run` (trusted caller input; covers --experiment, -D overrides, --tags exclusions, etc.) + required: false + default: "" + version: + description: coder-eval version to install from PyPI, or "local" to install from the action checkout + required: false + default: "0.8.6" # <-- kept in sync with releases by release.yml + run-dir: + description: Run directory (--run-dir) + required: false + default: "runs/ci" + junit-path: + description: Where to write the JUnit XML report + required: false + default: "coder-eval-junit.xml" + step-summary: + description: Append run.md to the GitHub job summary ("true"/"false") + required: false + default: "true" + anthropic-api-key: + description: Anthropic API key (exported as ANTHROPIC_API_KEY for the run step) + required: false + default: "" + +outputs: + run-dir: + description: The run directory containing run.json/run.md + value: ${{ steps.run.outputs.run-dir }} + junit-path: + description: Path to the written JUnit XML report + value: ${{ steps.run.outputs.junit-path }} + +runs: + using: composite + steps: + - name: Install uv + uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 + - name: Install coder-eval + shell: bash + env: + CE_VERSION: ${{ inputs.version }} + CE_ACTION_PATH: ${{ github.action_path }} + run: | + set -euo pipefail + if [ "$CE_VERSION" = "local" ]; then + uv tool install "$CE_ACTION_PATH" + else + uv tool install "coder-eval==$CE_VERSION" + fi + - name: Run coder-eval + id: run + shell: bash + env: + ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }} + CE_TASKS: ${{ inputs.tasks }} + CE_TAGS: ${{ inputs.tags }} + CE_MODEL: ${{ inputs.model }} + CE_EXTRA_ARGS: ${{ inputs.extra-args }} + CE_RUN_DIR: ${{ inputs.run-dir }} + CE_JUNIT: ${{ inputs.junit-path }} + CE_SUMMARY: ${{ inputs.step-summary }} + run: | + set -uo pipefail + args=(run --run-dir "$CE_RUN_DIR" --junit-xml "$CE_JUNIT") + [ -n "$CE_TAGS" ] && args+=(--tags "$CE_TAGS") + [ -n "$CE_MODEL" ] && args+=(--model "$CE_MODEL") + # extra-args is a trusted caller input, split on whitespace intentionally + # shellcheck disable=SC2206 + [ -n "$CE_EXTRA_ARGS" ] && args+=($CE_EXTRA_ARGS) + # shellcheck disable=SC2206 + [ -n "$CE_TASKS" ] && args+=($CE_TASKS) + set +e + coder-eval "${args[@]}" + CODE=$? + set -e + echo "run-dir=$CE_RUN_DIR" >> "$GITHUB_OUTPUT" + echo "junit-path=$CE_JUNIT" >> "$GITHUB_OUTPUT" + if [ "$CE_SUMMARY" = "true" ] && [ -f "$CE_RUN_DIR/run.md" ]; then + head -c 1000000 "$CE_RUN_DIR/run.md" >> "$GITHUB_STEP_SUMMARY" + fi + exit $CODE diff --git a/docs/tutorials/02-ci-pipeline.md b/docs/tutorials/02-ci-pipeline.md index 63143087..c19f3309 100644 --- a/docs/tutorials/02-ci-pipeline.md +++ b/docs/tutorials/02-ci-pipeline.md @@ -158,6 +158,31 @@ jobs: - **`if: always()`** on the verdict and upload steps means you still get results even when a task fails. +## Shortcut: the packaged action + +The five steps above spell out the mechanics, but coder_eval also ships a +composite action at the repo root that bundles install + run + JUnit report + +job-summary + fail-on-failure into one step: + +```yaml + - uses: actions/setup-node@v4 # the claude-code agent needs the Claude CLI… + with: { node-version: '20' } + - run: npm install -g @anthropic-ai/claude-code + + - uses: UiPath/coder_eval@v0 # …then run the gate (pin @vX.Y.Z in production) + with: + tasks: tests/tasks/**/*.yaml + model: claude-haiku-4-5-20251001 + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} +``` + +The action is agent-agnostic — it installs `coder-eval` but not the agent +runtime, so provide the `claude` CLI (or your agent's runtime) in the job first. +See the [README "Use as a GitHub Action"](../../README.md#use-as-a-github-action) +section for the full inputs table and the fork/`pull_request_target` security +caveat. The rest of this tutorial's hand-rolled workflow is still useful when you +need finer control than the action's inputs expose. + ## Publishing test results (JUnit) The verdict step above parses `task.json` by hand. If your CI already ingests From 578e9ed698473da347e990c4cc14e5b8044b4c5e Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 18:11:40 -0700 Subject: [PATCH 4/8] chore(deps): lock defusedxml (dev-only, test-side XML parsing) Companion to the pyproject dev-group addition in 1/3. Co-Authored-By: Claude Opus 4.8 (1M context) --- uv.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uv.lock b/uv.lock index 13de44f4..6021d057 100644 --- a/uv.lock +++ b/uv.lock @@ -480,6 +480,7 @@ codex = [ ] dev = [ { name = "bandit" }, + { name = "defusedxml" }, { name = "mcp" }, { name = "pip-audit" }, { name = "pre-commit" }, @@ -503,6 +504,7 @@ requires-dist = [ { name = "bandit", extras = ["toml"], marker = "extra == 'dev'", specifier = ">=1.9.4" }, { name = "claude-agent-sdk", specifier = ">=0.2.124" }, { name = "click", specifier = ">=8.3.3" }, + { name = "defusedxml", marker = "extra == 'dev'", specifier = ">=0.7.1" }, { name = "google-antigravity", marker = "extra == 'antigravity'", specifier = "==0.1.7" }, { name = "jmespath", specifier = ">=1.1.0" }, { name = "jsonschema", specifier = ">=4.26.0" }, From f3a190e832c02e9dc0e665f78bf622ccebcb1e90 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 18:19:33 -0700 Subject: [PATCH 5/8] fix(reports): code review fixes for the JUnit CI gate Harden reports_junit.py against schema-skewed / crafted run.json rows, which are untyped dicts (RunSummary.task_results is list[dict[str, Any]]) and may be blob-pulled from elsewhere: - Path containment: a crafted variant_id/task_id could steer the task.json lookup outside the run dir (absolute value discards run_dir, ".." walks up). Reject unsafe path components and verify containment after resolve(). - UnicodeDecodeError is a ValueError but NOT a json.JSONDecodeError, so a task.json with undecodable bytes aborted the whole report instead of falling back. Catch ValueError. - Ambiguous replicate: with replicate_index absent and several replicate dirs, the writer attributed replicate 00's failure detail to the row. Degrade to the status-only body instead of misattributing. - bool is an int subclass, so a bool replicate_index rendered as "[01]" and a bool duration as a time. Exclude bool explicitly. - Guard non-finite/negative durations (NaN would emit an invalid time="nan"). - Coerce non-str variant_id/task_path rather than crashing on dict keys/Path(). - Skipped testcases now use the suffix-stripped path, not the bare stem, so two skipped tasks sharing a basename keep distinct JUnit identities. - A row with no status reads as "" rather than "None". Found by the final multi-model review (gpt-5.6, gemini-3.1-pro, gpt-5.3-codex); each fix has a regression test that fails without it. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/coder_eval/reports_junit.py | 92 ++++++++++++++++++----- tests/conftest.py | 2 +- tests/test_reports_junit.py | 127 +++++++++++++++++++++++++++++++- 3 files changed, 201 insertions(+), 20 deletions(-) diff --git a/src/coder_eval/reports_junit.py b/src/coder_eval/reports_junit.py index 194a6f5e..a6c3cb5f 100644 --- a/src/coder_eval/reports_junit.py +++ b/src/coder_eval/reports_junit.py @@ -21,6 +21,7 @@ import json import logging +import math import re import xml.etree.ElementTree as ET from pathlib import Path @@ -42,6 +43,11 @@ # Per-testcase failure/error body cap (chars). Agent detail dumps can be huge. _BODY_LIMIT = 10_000 +# Serialized status values we recognize, for distinguishing a known status from a +# schema-skewed one when labelling a failure/error (classification itself goes +# through FinalStatus.category — see _category_of). +_KNOWN_STATUSES = frozenset(s.value for s in FinalStatus) + def _xml_safe(text: str) -> str: """Strip characters outside XML 1.0's legal set (ET handles ``<>&`` itself).""" @@ -74,24 +80,65 @@ def _set_counts(elem: ET.Element, cases: list[ET.Element]) -> None: elem.set("skipped", str(sum(1 for c in cases if c.find("skipped") is not None))) +def _variant_of(row: dict[str, Any]) -> str: + """Variant bucket for a row — ``str``-coerced, ``None``/empty → ``"default"``. + + Rows are untyped dicts, so a non-string ``variant_id`` must not reach a dict + key or an XML attribute as a non-``str``. + """ + return str(row.get("variant_id") or "default") + + +def _status_of(row: dict[str, Any]) -> str: + """Row status as a string; an absent key reads as ``""``, not ``"None"``.""" + raw = row.get("status") + return str(raw) if raw is not None else "" + + +def _is_safe_component(value: str) -> bool: + """True when ``value`` is usable as a single, contained path segment. + + ``run.json`` rows are untyped and may be blob-pulled from elsewhere, so a + crafted ``variant_id``/``task_id`` must not steer the lookup outside the run + directory (an absolute value would discard ``run_dir`` entirely, and ``..`` + would walk up). + """ + return bool(value) and value not in {".", ".."} and "/" not in value and "\\" not in value + + def _load_task_json(run_dir: Path, row: dict[str, Any], variant: str) -> dict[str, Any] | None: """Best-effort load of a failed row's ``task.json`` as a plain dict. Plain-dict access (not ``EvaluationResult.model_validate``) keeps the writer tolerant of schema skew in blob-pulled/older runs and avoids materializing - the large ``turns`` array. Any error (missing dir/file, bad JSON) → ``None``. + the large ``turns`` array. Any problem — unsafe path component, missing + dir/file, undecodable bytes, bad JSON, or an ambiguous replicate — yields + ``None`` so the caller falls back to a status-only body. """ task_id = str(row.get("task_id", "")) + if not _is_safe_component(variant) or not _is_safe_component(task_id): + return None + replicate_index = row.get("replicate_index") task_dir = run_dir / variant / task_id - if isinstance(replicate_index, int): + if isinstance(replicate_index, int) and not isinstance(replicate_index, bool): candidate = task_dir / f"{replicate_index:02d}" / "task.json" else: matches = sorted(task_dir.glob("*/task.json")) + # With no replicate index, picking one of several would misattribute + # another replicate's failure detail to this row — degrade instead. + if len(matches) > 1: + return None candidate = matches[0] if matches else task_dir / "task.json" + try: + # Belt-and-braces containment check (catches symlink escapes too). + if not candidate.resolve().is_relative_to(run_dir.resolve()): + return None + # ValueError covers both json.JSONDecodeError and UnicodeDecodeError + # (undecodable bytes) — neither may abort report generation. return json.loads(candidate.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): + except (OSError, ValueError): return None @@ -102,7 +149,7 @@ def _criteria_body(row: dict[str, Any], run_dir: Path, variant: str) -> str: status + weighted-score line when task.json is missing/corrupt. Capped via the shared ``truncate``. """ - status = str(row.get("status")) + status = _status_of(row) data = _load_task_json(run_dir, row, variant) criteria = data.get("success_criteria_results") if isinstance(data, dict) else None @@ -135,16 +182,27 @@ def _criteria_body(row: dict[str, Any], run_dir: Path, variant: str) -> str: def _task_case(row: dict[str, Any], run_dir: Path) -> ET.Element: """Build one ```` for a task row.""" - variant = row.get("variant_id") or "default" + variant = _variant_of(row) task_id = str(row.get("task_id", "")) + + # `bool` is an int subclass — exclude it so True never renders as "[01]". replicate_index = row.get("replicate_index") - name = f"{task_id}[{replicate_index:02d}]" if isinstance(replicate_index, int) else task_id + has_replicate = isinstance(replicate_index, int) and not isinstance(replicate_index, bool) + name = f"{task_id}[{replicate_index:02d}]" if has_replicate else task_id task_path = row.get("task_path") - classname = Path(task_path).stem if task_path else variant + classname = (Path(task_path).stem or variant) if isinstance(task_path, str) and task_path else variant + # time must be a finite, non-negative float: NaN/inf would serialize as + # "nan"/"inf" and make the report invalid for JUnit ingesters. duration = row.get("duration") - time_str = f"{duration:.3f}" if isinstance(duration, int | float) else "0.000" + valid_duration = ( + isinstance(duration, int | float) + and not isinstance(duration, bool) + and math.isfinite(duration) + and duration >= 0 + ) + time_str = f"{duration:.3f}" if valid_duration else "0.000" case = ET.Element( "testcase", @@ -165,16 +223,12 @@ def _task_case(row: dict[str, Any], run_dir: Path) -> ET.Element: for key, value in props: ET.SubElement(properties, "property", {"name": key, "value": _xml_safe(str(value))}) - category = _category_of(str(row.get("status"))) + status = _status_of(row) + category = _category_of(status) if category == "succeeded": return case - status = str(row.get("status")) - try: - FinalStatus(status) - message = status - except ValueError: - message = f"unknown status: {status}" + message = status if status in _KNOWN_STATUSES else f"unknown status: {status}" tag = "failure" if category == "failed" else "error" child = ET.SubElement(case, tag, {"message": _xml_safe(message)}) child.text = _criteria_body(row, run_dir, variant) @@ -188,10 +242,13 @@ def _skipped_suite(summary: RunSummary) -> ET.Element | None: suite = ET.Element("testsuite", {"name": "skipped"}) cases: list[ET.Element] = [] for entry in summary.skipped_tasks: + # Use the suffix-stripped PATH, not just the stem: two skipped tasks + # sharing a basename (suiteA/task.yaml, suiteB/task.yaml) would otherwise + # collapse into one identity that some JUnit ingesters merge. case = ET.SubElement( suite, "testcase", - {"name": _xml_safe(Path(entry.path).stem), "classname": "skipped"}, + {"name": _xml_safe(str(Path(entry.path).with_suffix(""))), "classname": "skipped"}, ) ET.SubElement(case, "skipped", {"message": _xml_safe(entry.reason)}) cases.append(case) @@ -268,8 +325,7 @@ def generate_junit_xml(run_dir: Path) -> str: # Group task rows by variant, preserving first-seen order. grouped: dict[str, list[dict[str, Any]]] = {} for row in summary.task_results: - variant = row.get("variant_id") or "default" - grouped.setdefault(variant, []).append(row) + grouped.setdefault(_variant_of(row), []).append(row) all_cases: list[ET.Element] = [] for variant, rows in grouped.items(): diff --git a/tests/conftest.py b/tests/conftest.py index 023745d0..07cf3c7d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -84,7 +84,7 @@ def _build( counts = {"succeeded": 0, "failed": 0, "error": 0} for row in rows: try: - category = FinalStatus(row["status"]).category + category = FinalStatus(row.get("status")).category except ValueError: category = "error" # unknown status → error bucket (mirrors the writer) counts[category] += 1 diff --git a/tests/test_reports_junit.py b/tests/test_reports_junit.py index a84c42e0..0b041ffb 100644 --- a/tests/test_reports_junit.py +++ b/tests/test_reports_junit.py @@ -8,6 +8,7 @@ from __future__ import annotations import json +import math from collections.abc import Callable from pathlib import Path from typing import Any @@ -302,8 +303,9 @@ def test_skipped_tasks_suite(write_run_json: Callable[..., Path], tmp_path: Path assert ts is not None skipped_cases = ts.findall("testcase") assert len(skipped_cases) == 2 + # Suffix-stripped full path (not just the stem) so same-basename tasks stay distinct. names = {c.get("name") for c in skipped_cases} - assert names == {"broken", "opt"} + assert names == {"tasks/broken", "tasks/opt"} assert all(c.find("skipped") is not None for c in skipped_cases) @@ -456,3 +458,126 @@ def test_write_junit_xml_creates_parents_and_returns_path(write_run_json: Callab assert written == out assert out.is_file() fromstring(out.read_text(encoding="utf-8")) # parses + + +# -------------------------------------------------------------------------- +# Robustness against schema-skewed / crafted run.json rows (final-review findings) +# -------------------------------------------------------------------------- + + +def test_malformed_row_types_degrade_gracefully(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """Loose `task_results` dicts are untyped; a skewed row must not abort the report. + + Covers: bool replicate_index (bool is an int subclass), non-finite duration + (would emit an invalid time="nan"), non-str variant_id / task_path, and a + row with no `status` key at all. + """ + run_dir = tmp_path / "run" + rows: list[dict[str, Any]] = [ + {"task_id": "t_bool", "status": "SUCCESS", "replicate_index": True, "duration": 1.0}, + {"task_id": "t_nan", "status": "SUCCESS", "duration": float("nan")}, + {"task_id": "t_inf", "status": "SUCCESS", "duration": float("inf")}, + {"task_id": "t_neg", "status": "SUCCESS", "duration": -5.0}, + {"task_id": "t_variant", "status": "SUCCESS", "variant_id": 17}, + {"task_id": "t_path", "status": "SUCCESS", "task_path": 42}, + {"task_id": "t_nostatus"}, + ] + write_run_json(run_dir, rows) + + xml = generate_junit_xml(run_dir) + root = fromstring(xml) # must still be well-formed + + cases = [c for ts in root.findall("testsuite") for c in ts.findall("testcase")] + assert len(cases) == len(rows) + # Every time attribute must be a finite, non-negative float (JUnit requirement). + for case in cases: + t = float(case.get("time")) + assert math.isfinite(t) and t >= 0.0 # finite (not NaN/inf), non-negative + # A bool replicate_index must NOT be formatted as a [NN] replicate suffix. + names = {c.get("name") for c in cases} + assert "t_bool" in names + # The missing-status row lands in the error bucket, not silently as a pass. + assert int(root.get("errors")) == 1 + + +def test_task_json_invalid_utf8_falls_back(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """A task.json with undecodable bytes must degrade, not raise (UnicodeDecodeError + is a ValueError but NOT a json.JSONDecodeError).""" + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("t_fail", "FAILURE", variant_id="v1", replicate_index=0)]) + task_dir = run_dir / "v1" / "t_fail" / "00" + task_dir.mkdir(parents=True, exist_ok=True) + (task_dir / "task.json").write_bytes(b"\xff\xfe\x00 not utf-8") + + root = fromstring(generate_junit_xml(run_dir)) + body = _find_testsuite(root, "v1").find("testcase").find("failure").text or "" + assert "FAILURE" in body # status-only fallback body + + +def test_task_json_lookup_cannot_escape_run_dir(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """A crafted variant_id/task_id must not make the writer read outside run_dir.""" + outside = tmp_path / "outside" + outside.mkdir() + (outside / "task.json").write_text( + json.dumps( + { + "success_criteria_results": [ + { + "criterion_type": "leaked", + "description": "SECRET", + "score": 0.0, + "pass_threshold": 0.9, + "details": "LEAKED_SECRET_DETAIL", + "error": None, + } + ] + } + ), + encoding="utf-8", + ) + run_dir = tmp_path / "run" + write_run_json(run_dir, [{"task_id": "..", "status": "FAILURE", "variant_id": "../outside"}]) + + root = fromstring(generate_junit_xml(run_dir)) + xml_text = generate_junit_xml(run_dir) + assert "LEAKED_SECRET_DETAIL" not in xml_text + body = root.find("testsuite").find("testcase").find("failure").text or "" + assert "FAILURE" in body # fell back, did not read the outside file + + +def test_ambiguous_replicate_does_not_misattribute(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """replicate_index=None with MULTIPLE replicate dirs must fall back rather than + attribute replicate 00's failure detail to the row.""" + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("task", "FAILURE", variant_id="v1", replicate_index=None)]) + for idx, marker in ((0, "REP0ONLY"), (1, "REP1ONLY")): + _write_task_json( + run_dir, + "v1", + "task", + idx, + [ + { + "criterion_type": "c", + "description": "d", + "score": 0.0, + "pass_threshold": 0.9, + "details": marker, + "error": None, + } + ], + ) + root = fromstring(generate_junit_xml(run_dir)) + body = _find_testsuite(root, "v1").find("testcase").find("failure").text or "" + assert "REP0ONLY" not in body and "REP1ONLY" not in body + assert "FAILURE" in body + + +def test_skipped_names_unique_for_same_stem(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """Two skipped tasks sharing a basename must get distinct testcase identities.""" + run_dir = tmp_path / "run" + write_run_json(run_dir, [], skipped=[("suiteA/task.yaml", "skip: true"), ("suiteB/task.yaml", "skip: true")]) + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "skipped") + names = {c.get("name") for c in ts.findall("testcase")} + assert len(names) == 2, f"skipped testcase names collided: {names}" From 870b22f6cf2cbbc056548e81553808757571d718 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 18:32:43 -0700 Subject: [PATCH 6/8] fix(reports): make skipped-task JUnit names platform-independent _skipped_suite built the testcase name with str(Path(path).with_suffix("")), which yields "tasks\opt" on Windows and "tasks/opt" on Linux. That made the JUnit testcase identity depend on the OS that generated the report, so the same logical run would split into two identities in CI history/flake tracking (and it broke the Windows smoke job). Normalize separators and parse with PurePosixPath so the emitted name is always "/"-separated. Adds a regression test feeding a Windows-style path. Also switch the action's model in the README/tutorial examples from Haiku to Sonnet (docs only; the dogfood job stays on Haiku for per-PR cost). Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 2 +- docs/tutorials/02-ci-pipeline.md | 2 +- src/coder_eval/reports_junit.py | 22 +++++++++++++++++----- tests/test_reports_junit.py | 15 ++++++++++++++- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ccb79aff..a923d991 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ the job summary, and fails the step on any task/gate failure: - uses: UiPath/coder_eval@v0 # becomes @v1 once 1.0.0 ships; @vX.Y.Z pins exactly with: tasks: tests/tasks/**/*.yaml - model: claude-haiku-4-5-20251001 + model: claude-sonnet-5 anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} ``` diff --git a/docs/tutorials/02-ci-pipeline.md b/docs/tutorials/02-ci-pipeline.md index c19f3309..1e7fb200 100644 --- a/docs/tutorials/02-ci-pipeline.md +++ b/docs/tutorials/02-ci-pipeline.md @@ -172,7 +172,7 @@ job-summary + fail-on-failure into one step: - uses: UiPath/coder_eval@v0 # …then run the gate (pin @vX.Y.Z in production) with: tasks: tests/tasks/**/*.yaml - model: claude-haiku-4-5-20251001 + model: claude-sonnet-5 anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} ``` diff --git a/src/coder_eval/reports_junit.py b/src/coder_eval/reports_junit.py index a6c3cb5f..9786601e 100644 --- a/src/coder_eval/reports_junit.py +++ b/src/coder_eval/reports_junit.py @@ -24,7 +24,7 @@ import math import re import xml.etree.ElementTree as ET -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any, Literal from .evaluation.judge_context import truncate @@ -235,6 +235,21 @@ def _task_case(row: dict[str, Any], run_dir: Path) -> ET.Element: return case +def _skipped_name(path: str) -> str: + """Stable testcase name for a skipped task: suffix-stripped, ``/``-separated. + + Uses the whole path rather than just the stem, because two skipped tasks + sharing a basename (``suiteA/task.yaml``, ``suiteB/task.yaml``) would + otherwise collapse into one identity that some JUnit ingesters merge. + + Separators are normalized to ``/`` and the path is parsed with + ``PurePosixPath`` so the emitted name does not depend on the OS that + generated the report — the same logical run must produce the same testcase + identity on Windows and Linux, or CI history/flake tracking splits in two. + """ + return str(PurePosixPath(path.replace("\\", "/")).with_suffix("")) + + def _skipped_suite(summary: RunSummary) -> ET.Element | None: """Build the synthetic ``skipped`` testsuite from ``RunSummary.skipped_tasks``.""" if not summary.skipped_tasks: @@ -242,13 +257,10 @@ def _skipped_suite(summary: RunSummary) -> ET.Element | None: suite = ET.Element("testsuite", {"name": "skipped"}) cases: list[ET.Element] = [] for entry in summary.skipped_tasks: - # Use the suffix-stripped PATH, not just the stem: two skipped tasks - # sharing a basename (suiteA/task.yaml, suiteB/task.yaml) would otherwise - # collapse into one identity that some JUnit ingesters merge. case = ET.SubElement( suite, "testcase", - {"name": _xml_safe(str(Path(entry.path).with_suffix(""))), "classname": "skipped"}, + {"name": _xml_safe(_skipped_name(entry.path)), "classname": "skipped"}, ) ET.SubElement(case, "skipped", {"message": _xml_safe(entry.reason)}) cases.append(case) diff --git a/tests/test_reports_junit.py b/tests/test_reports_junit.py index 0b041ffb..73f7ef18 100644 --- a/tests/test_reports_junit.py +++ b/tests/test_reports_junit.py @@ -303,9 +303,12 @@ def test_skipped_tasks_suite(write_run_json: Callable[..., Path], tmp_path: Path assert ts is not None skipped_cases = ts.findall("testcase") assert len(skipped_cases) == 2 - # Suffix-stripped full path (not just the stem) so same-basename tasks stay distinct. + # Suffix-stripped full path (not just the stem) so same-basename tasks stay + # distinct. Always '/'-separated: the same logical run must yield the same + # testcase identity on Windows and Linux, or CI history splits in two. names = {c.get("name") for c in skipped_cases} assert names == {"tasks/broken", "tasks/opt"} + assert not any("\\" in n for n in names) assert all(c.find("skipped") is not None for c in skipped_cases) @@ -581,3 +584,13 @@ def test_skipped_names_unique_for_same_stem(write_run_json: Callable[..., Path], ts = _find_testsuite(root, "skipped") names = {c.get("name") for c in ts.findall("testcase")} assert len(names) == 2, f"skipped testcase names collided: {names}" + + +def test_skipped_name_is_platform_independent(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """A Windows-style path recorded in run.json must still emit a '/'-separated + name, so a report generated on Windows matches one generated on Linux.""" + run_dir = tmp_path / "run" + write_run_json(run_dir, [], skipped=[(r"tasks\win\task.yaml", "skip: true")]) + root = fromstring(generate_junit_xml(run_dir)) + name = _find_testsuite(root, "skipped").find("testcase").get("name") + assert name == "tasks/win/task" From 95fb7b2acbafb574cbbd301f0f5c2124c1e3dc88 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 18:53:31 -0700 Subject: [PATCH 7/8] chore: re-trigger CI (GitHub dropped the force-push event) Co-Authored-By: Claude From 3b00991cd03e35bcbf0ae48e3989f595ecab6f5c Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 22 Jul 2026 16:42:33 -0700 Subject: [PATCH 8/8] feat(action): generic env passthrough + minimum-task-score gate Replace the single `anthropic-api-key` input with a generic `env` NAME=VALUE passthrough (exported for the run step only, never $GITHUB_ENV) and add an optional `minimum-task-score` floor read from the always-written run.json spine. Both default off, so existing behavior is unchanged. README, tutorial, and the dogfood job route ANTHROPIC_API_KEY through `env`. Code-review hardening (gemini-3.1-pro + gpt-5.3-codex + opus): - score gate skips non-finite weighted_score (json.loads parses NaN; NaN makes min()/>= order-dependent and could mask a below-floor task) - minimum-task-score validated (numeric, finite, in [0.0, 1.0]) with a clean ::error:: instead of a raw traceback - malformed env entries reported by position, never echoing the raw line, so a mis-prefixed secret can't leak into logs Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/pr-checks.yml | 10 ++- README.md | 25 +++++++- action.yml | 107 +++++++++++++++++++++++++++++-- docs/tutorials/02-ci-pipeline.md | 3 +- 4 files changed, 137 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index f985560f..9f55e5b1 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -860,7 +860,15 @@ jobs: model: claude-haiku-4-5-20251001 run-dir: runs/ci-action-dogfood junit-path: runs/ci-action-dogfood/junit.xml - anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + # Credentials go through the generic env passthrough (the only channel); + # ANTHROPIC_API_KEY reaching the run is proven by the API-backed task + # succeeding. A floor of 0.0 passes for any produced score (exercises + # the gate path green in CI without flakiness); the second line + # exercises multi-line env parsing. + minimum-task-score: "0.0" + env: | + ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} + CE_DOGFOOD_MARKER=1 - name: Verify outputs and JUnit file env: diff --git a/README.md b/README.md index a923d991..3d5dd4e3 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,8 @@ the job summary, and fails the step on any task/gate failure: with: tasks: tests/tasks/**/*.yaml model: claude-sonnet-5 - anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + env: | + ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} ``` | Input | Default | Purpose | @@ -120,7 +121,8 @@ the job summary, and fails the step on any task/gate failure: | `run-dir` | `runs/ci` | Run directory | | `junit-path` | `coder-eval-junit.xml` | Where to write the JUnit report | | `step-summary` | `true` | Append `run.md` to the job summary | -| `anthropic-api-key` | — | Exported as `ANTHROPIC_API_KEY` for the run | +| `env` | — | Credentials/backend passthrough: newline-separated `NAME=VALUE` pairs, exported for the run step only | +| `minimum-task-score` | *(off)* | Strict floor (0.0–1.0): fail the step if any task's `weighted_score` is below it | Outputs: `run-dir` and `junit-path`. Feed the JUnit file to your platform's test-report renderer — e.g. on GitHub Actions with @@ -133,6 +135,25 @@ test-report renderer — e.g. on GitHub Actions with report_paths: coder-eval-junit.xml ``` +**Credentials and backend config** are the sole responsibility of `env` — a +passthrough exported for the run step only (never written to `$GITHUB_ENV`, so +it can't leak into later steps). Set whatever the run needs, Anthropic or not: + +```yaml +- uses: UiPath/coder_eval@v0 + with: + tasks: tests/tasks/**/*.yaml + minimum-task-score: "0.8" # fail the build if any task scores below 0.8 + env: | + CODER_EVAL_API_BACKEND=bedrock + AWS_BEARER_TOKEN_BEDROCK=${{ secrets.BEDROCK_TOKEN }} +``` + +`minimum-task-score` is a strict floor **on top of** coder-eval's own exit +code: the step fails if *either* coder-eval exits non-zero *or* any task's +`weighted_score` falls below the floor. Leave it unset to gate on the exit code +alone. + > **Agent runtime is the caller's responsibility.** The action is agent-agnostic — > it installs `coder-eval` but no coding-agent runtime. Tasks using the default > `claude-code` agent need the `claude` CLI on `PATH` (`actions/setup-node` + diff --git a/action.yml b/action.yml index c9006644..07b15081 100644 --- a/action.yml +++ b/action.yml @@ -46,8 +46,25 @@ inputs: description: Append run.md to the GitHub job summary ("true"/"false") required: false default: "true" - anthropic-api-key: - description: Anthropic API key (exported as ANTHROPIC_API_KEY for the run step) + env: + description: >- + Environment passthrough — newline-separated NAME=VALUE pairs exported for + the coder-eval process only (scoped to the run step; NOT written to + $GITHUB_ENV, so nothing leaks into later job steps). This is the sole + channel for credentials and backend config: set ANTHROPIC_API_KEY, + CODER_EVAL_API_BACKEND, model vars, EVALBOARD_*, plugin paths, etc. Names + must match ^[A-Za-z_][A-Za-z0-9_]*$; blank lines and `#` comments are + ignored. Wire values from repository secrets (secrets.MY_KEY) — never + inline a secret literal. + required: false + default: "" + minimum-task-score: + description: >- + Optional strict floor (0.0–1.0): EVERY scored task, in every variant, must + reach it or the step fails. A gate ON TOP OF coder-eval's own exit code — + the step fails if EITHER coder-eval exits non-zero OR any task's + weighted_score is below this. Empty (the default) disables the floor, + leaving coder-eval's exit code the sole gate. required: false default: "" @@ -80,7 +97,6 @@ runs: id: run shell: bash env: - ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }} CE_TASKS: ${{ inputs.tasks }} CE_TAGS: ${{ inputs.tags }} CE_MODEL: ${{ inputs.model }} @@ -88,8 +104,36 @@ runs: CE_RUN_DIR: ${{ inputs.run-dir }} CE_JUNIT: ${{ inputs.junit-path }} CE_SUMMARY: ${{ inputs.step-summary }} + CE_ENV: ${{ inputs.env }} + CE_MIN_SCORE: ${{ inputs.minimum-task-score }} run: | set -uo pipefail + + # Generic env passthrough. Each NAME=VALUE line is exported for the + # coder-eval child of THIS step only — deliberately not written to + # $GITHUB_ENV, so a forwarded secret never bleeds into later job steps. + # Only NAME is validated; VALUE is treated as opaque data (never eval'd), + # so no crafted value can inject shell. + n=0 + while IFS= read -r line; do + n=$((n + 1)) + line="${line%$'\r'}" # tolerate CRLF inputs + line="${line#"${line%%[![:space:]]*}"}" # left-trim + [ -z "$line" ] && continue + case "$line" in '#'*) continue ;; esac # allow comment lines + # Never echo $line/$value on error — a caller who forgets the `NAME=` + # prefix would otherwise print a forwarded secret verbatim (GitHub + # only masks values it knows are secrets). Report by position instead. + if [ "$line" = "${line#*=}" ]; then + echo "::error::env entry #$n is not NAME=VALUE (no '=' found)"; exit 1 + fi + name="${line%%=*}" + if [[ ! "$name" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then + echo "::error::env entry #$n has an invalid name (must match ^[A-Za-z_][A-Za-z0-9_]*\$)"; exit 1 + fi + export "$name=${line#*=}" + done <<< "$CE_ENV" + args=(run --run-dir "$CE_RUN_DIR" --junit-xml "$CE_JUNIT") [ -n "$CE_TAGS" ] && args+=(--tags "$CE_TAGS") [ -n "$CE_MODEL" ] && args+=(--model "$CE_MODEL") @@ -107,4 +151,59 @@ runs: if [ "$CE_SUMMARY" = "true" ] && [ -f "$CE_RUN_DIR/run.md" ]; then head -c 1000000 "$CE_RUN_DIR/run.md" >> "$GITHUB_STEP_SUMMARY" fi - exit $CODE + + # Optional per-task score floor. Reads the always-written run.json spine + # (task_results[*].weighted_score) rather than experiment.json, so it + # works for plain and experiment runs alike. A gate ON TOP OF CODE: + # empty CE_MIN_SCORE disables it, leaving CODE the sole verdict. Runs + # regardless of CODE (run.json is emitted even on a red run), then the + # two verdicts are combined so both surface. + GATE=0 + if [ -n "$CE_MIN_SCORE" ]; then + if [ ! -f "$CE_RUN_DIR/run.json" ]; then + echo "::error::score gate: $CE_RUN_DIR/run.json not found (cannot verify minimum-task-score=$CE_MIN_SCORE)" + GATE=1 + else + RUN_JSON="$CE_RUN_DIR/run.json" MIN_SCORE="$CE_MIN_SCORE" python3 <<'PY' || GATE=1 + import json, math, os, sys + + raw = os.environ["MIN_SCORE"] + try: + floor = float(raw) + except ValueError: + print(f"::error::minimum-task-score must be a number, got '{raw}'") + sys.exit(1) + if not math.isfinite(floor) or not (0.0 <= floor <= 1.0): + print(f"::error::minimum-task-score must be within [0.0, 1.0], got '{raw}'") + sys.exit(1) + data = json.load(open(os.environ["RUN_JSON"], encoding="utf-8")) + rows = [] + for r in data.get("task_results", []): + s = r.get("weighted_score") + # bool is an int subclass — exclude it; skip errored rows (score None), + # which coder-eval's own exit code already accounts for; and skip + # non-finite (NaN/inf) — json.loads accepts them, and NaN makes + # min()/>= order-dependent, which could silently MASK a below-floor + # task. A blob-pulled/malformed run.json must fail closed, not pass. + if isinstance(s, (int, float)) and not isinstance(s, bool) and math.isfinite(s): + rows.append((float(s), str(r.get("task_id", "?")), str(r.get("variant_id") or "default"))) + for s, t, v in sorted(rows): + print(f" [{'ok' if s >= floor else 'BELOW'}] {v}/{t}: {s:.3f}") + worst = min(rows) if rows else None + ok = worst is not None and worst[0] >= floor + if ok: + print(f"score gate passed: lowest {worst[0]:.3f} >= floor {floor:.3f}") + elif worst is not None: + print(f"::error::score gate FAILED: lowest {worst[0]:.3f} ({worst[2]}/{worst[1]}) < floor {floor:.3f}") + else: + print(f"::error::score gate: no scored tasks in run.json to check against floor {floor:.3f}") + sys.exit(0 if ok else 1) + PY + fi + fi + + # Combine: coder-eval's own failure OR the score floor fails the step. + if [ "$CODE" -ne 0 ]; then + exit "$CODE" + fi + [ "$GATE" -eq 0 ] || exit 1 diff --git a/docs/tutorials/02-ci-pipeline.md b/docs/tutorials/02-ci-pipeline.md index 1e7fb200..6943ae79 100644 --- a/docs/tutorials/02-ci-pipeline.md +++ b/docs/tutorials/02-ci-pipeline.md @@ -173,7 +173,8 @@ job-summary + fail-on-failure into one step: with: tasks: tests/tasks/**/*.yaml model: claude-sonnet-5 - anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + env: | + ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} ``` The action is agent-agnostic — it installs `coder-eval` but not the agent