diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 28578b09..f985560f 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -831,3 +831,53 @@ jobs: name: byoa-live-output path: tmp/ retention-days: 7 + + action-dogfood: + name: Action Dogfood (composite action, real API) + runs-on: ubuntu-latest + timeout-minutes: 15 + # Skip on fork PRs where secrets aren't available (matches e2e-smoke). + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # The composite action is agent-agnostic and does NOT install a coding-agent + # runtime. The dogfood task uses the default claude-code agent, so provide + # Node + the Claude CLI here (as e2e-smoke does), before invoking the action. + - name: Set up Node.js 20 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "20" + - name: Install Claude CLI + run: npm install -g @anthropic-ai/claude-code + + - name: Run coder-eval via local action + id: dogfood + uses: ./ + with: + version: local + tasks: tasks/hello_date.yaml + model: claude-haiku-4-5-20251001 + run-dir: runs/ci-action-dogfood + junit-path: runs/ci-action-dogfood/junit.xml + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + + - name: Verify outputs and JUnit file + env: + JUNIT: ${{ steps.dogfood.outputs.junit-path }} + RUNDIR: ${{ steps.dogfood.outputs.run-dir }} + run: | + set -euo pipefail + test -n "$JUNIT" && test -f "$JUNIT" || { echo "junit output missing"; exit 1; } + # Well-formedness check on a file this job just generated (trusted input; + # our writer emits no DTDs/entities) — stdlib ET is fine here. + python3 -c "import sys, xml.etree.ElementTree as ET; ET.parse(sys.argv[1])" "$JUNIT" + test -f "$RUNDIR/run.json" || { echo "run.json missing"; exit 1; } + + - name: Upload dogfood run on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: action-dogfood-runs + path: runs/ci-action-dogfood/ + retention-days: 7 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ff2a7ebb..7e65a649 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -182,23 +182,52 @@ jobs: echo "version=$V" >> "$GITHUB_OUTPUT" echo "Publishing version: $V" - - name: Regenerate uv.lock and amend release commit + - name: Regenerate uv.lock, bump action.yml pin, and amend release commit if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != '' + env: + # Passed via env (not interpolated into the script) per GitHub's + # injection guidance. + VERSION: ${{ steps.release.outputs.version }} run: | + set -euo pipefail + git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.name "github-actions[bot]" + # Bump the composite action's default `version:` pin to the just-released + # version so `UiPath/coder_eval@vX.Y.Z` installs `coder-eval==X.Y.Z`. The + # anchor is indentation-tolerant and keyed on the unique trailing + # "# <-- kept in sync" comment; the grep guard fails the release loudly + # if a reformat ever detaches it (rather than shipping a stale pin). + sed -i -E 's/^([[:space:]]*default: ")[0-9]+\.[0-9]+\.[0-9]+(" # <-- kept in sync)/\1'"${VERSION}"'\2/' action.yml + grep -q "default: \"${VERSION}\"" action.yml || { echo "action.yml version bump failed"; exit 1; } + git add action.yml + # Regenerate the lock too; stage it (a no-op if unchanged). uv lock - if ! git diff --quiet uv.lock; then - git config user.email "github-actions[bot]@users.noreply.github.com" - git config user.name "github-actions[bot]" - git add uv.lock + git add uv.lock + # Amend only if action.yml/uv.lock actually changed the tree. + if ! git diff --cached --quiet; then git commit --amend --no-edit # Amend replaced the commit the tag points at; re-point it before pushing. - git tag -f "v${{ steps.release.outputs.version }}" + git tag -f "v${VERSION}" fi - name: Push release commit and tags if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != '' run: git push origin main "v${{ steps.release.outputs.version }}" + - name: Move major action tag (vN -> this release) + # Real releases only: a prerelease never tags or pushes to main, so it must + # not move the tag consumers pin (mirrors `:latest` staying put). + if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != '' + env: + VERSION: ${{ steps.release.outputs.version }} + run: | + set -euo pipefail + # Consumers pin `UiPath/coder_eval@v0` (becomes `@v1` at 1.0.0). Force-move + # the moving major tag to this release. Force on a missing tag creates it. + MAJOR="v${VERSION%%.*}" + git tag -f "$MAJOR" "v${VERSION}" + git push -f origin "$MAJOR" + - name: Build wheel + sdist if: steps.ver.outputs.version != '' run: uv build diff --git a/CLAUDE.md b/CLAUDE.md index 5c5cf392..ed636bf2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,6 +20,7 @@ coder_eval/ ├── orchestrator.py # Main evaluation loop ├── reports.py # Markdown/JSON report generation (run-level + per-suite rollup via write_suite_rollups) ├── reports_experiment.py # Experiment/cross-variant report generation +├── reports_junit.py # JUnit XML report from a finalized run dir (run.json spine; for CI test-report ingestion) ├── analysis.py # Command statistics aggregation ├── logging_config.py # Structured logging setup ├── path_utils.py # Run ID generation, path utilities @@ -120,6 +121,7 @@ tasks/ # Task definition YAML files tests/ # Test suite docs/ # Documentation templates/ # Sandbox template directories +action.yml # Published composite GitHub Action (coder-eval as a CI gate). release.yml maintains its `version:` default + the moving `v` tag. ``` ## Key Architectural Patterns diff --git a/README.md b/README.md index 06e72a0b..c47809fc 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,53 @@ live in this repo — clone it or point the CLI at your own task files.) See [Tutorial 02 — Running coder_eval in CI](docs/tutorials/02-ci-pipeline.md) for the full setup. +## Use as a GitHub Action + +A composite action at the repo root runs `coder-eval` as a CI gate — it installs +the pinned CLI, runs your tasks, writes a JUnit XML report, appends `run.md` to +the job summary, and fails the step on any task/gate failure: + +```yaml +- uses: UiPath/coder_eval@v0 # becomes @v1 once 1.0.0 ships; @vX.Y.Z pins exactly + with: + tasks: tests/tasks/**/*.yaml + model: claude-sonnet-5 + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} +``` + +| Input | Default | Purpose | +| --- | --- | --- | +| `tasks` | *(all `tasks/`)* | Task YAML path(s)/glob | +| `tags` | — | `--tags` filter | +| `model` | — | `--model` override | +| `extra-args` | — | Verbatim extra args (`--experiment`, `-D …`, …) | +| `version` | pinned release | PyPI version, or `local` to install from the checkout | +| `run-dir` | `runs/ci` | Run directory | +| `junit-path` | `coder-eval-junit.xml` | Where to write the JUnit report | +| `step-summary` | `true` | Append `run.md` to the job summary | +| `anthropic-api-key` | — | Exported as `ANTHROPIC_API_KEY` for the run | + +Outputs: `run-dir` and `junit-path`. Feed the JUnit file to your platform's +test-report renderer — e.g. on GitHub Actions with +[`mikepenz/action-junit-report`](https://github.com/mikepenz/action-junit-report): + +```yaml +- uses: mikepenz/action-junit-report@v5 + if: always() + with: + report_paths: coder-eval-junit.xml +``` + +> **Agent runtime is the caller's responsibility.** The action is agent-agnostic — +> it installs `coder-eval` but no coding-agent runtime. Tasks using the default +> `claude-code` agent need the `claude` CLI on `PATH` (`actions/setup-node` + +> `npm install -g @anthropic-ai/claude-code`) in the job before the action runs. + +> **Security.** Evaluated tasks execute agent-generated code. Do **not** run this +> action under `pull_request_target` with secrets exposed to untrusted fork PRs — +> use `pull_request` and gate on the same-repo condition, as this repo's own +> dogfood job does. + ## Telemetry > 📊 **Usage telemetry is on by default.** `coder-eval` sends **anonymous** usage diff --git a/action.yml b/action.yml new file mode 100644 index 00000000..2c03737d --- /dev/null +++ b/action.yml @@ -0,0 +1,110 @@ +name: coder-eval +description: Run coder-eval evaluation tasks as a CI gate, with JUnit XML output and a job-summary report. +branding: + icon: check-circle + color: orange + +# This action installs and runs the `coder-eval` CLI. It is agent-agnostic: it +# does NOT install any coding-agent runtime. Tasks that use the default +# `claude-code` agent need the `claude` CLI on PATH (Node + the +# `@anthropic-ai/claude-code` npm package) provided by the calling job before +# this action runs. See the README "Use as a GitHub Action" section. +# +# SECURITY: evaluated tasks execute agent-generated code. Do NOT run this action +# under `pull_request_target` with secrets exposed to untrusted fork PRs. + +inputs: + tasks: + description: Task YAML path(s)/glob passed to `coder-eval run` (empty = all tasks/ recursively) + required: false + default: "" + tags: + description: Only run tasks matching any of these comma-separated tags (--tags) + required: false + default: "" + model: + description: Override agent model for all tasks (--model) + required: false + default: "" + extra-args: + description: Extra arguments appended verbatim to `coder-eval run` (trusted caller input; covers --experiment, -D overrides, --tags exclusions, etc.) + required: false + default: "" + version: + description: coder-eval version to install from PyPI, or "local" to install from the action checkout + required: false + default: "0.8.7" # <-- kept in sync with releases by release.yml + run-dir: + description: Run directory (--run-dir) + required: false + default: "runs/ci" + junit-path: + description: Where to write the JUnit XML report + required: false + default: "coder-eval-junit.xml" + step-summary: + description: Append run.md to the GitHub job summary ("true"/"false") + required: false + default: "true" + anthropic-api-key: + description: Anthropic API key (exported as ANTHROPIC_API_KEY for the run step) + required: false + default: "" + +outputs: + run-dir: + description: The run directory containing run.json/run.md + value: ${{ steps.run.outputs.run-dir }} + junit-path: + description: Path to the written JUnit XML report + value: ${{ steps.run.outputs.junit-path }} + +runs: + using: composite + steps: + - name: Install uv + uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 + - name: Install coder-eval + shell: bash + env: + CE_VERSION: ${{ inputs.version }} + CE_ACTION_PATH: ${{ github.action_path }} + run: | + set -euo pipefail + if [ "$CE_VERSION" = "local" ]; then + uv tool install "$CE_ACTION_PATH" + else + uv tool install "coder-eval==$CE_VERSION" + fi + - name: Run coder-eval + id: run + shell: bash + env: + ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }} + CE_TASKS: ${{ inputs.tasks }} + CE_TAGS: ${{ inputs.tags }} + CE_MODEL: ${{ inputs.model }} + CE_EXTRA_ARGS: ${{ inputs.extra-args }} + CE_RUN_DIR: ${{ inputs.run-dir }} + CE_JUNIT: ${{ inputs.junit-path }} + CE_SUMMARY: ${{ inputs.step-summary }} + run: | + set -uo pipefail + args=(run --run-dir "$CE_RUN_DIR" --junit-xml "$CE_JUNIT") + [ -n "$CE_TAGS" ] && args+=(--tags "$CE_TAGS") + [ -n "$CE_MODEL" ] && args+=(--model "$CE_MODEL") + # extra-args is a trusted caller input, split on whitespace intentionally + # shellcheck disable=SC2206 + [ -n "$CE_EXTRA_ARGS" ] && args+=($CE_EXTRA_ARGS) + # shellcheck disable=SC2206 + [ -n "$CE_TASKS" ] && args+=($CE_TASKS) + set +e + coder-eval "${args[@]}" + CODE=$? + set -e + echo "run-dir=$CE_RUN_DIR" >> "$GITHUB_OUTPUT" + echo "junit-path=$CE_JUNIT" >> "$GITHUB_OUTPUT" + if [ "$CE_SUMMARY" = "true" ] && [ -f "$CE_RUN_DIR/run.md" ]; then + head -c 1000000 "$CE_RUN_DIR/run.md" >> "$GITHUB_STEP_SUMMARY" + fi + exit $CODE diff --git a/docs/tutorials/02-ci-pipeline.md b/docs/tutorials/02-ci-pipeline.md index 7dc04eea..1e7fb200 100644 --- a/docs/tutorials/02-ci-pipeline.md +++ b/docs/tutorials/02-ci-pipeline.md @@ -158,6 +158,77 @@ jobs: - **`if: always()`** on the verdict and upload steps means you still get results even when a task fails. +## Shortcut: the packaged action + +The five steps above spell out the mechanics, but coder_eval also ships a +composite action at the repo root that bundles install + run + JUnit report + +job-summary + fail-on-failure into one step: + +```yaml + - uses: actions/setup-node@v4 # the claude-code agent needs the Claude CLI… + with: { node-version: '20' } + - run: npm install -g @anthropic-ai/claude-code + + - uses: UiPath/coder_eval@v0 # …then run the gate (pin @vX.Y.Z in production) + with: + tasks: tests/tasks/**/*.yaml + model: claude-sonnet-5 + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} +``` + +The action is agent-agnostic — it installs `coder-eval` but not the agent +runtime, so provide the `claude` CLI (or your agent's runtime) in the job first. +See the [README "Use as a GitHub Action"](../../README.md#use-as-a-github-action) +section for the full inputs table and the fork/`pull_request_target` security +caveat. The rest of this tutorial's hand-rolled workflow is still useful when you +need finer control than the action's inputs expose. + +## Publishing test results (JUnit) + +The verdict step above parses `task.json` by hand. If your CI already ingests +JUnit XML — for the per-test annotations, history, and flake tracking most CI +systems render from it — let `coder-eval` emit it directly instead. Add +`--junit-xml` to the run: + +```yaml + - name: Run evaluations + run: coder-eval run $TASK_GLOBS -j 4 --junit-xml coder-eval-junit.xml +``` + +The file is written **before** the exit-code decision, so a failing run still +produces a report. You can also regenerate it from any existing run directory +without re-running the suite: + +```bash +coder-eval report runs/latest -f junit # writes runs/latest/junit.xml +coder-eval report runs/latest -f junit -o out.xml # custom path +``` + +The mapping: one `` per task row (grouped into a `` per +variant), failed/errored rows carry a ``/`` with the per-criterion +breakdown, skipped tasks and failing suite gates get their own synthetic suites. + +Feed the file to whatever your platform uses to render test results. On GitHub +Actions, [`mikepenz/action-junit-report`](https://github.com/mikepenz/action-junit-report): + +```yaml + - name: Publish test report + if: always() + uses: mikepenz/action-junit-report@v5 + with: + report_paths: coder-eval-junit.xml +``` + +On Azure DevOps, `PublishTestResults@2`: + +```yaml + - task: PublishTestResults@2 + condition: always() + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: 'coder-eval-junit.xml' +``` + ## Secrets Add these under **Settings → Secrets and variables → Actions**: diff --git a/pyproject.toml b/pyproject.toml index 2fecf15e..a09cd45a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,7 @@ dev = [ "pip-audit>=2.10.0", "bandit[toml]>=1.9.4", "pre-commit>=4.5.1", + "defusedxml>=0.7.1", # test-side JUnit XML round-trip parsing (defense-in-depth; production code never parses XML) ] # Optional extra that enables UiPath-specific features: # - the `uipath` Python SDK on the host (handy for local sandbox parity with diff --git a/src/coder_eval/cli/report_command.py b/src/coder_eval/cli/report_command.py index 4d801c99..6c5fcfc4 100644 --- a/src/coder_eval/cli/report_command.py +++ b/src/coder_eval/cli/report_command.py @@ -27,7 +27,10 @@ def report_command( "md", "--format", "-f", - help="Output format: 'md' (default markdown), 'html' (render task.json files as HTML).", + help=( + "Output format: 'md' (default markdown), 'html' (render task.json files as HTML), " + "'junit' (JUnit XML from run.json)." + ), ), ) -> None: """Display or export a run report. @@ -44,16 +47,31 @@ def report_command( # Re-generate HTML reports for every task.json under a run dir coder-eval report runs/latest --format html + + # Write a JUnit XML report (defaults to /junit.xml) + coder-eval report runs/latest --format junit """ fmt = report_format.lower() - if fmt not in ("md", "html"): - console.print(f"[red]Error: unknown --format '{report_format}' (expected 'md' or 'html')[/red]") + if fmt not in ("md", "html", "junit"): + console.print(f"[red]Error: unknown --format '{report_format}' (expected 'md', 'html', or 'junit')[/red]") raise typer.Exit(1) if fmt == "html": _regenerate_html_reports(run_dir, output_file) return + if fmt == "junit": + from ..reports_junit import write_junit_xml + + target = output_file if output_file is not None else run_dir / "junit.xml" + try: + written = write_junit_xml(run_dir, target) + except FileNotFoundError as e: + console.print(f"[red]Error: {e}[/red]") + raise typer.Exit(1) from e + console.print(f"[green][OK]JUnit report written to {written}[/green]") + return + try: report_md, source_path = ReportGenerator.load_from_run_dir(run_dir) console.print(f"[dim]Reading report from {source_path}[/dim]\n") diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index b35366ca..dff081da 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -173,6 +173,11 @@ def run_command( "--log-file", help="Log to file in addition to console", ), + junit_xml: Path | None = typer.Option( # noqa: B008 + None, + "--junit-xml", + help="Write a JUnit XML report of task results to this path (for CI test-report ingestion).", + ), tags: str | None = typer.Option( None, "--tags", @@ -365,6 +370,7 @@ def run_command( verbose=verbose, resume=resume, include_skipped=include_skipped, + junit_xml=junit_xml, ) ) except KeyboardInterrupt: @@ -389,6 +395,7 @@ async def _run_all_tasks( verbose: bool = False, resume: bool = False, include_skipped: bool = False, + junit_xml: Path | None = None, ) -> None: """Async entry point for running all tasks (optionally in parallel). @@ -407,6 +414,8 @@ async def _run_all_tasks( from -D/--set and the bespoke flag aliases stream_mode: Optional stream mode ('full' or 'minimal') for real-time output experiment_path: Optional path to experiment YAML (default: experiments/default.yaml) + junit_xml: Optional path to write a JUnit XML report to, after the run + summary is persisted and before the failure exit-code gate. """ # Prepare run directory run_dir = prepare_run_directory(run_dir) @@ -464,6 +473,16 @@ async def _run_all_tasks( # Print execution summary print_execution_summary(run_dir, summary) + + # Write the JUnit report (if requested) BEFORE the exit-code gate below, + # so a failing run still produces the report. suite.json + run.json are + # already on disk (written inside _run_with_experiment). A write error + # propagates (loud failure, exit != 0) rather than being swallowed. + if junit_xml is not None: + from ..reports_junit import write_junit_xml + + written = write_junit_xml(run_dir, junit_xml) + console.print(f"[green][OK]JUnit report written to {written}[/green]") finally: # Explicit flush before process exit (belt-and-suspenders with atexit). # In a `finally` so it runs on the success path and on any raised diff --git a/src/coder_eval/reports_junit.py b/src/coder_eval/reports_junit.py new file mode 100644 index 00000000..9786601e --- /dev/null +++ b/src/coder_eval/reports_junit.py @@ -0,0 +1,378 @@ +"""Disk-driven JUnit XML report generation from a finalized run directory. + +Reads a run directory's on-disk contracts — ``run.json`` (required, the +``RunSummary`` spine), ``*/*/suite.json`` (optional suite gates), and per-failed +row ``task.json`` (optional, best-effort failure detail) — and produces a JUnit +XML string that CI test-report ingesters (GitHub ``mikepenz/action-junit-report``, +Azure DevOps ``PublishTestResults@2``, …) understand. + +Single code path: ``coder-eval run --junit-xml`` and ``coder-eval report -f +junit`` both call :func:`generate_junit_xml`, so the two entry points can never +drift. + +SECURITY: this module only *builds* an element tree from JSON-derived data and +serializes it — it never *parses* XML, so stdlib ``xml.etree.ElementTree`` is +safe here (XXE / billion-laughs are parser-side attacks on untrusted XML input, +which has no surface). Every string entering the tree is scrubbed of characters +outside the XML 1.0 legal set via :func:`_xml_safe`. +""" + +from __future__ import annotations + +import json +import logging +import math +import re +import xml.etree.ElementTree as ET +from pathlib import Path, PurePosixPath +from typing import Any, Literal + +from .evaluation.judge_context import truncate +from .models import FinalStatus, RunSummary, SuiteRollup + + +logger = logging.getLogger(__name__) + +# Characters outside XML 1.0's legal set. Kept as a plain (non-raw) ASCII-only +# string with doubled backslashes so this source file carries no literal astral +# or control characters. ``ElementTree`` will happily serialize control chars +# into *invalid* XML, so every agent-derived string is scrubbed before it enters +# the tree. +_ILLEGAL_XML = re.compile("[^\\x09\\x0A\\x0D\\x20-\\uD7FF\\uE000-\\uFFFD\\U00010000-\\U0010FFFF]") + +# Per-testcase failure/error body cap (chars). Agent detail dumps can be huge. +_BODY_LIMIT = 10_000 + +# Serialized status values we recognize, for distinguishing a known status from a +# schema-skewed one when labelling a failure/error (classification itself goes +# through FinalStatus.category — see _category_of). +_KNOWN_STATUSES = frozenset(s.value for s in FinalStatus) + + +def _xml_safe(text: str) -> str: + """Strip characters outside XML 1.0's legal set (ET handles ``<>&`` itself).""" + return _ILLEGAL_XML.sub("", text) + + +def _category_of(status: str) -> Literal["succeeded", "failed", "error"]: + """Map a serialized status string to a reporting category via the SSOT. + + Goes through ``FinalStatus(value).category`` (an explicit allowlist, CE018); + an unknown status value (older/newer schema) falls to the error bucket + explicitly rather than via a denylist. + """ + try: + return FinalStatus(status).category + except ValueError: + return "error" + + +def _set_counts(elem: ET.Element, cases: list[ET.Element]) -> None: + """Set ``tests``/``failures``/``errors``/``skipped`` from the actual children. + + The invariant the whole writer is built around: count attributes always + equal the emitted child elements (no separately-tracked counters that can + drift). + """ + elem.set("tests", str(len(cases))) + elem.set("failures", str(sum(1 for c in cases if c.find("failure") is not None))) + elem.set("errors", str(sum(1 for c in cases if c.find("error") is not None))) + elem.set("skipped", str(sum(1 for c in cases if c.find("skipped") is not None))) + + +def _variant_of(row: dict[str, Any]) -> str: + """Variant bucket for a row — ``str``-coerced, ``None``/empty → ``"default"``. + + Rows are untyped dicts, so a non-string ``variant_id`` must not reach a dict + key or an XML attribute as a non-``str``. + """ + return str(row.get("variant_id") or "default") + + +def _status_of(row: dict[str, Any]) -> str: + """Row status as a string; an absent key reads as ``""``, not ``"None"``.""" + raw = row.get("status") + return str(raw) if raw is not None else "" + + +def _is_safe_component(value: str) -> bool: + """True when ``value`` is usable as a single, contained path segment. + + ``run.json`` rows are untyped and may be blob-pulled from elsewhere, so a + crafted ``variant_id``/``task_id`` must not steer the lookup outside the run + directory (an absolute value would discard ``run_dir`` entirely, and ``..`` + would walk up). + """ + return bool(value) and value not in {".", ".."} and "/" not in value and "\\" not in value + + +def _load_task_json(run_dir: Path, row: dict[str, Any], variant: str) -> dict[str, Any] | None: + """Best-effort load of a failed row's ``task.json`` as a plain dict. + + Plain-dict access (not ``EvaluationResult.model_validate``) keeps the writer + tolerant of schema skew in blob-pulled/older runs and avoids materializing + the large ``turns`` array. Any problem — unsafe path component, missing + dir/file, undecodable bytes, bad JSON, or an ambiguous replicate — yields + ``None`` so the caller falls back to a status-only body. + """ + task_id = str(row.get("task_id", "")) + if not _is_safe_component(variant) or not _is_safe_component(task_id): + return None + + replicate_index = row.get("replicate_index") + task_dir = run_dir / variant / task_id + if isinstance(replicate_index, int) and not isinstance(replicate_index, bool): + candidate = task_dir / f"{replicate_index:02d}" / "task.json" + else: + matches = sorted(task_dir.glob("*/task.json")) + # With no replicate index, picking one of several would misattribute + # another replicate's failure detail to this row — degrade instead. + if len(matches) > 1: + return None + candidate = matches[0] if matches else task_dir / "task.json" + + try: + # Belt-and-braces containment check (catches symlink escapes too). + if not candidate.resolve().is_relative_to(run_dir.resolve()): + return None + # ValueError covers both json.JSONDecodeError and UnicodeDecodeError + # (undecodable bytes) — neither may abort report generation. + return json.loads(candidate.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + + +def _criteria_body(row: dict[str, Any], run_dir: Path, variant: str) -> str: + """Build the failure/error body for a non-succeeded row. + + Prefers per-criterion lines from the row's ``task.json``; falls back to a + status + weighted-score line when task.json is missing/corrupt. Capped via + the shared ``truncate``. + """ + status = _status_of(row) + data = _load_task_json(run_dir, row, variant) + criteria = data.get("success_criteria_results") if isinstance(data, dict) else None + + lines: list[str] = [] + if isinstance(criteria, list) and criteria: + for crit in criteria: + if not isinstance(crit, dict): + continue + ctype = str(crit.get("criterion_type", "unknown")) + description = str(crit.get("description", "")) + score = crit.get("score") + threshold = crit.get("pass_threshold") + passed = isinstance(score, int | float) and isinstance(threshold, int | float) and score >= threshold + if passed: + lines.append(f"[PASS] {ctype}: {description}") + continue + score_str = f"{score:.2f}" if isinstance(score, int | float) else str(score) + thr_str = f"{threshold:.2f}" if isinstance(threshold, int | float) else str(threshold) + lines.append(f"[FAIL] {ctype}: score {score_str} < threshold {thr_str} — {description}") + detail = crit.get("details") or crit.get("error") + if detail: + lines.append(str(detail)) + else: + weighted = row.get("weighted_score") + weighted_str = f"{weighted:.2f}" if isinstance(weighted, int | float) else str(weighted) + lines.append(f"status={status} weighted_score={weighted_str}") + + return _xml_safe(truncate("\n".join(lines), _BODY_LIMIT)) + + +def _task_case(row: dict[str, Any], run_dir: Path) -> ET.Element: + """Build one ```` for a task row.""" + variant = _variant_of(row) + task_id = str(row.get("task_id", "")) + + # `bool` is an int subclass — exclude it so True never renders as "[01]". + replicate_index = row.get("replicate_index") + has_replicate = isinstance(replicate_index, int) and not isinstance(replicate_index, bool) + name = f"{task_id}[{replicate_index:02d}]" if has_replicate else task_id + + task_path = row.get("task_path") + classname = (Path(task_path).stem or variant) if isinstance(task_path, str) and task_path else variant + + # time must be a finite, non-negative float: NaN/inf would serialize as + # "nan"/"inf" and make the report invalid for JUnit ingesters. + duration = row.get("duration") + valid_duration = ( + isinstance(duration, int | float) + and not isinstance(duration, bool) + and math.isfinite(duration) + and duration >= 0 + ) + time_str = f"{duration:.3f}" if valid_duration else "0.000" + + case = ET.Element( + "testcase", + {"name": _xml_safe(name), "classname": _xml_safe(classname), "time": time_str}, + ) + + # Properties — emit only non-None values (no "None" strings in the tree). + prop_specs = [ + ("model_used", row.get("model_used")), + ("weighted_score", row.get("weighted_score")), + ("total_cost_usd", row.get("total_cost_usd")), + ("total_tokens", row.get("total_tokens")), + ("visible_turns", row.get("visible_turns")), + ] + props = [(k, v) for k, v in prop_specs if v is not None] + if props: + properties = ET.SubElement(case, "properties") + for key, value in props: + ET.SubElement(properties, "property", {"name": key, "value": _xml_safe(str(value))}) + + status = _status_of(row) + category = _category_of(status) + if category == "succeeded": + return case + + message = status if status in _KNOWN_STATUSES else f"unknown status: {status}" + tag = "failure" if category == "failed" else "error" + child = ET.SubElement(case, tag, {"message": _xml_safe(message)}) + child.text = _criteria_body(row, run_dir, variant) + return case + + +def _skipped_name(path: str) -> str: + """Stable testcase name for a skipped task: suffix-stripped, ``/``-separated. + + Uses the whole path rather than just the stem, because two skipped tasks + sharing a basename (``suiteA/task.yaml``, ``suiteB/task.yaml``) would + otherwise collapse into one identity that some JUnit ingesters merge. + + Separators are normalized to ``/`` and the path is parsed with + ``PurePosixPath`` so the emitted name does not depend on the OS that + generated the report — the same logical run must produce the same testcase + identity on Windows and Linux, or CI history/flake tracking splits in two. + """ + return str(PurePosixPath(path.replace("\\", "/")).with_suffix("")) + + +def _skipped_suite(summary: RunSummary) -> ET.Element | None: + """Build the synthetic ``skipped`` testsuite from ``RunSummary.skipped_tasks``.""" + if not summary.skipped_tasks: + return None + suite = ET.Element("testsuite", {"name": "skipped"}) + cases: list[ET.Element] = [] + for entry in summary.skipped_tasks: + case = ET.SubElement( + suite, + "testcase", + {"name": _xml_safe(_skipped_name(entry.path)), "classname": "skipped"}, + ) + ET.SubElement(case, "skipped", {"message": _xml_safe(entry.reason)}) + cases.append(case) + _set_counts(suite, cases) + return suite + + +def _suite_gate_suite(run_dir: Path) -> ET.Element | None: + """Build the synthetic ``suite-gates`` testsuite from ``*/*/suite.json``. + + A suite.json that fails ``SuiteRollup`` validation is skipped with a warning + (schema skew must not kill report generation). Returns ``None`` when no valid + rollup is found. + """ + rollups: list[SuiteRollup] = [] + for path in sorted(run_dir.glob("*/*/suite.json")): + try: + rollups.append(SuiteRollup.model_validate_json(path.read_text(encoding="utf-8"))) + except (OSError, ValueError) as e: + logger.warning("Skipping unparseable suite rollup %s: %s", path, e) + if not rollups: + return None + + suite = ET.Element("testsuite", {"name": "suite-gates"}) + cases: list[ET.Element] = [] + for rollup in rollups: + case = ET.SubElement( + suite, + "testcase", + {"name": _xml_safe(f"{rollup.variant_id}/{rollup.suite_id}"), "classname": "suite-gates"}, + ) + cases.append(case) + if rollup.passed: + continue + lines: list[str] = [] + for agg in rollup.criterion_aggregates: + for check in agg.threshold_checks: + if check.passed: + continue + if check.actual_value is None: + lines.append(f"{check.metric}: metric not emitted (min {check.min_value})") + else: + lines.append(f"{check.metric}: {check.actual_value} < {check.min_value}") + if agg.error: + lines.append(f"{agg.criterion_type}: {agg.error}") + failure = ET.SubElement(case, "failure", {"message": "suite thresholds not met"}) + failure.text = _xml_safe(truncate("\n".join(lines), _BODY_LIMIT)) + _set_counts(suite, cases) + return suite + + +def generate_junit_xml(run_dir: Path) -> str: + """Read ``run_dir`` and return a JUnit XML string. + + Args: + run_dir: A finalized run directory containing ``run.json``. + + Returns: + JUnit XML with an ```` declaration. + + Raises: + FileNotFoundError: When ``run.json`` is absent (not a finalized run dir). + pydantic.ValidationError: When ``run.json`` fails ``RunSummary`` validation + (the spine contract is broken; a fabricated report would be worse). + """ + run_json = run_dir / "run.json" + if not run_json.is_file(): + raise FileNotFoundError(f"No run.json in {run_dir} — not a finalized run directory (see 'coder-eval run')") + summary = RunSummary.model_validate_json(run_json.read_text(encoding="utf-8")) + + root = ET.Element("testsuites", {"name": _xml_safe(summary.run_id)}) + root.set("time", f"{summary.total_duration_seconds:.3f}") + + # Group task rows by variant, preserving first-seen order. + grouped: dict[str, list[dict[str, Any]]] = {} + for row in summary.task_results: + grouped.setdefault(_variant_of(row), []).append(row) + + all_cases: list[ET.Element] = [] + for variant, rows in grouped.items(): + suite = ET.SubElement(root, "testsuite", {"name": _xml_safe(variant)}) + cases = [_task_case(row, run_dir) for row in rows] + for case in cases: + suite.append(case) + _set_counts(suite, cases) + all_cases.extend(cases) + + skipped_suite = _skipped_suite(summary) + if skipped_suite is not None: + root.append(skipped_suite) + all_cases.extend(skipped_suite.findall("testcase")) + + gate_suite = _suite_gate_suite(run_dir) + if gate_suite is not None: + root.append(gate_suite) + all_cases.extend(gate_suite.findall("testcase")) + + _set_counts(root, all_cases) + + return ET.tostring(root, encoding="utf-8", xml_declaration=True).decode("utf-8") + + +def write_junit_xml(run_dir: Path, output_path: Path) -> Path: + """Generate JUnit XML for ``run_dir`` and write it to ``output_path``. + + Thin persist wrapper mirroring the ``build_run_summary``/``write_run_summary`` + seam. Creates parent directories as needed. + + Returns: + The path written. + """ + xml = generate_junit_xml(run_dir) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(xml, encoding="utf-8") + return output_path diff --git a/tests/conftest.py b/tests/conftest.py index 61f9b118..07cf3c7d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,10 @@ """Pytest configuration and shared fixtures.""" import logging +from collections.abc import Callable +from datetime import datetime +from pathlib import Path +from typing import Any import pytest @@ -54,3 +58,51 @@ def reset_logging_after_test(): # Reset to default level app_logger.setLevel(logging.NOTSET) + + +@pytest.fixture +def write_run_json() -> Callable[..., Path]: + """Factory that writes a valid ``run.json`` (a real ``RunSummary``) into a run dir. + + Shared by the JUnit writer tests (Phase 1) and the CLI report/run tests + (Phase 2) so both track the ``RunSummary`` model instead of hand-typed JSON. + Task-status counts (``tasks_succeeded``/``failed``/``error``) are computed + from each row's ``status`` via ``FinalStatus.category`` so the model's + task-count invariant always holds. Callers pass row dicts shaped like + ``eval_result_to_task_dict`` output; only ``task_id`` and ``status`` are + required per row. + """ + from coder_eval.models import FinalStatus, RunSummary, SkippedTask + + def _build( + run_dir: Path, + rows: list[dict[str, Any]], + *, + skipped: list[tuple[str, str]] | None = None, + run_id: str = "2026-07-21_12-00-00", + ) -> Path: + counts = {"succeeded": 0, "failed": 0, "error": 0} + for row in rows: + try: + category = FinalStatus(row.get("status")).category + except ValueError: + category = "error" # unknown status → error bucket (mirrors the writer) + counts[category] += 1 + summary = RunSummary( + run_id=run_id, + start_time=datetime(2026, 7, 21, 12, 0, 0), + end_time=datetime(2026, 7, 21, 12, 5, 0), + total_duration_seconds=300.0, + tasks_run=len(rows), + tasks_succeeded=counts["succeeded"], + tasks_failed=counts["failed"], + tasks_error=counts["error"], + skipped_tasks=[SkippedTask(path=p, reason=r) for p, r in (skipped or [])], + task_results=rows, + framework_version="test", + ) + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "run.json").write_text(summary.model_dump_json(indent=2), encoding="utf-8") + return run_dir / "run.json" + + return _build diff --git a/tests/test_report_command.py b/tests/test_report_command.py index 3c7a7fe9..060e7f12 100644 --- a/tests/test_report_command.py +++ b/tests/test_report_command.py @@ -144,3 +144,55 @@ def test_report_html_malformed_task_json_skipped_and_fails(tmp_path: Path) -> No assert "Skipping" in res.stdout assert "1 task(s) failed" in res.stdout assert (run_dir / "task-good" / "task.html").exists() + + +# -------------------------------------------------------------------------- +# JUnit path (-f junit) +# -------------------------------------------------------------------------- + + +def test_report_junit_default_output(write_run_json, tmp_path: Path) -> None: + from defusedxml.ElementTree import fromstring + + run_dir = tmp_path / "run" + write_run_json(run_dir, [{"task_id": "t", "status": "SUCCESS"}]) + + res = runner.invoke(app, ["report", str(run_dir), "-f", "junit"]) + assert res.exit_code == 0 + out = run_dir / "junit.xml" + assert out.is_file() + root = fromstring(out.read_text(encoding="utf-8")) + assert root.tag == "testsuites" + + +def test_report_junit_custom_output(write_run_json, tmp_path: Path) -> None: + from defusedxml.ElementTree import fromstring + + run_dir = tmp_path / "run" + write_run_json(run_dir, [{"task_id": "t", "status": "SUCCESS"}]) + out = tmp_path / "out" / "custom.xml" + + res = runner.invoke(app, ["report", str(run_dir), "-f", "junit", "-o", str(out)]) + assert res.exit_code == 0 + assert out.is_file() + fromstring(out.read_text(encoding="utf-8")) + assert not (run_dir / "junit.xml").exists() + + +def test_report_junit_missing_run_json_exits_1(tmp_path: Path) -> None: + run_dir = tmp_path / "run" + run_dir.mkdir() # no run.json + + res = runner.invoke(app, ["report", str(run_dir), "-f", "junit"]) + assert res.exit_code == 1 + assert "run.json" in res.stdout + + +def test_report_unknown_format_message_lists_junit(tmp_path: Path) -> None: + run_dir = tmp_path / "run" + run_dir.mkdir() + + res = runner.invoke(app, ["report", str(run_dir), "--format", "bogus"]) + assert res.exit_code == 1 + assert "unknown --format" in res.stdout + assert "junit" in res.stdout diff --git a/tests/test_reports_junit.py b/tests/test_reports_junit.py new file mode 100644 index 00000000..24949413 --- /dev/null +++ b/tests/test_reports_junit.py @@ -0,0 +1,595 @@ +"""Unit tests for the disk-driven JUnit XML writer (``reports_junit``). + +All tests are hermetic: the run directory is built by hand under ``tmp_path`` +(no agents, no API). Test-side XML parsing uses ``defusedxml`` as +defense-in-depth even though the parsed strings are self-generated. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest +from defusedxml.ElementTree import fromstring + +from coder_eval.models import SuiteRollup, ThresholdCheck +from coder_eval.reports_junit import generate_junit_xml, write_junit_xml + + +def _write_task_json( + run_dir: Path, + variant: str, + task_id: str, + replicate_index: int, + criteria: list[dict[str, Any]], +) -> None: + """Write a minimal task.json (plain dict) at the run-layout location.""" + task_dir = run_dir / variant / task_id / f"{replicate_index:02d}" + task_dir.mkdir(parents=True, exist_ok=True) + (task_dir / "task.json").write_text(json.dumps({"success_criteria_results": criteria}), encoding="utf-8") + + +def _row( + task_id: str, + status: str, + *, + variant_id: str | None = "default", + replicate_index: int | None = 0, + duration: float | None = 1.5, + task_path: str | None = "tasks/sample.yaml", + weighted_score: float | None = 0.5, + model_used: str | None = "claude-haiku-4-5-20251001", + total_tokens: int | None = 1234, + total_cost_usd: float | None = 0.01, + visible_turns: int | None = 3, +) -> dict[str, Any]: + return { + "task_id": task_id, + "status": status, + "variant_id": variant_id, + "replicate_index": replicate_index, + "duration": duration, + "task_path": task_path, + "weighted_score": weighted_score, + "model_used": model_used, + "total_tokens": total_tokens, + "total_cost_usd": total_cost_usd, + "visible_turns": visible_turns, + } + + +def _find_testsuite(root: Any, name: str) -> Any: + for ts in root.findall("testsuite"): + if ts.get("name") == name: + return ts + return None + + +def test_happy_path_grouping_and_counts(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [ + _row("t_pass", "SUCCESS", variant_id="v1"), + _row("t_fail", "FAILURE", variant_id="v1"), + _row("t_err", "ERROR", variant_id="v1"), + _row("t_pass", "SUCCESS", variant_id="v2"), + _row("t_fail", "FAILURE", variant_id="v2"), + _row("t_err", "ERROR", variant_id="v2"), + ] + write_run_json(run_dir, rows) + + root = fromstring(generate_junit_xml(run_dir)) + assert root.tag == "testsuites" + # Root counts match the emitted children. + assert int(root.get("tests")) == 6 + assert int(root.get("failures")) == 2 + assert int(root.get("errors")) == 2 + assert int(root.get("skipped")) == 0 + + for variant in ("v1", "v2"): + ts = _find_testsuite(root, variant) + assert ts is not None + cases = ts.findall("testcase") + assert int(ts.get("tests")) == len(cases) == 3 + assert int(ts.get("failures")) == 1 + assert int(ts.get("errors")) == 1 + # classname derives from task_path stem. + assert all(c.get("classname") == "sample" for c in cases) + # time is a float string. + assert all(float(c.get("time")) == 1.5 for c in cases) + + +def test_root_counts_equal_summed_children(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("a", "SUCCESS"), _row("b", "FAILURE"), _row("c", "MAX_TURNS_EXHAUSTED")] + write_run_json(run_dir, rows) + root = fromstring(generate_junit_xml(run_dir)) + + total_cases = 0 + total_failures = 0 + total_errors = 0 + for ts in root.findall("testsuite"): + cases = ts.findall("testcase") + total_cases += len(cases) + total_failures += sum(1 for c in cases if c.find("failure") is not None) + total_errors += sum(1 for c in cases if c.find("error") is not None) + assert int(root.get("tests")) == total_cases + assert int(root.get("failures")) == total_failures + assert int(root.get("errors")) == total_errors + # MAX_TURNS_EXHAUSTED is category "failed". + assert total_failures == 2 + assert total_errors == 0 + + +def test_failure_body_from_task_json(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t_fail", "FAILURE", variant_id="v1", replicate_index=0)] + write_run_json(run_dir, rows) + _write_task_json( + run_dir, + "v1", + "t_fail", + 0, + [ + { + "criterion_type": "file_exists", + "description": "output.txt must exist", + "score": 0.0, + "pass_threshold": 0.9, + "details": "file not found on disk", + "error": None, + }, + { + "criterion_type": "file_contains", + "description": "greeting present", + "score": 1.0, + "pass_threshold": 0.9, + "details": None, + "error": None, + }, + ], + ) + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "v1") + failure = ts.find("testcase").find("failure") + body = failure.text or "" + assert "file_exists" in body + assert "0.00" in body and "0.90" in body # score-vs-threshold line + assert "file not found on disk" in body + assert "output.txt must exist" in body + # passed criterion appears as a one-liner. + assert "file_contains" in body + + +def test_fallback_body_without_task_json(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t_fail", "FAILURE", variant_id="v1", weighted_score=0.42)] + write_run_json(run_dir, rows) + # No task.json written. + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "v1") + body = ts.find("testcase").find("failure").text or "" + assert "FAILURE" in body + assert "0.42" in body + + +def test_replicates_naming_and_per_replicate_task_json(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [ + _row("task", "FAILURE", variant_id="v1", replicate_index=0), + _row("task", "FAILURE", variant_id="v1", replicate_index=1), + ] + write_run_json(run_dir, rows) + _write_task_json( + run_dir, + "v1", + "task", + 0, + [ + { + "criterion_type": "c", + "description": "rep0", + "score": 0.0, + "pass_threshold": 0.9, + "details": "REP0DETAIL", + "error": None, + } + ], + ) + _write_task_json( + run_dir, + "v1", + "task", + 1, + [ + { + "criterion_type": "c", + "description": "rep1", + "score": 0.0, + "pass_threshold": 0.9, + "details": "REP1DETAIL", + "error": None, + } + ], + ) + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "v1") + names = {c.get("name") for c in ts.findall("testcase")} + assert names == {"task[00]", "task[01]"} + bodies = {c.get("name"): (c.find("failure").text or "") for c in ts.findall("testcase")} + assert "REP0DETAIL" in bodies["task[00]"] + assert "REP1DETAIL" in bodies["task[01]"] + + +def test_replicate_index_none_globs_task_json(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("task", "FAILURE", variant_id="v1", replicate_index=None)] + write_run_json(run_dir, rows) + _write_task_json( + run_dir, + "v1", + "task", + 0, + [ + { + "criterion_type": "c", + "description": "d", + "score": 0.0, + "pass_threshold": 0.9, + "details": "GLOBBED", + "error": None, + } + ], + ) + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "v1") + case = ts.find("testcase") + assert case.get("name") == "task" # no [NN] suffix + assert "GLOBBED" in (case.find("failure").text or "") + + +def test_variant_none_grouped_as_default(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t", "SUCCESS", variant_id=None, task_path=None)] + write_run_json(run_dir, rows) + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "default") + assert ts is not None + # task_path None → classname falls back to variant. + assert ts.find("testcase").get("classname") == "default" + + +def test_duration_none_renders_zero(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t", "SUCCESS", duration=None)] + write_run_json(run_dir, rows) + root = fromstring(generate_junit_xml(run_dir)) + case = root.find("testsuite").find("testcase") + assert float(case.get("time")) == 0.0 + + +def test_properties_omit_none_values(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [ + _row("t", "SUCCESS", total_cost_usd=None, total_tokens=None, weighted_score=None), + ] + write_run_json(run_dir, rows) + xml = generate_junit_xml(run_dir) + root = fromstring(xml) + props = root.find("testsuite").find("testcase").find("properties") + names = {p.get("name") for p in props.findall("property")} if props is not None else set() + assert "total_cost_usd" not in names + assert "total_tokens" not in names + assert "weighted_score" not in names + assert "model_used" in names + # No literal "None" strings leaked into the tree. + assert "None" not in xml + + +def test_skipped_tasks_suite(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t", "SUCCESS")] + write_run_json( + run_dir, + rows, + skipped=[("tasks/broken.yaml", "ValueError: bad schema"), ("tasks/opt.yaml", "skip: true")], + ) + root = fromstring(generate_junit_xml(run_dir)) + assert int(root.get("skipped")) == 2 + ts = _find_testsuite(root, "skipped") + assert ts is not None + skipped_cases = ts.findall("testcase") + assert len(skipped_cases) == 2 + # Suffix-stripped full path (not just the stem) so same-basename tasks stay + # distinct. Always '/'-separated: the same logical run must yield the same + # testcase identity on Windows and Linux, or CI history splits in two. + names = {c.get("name") for c in skipped_cases} + assert names == {"tasks/broken", "tasks/opt"} + assert not any("\\" in n for n in names) + assert all(c.find("skipped") is not None for c in skipped_cases) + + +def test_suite_gates_failing_and_none_metric(tmp_path: Path, write_run_json: Callable[..., Path]) -> None: + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("t", "SUCCESS", variant_id="v1")]) + rollup = SuiteRollup( + suite_id="s1", + variant_id="v1", + rows_total=10, + rows_passed=5, + rows_failed=5, + rows_error=0, + pass_rate=0.5, + passed=False, + ) + # Inject two failed threshold checks via a criterion aggregate. + from coder_eval.models import CriterionAggregate + + rollup.criterion_aggregates = [ + CriterionAggregate( + criterion_type="classification_match", + passed=False, + threshold_checks=[ + ThresholdCheck(metric="accuracy", min_value=0.9, actual_value=0.5, passed=False), + ThresholdCheck(metric="f1.macro", min_value=0.8, actual_value=None, passed=False), + ], + ), + # A criterion that declared suite_thresholds but aggregate() returned nothing. + CriterionAggregate( + criterion_type="skill_triggered", + passed=False, + error="aggregate() returned no metrics", + ), + ] + suite_dir = run_dir / "v1" / "s1" + suite_dir.mkdir(parents=True, exist_ok=True) + (suite_dir / "suite.json").write_text(rollup.model_dump_json(indent=2), encoding="utf-8") + + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "suite-gates") + assert ts is not None + case = ts.find("testcase") + assert case.get("name") == "v1/s1" + body = case.find("failure").text or "" + assert "accuracy" in body and "0.5" in body and "0.9" in body + assert "f1.macro" in body + assert "metric not emitted" in body + # CriterionAggregate.error is surfaced in the gate body. + assert "skill_triggered" in body and "aggregate() returned no metrics" in body + + +def test_suite_gates_passing_is_plain_case(tmp_path: Path, write_run_json: Callable[..., Path]) -> None: + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("t", "SUCCESS", variant_id="v1")]) + rollup = SuiteRollup( + suite_id="s1", + variant_id="v1", + rows_total=2, + rows_passed=2, + rows_failed=0, + rows_error=0, + pass_rate=1.0, + passed=True, + ) + suite_dir = run_dir / "v1" / "s1" + suite_dir.mkdir(parents=True, exist_ok=True) + (suite_dir / "suite.json").write_text(rollup.model_dump_json(indent=2), encoding="utf-8") + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "suite-gates") + case = ts.find("testcase") + assert case.find("failure") is None + + +def test_corrupt_suite_json_skipped_with_report_still_produced( + tmp_path: Path, write_run_json: Callable[..., Path], caplog: pytest.LogCaptureFixture +) -> None: + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("t", "SUCCESS", variant_id="v1")]) + suite_dir = run_dir / "v1" / "s1" + suite_dir.mkdir(parents=True, exist_ok=True) + (suite_dir / "suite.json").write_text("{not valid json", encoding="utf-8") + # Report still generated; no suite-gates suite (the only rollup was corrupt). + root = fromstring(generate_junit_xml(run_dir)) + assert _find_testsuite(root, "suite-gates") is None + + +def test_xml_safety_control_chars_and_markup(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t_fail", "FAILURE", variant_id="v1", replicate_index=0)] + write_run_json(run_dir, rows) + nasty = "\x1b[31mboom\x00 & 'quote'" + _write_task_json( + run_dir, + "v1", + "t_fail", + 0, + [ + { + "criterion_type": "c", + "description": "d", + "score": 0.0, + "pass_threshold": 0.9, + "details": nasty, + "error": None, + } + ], + ) + xml = generate_junit_xml(run_dir) + root = fromstring(xml) # must re-parse + body = _find_testsuite(root, "v1").find("testcase").find("failure").text or "" + assert "boom" in body + assert "\x00" not in body # scrubbed + assert "\x1b" not in body + + +def test_unknown_status_lands_in_errors(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t", "SOME_FUTURE_STATUS", variant_id="v1")] + write_run_json(run_dir, rows) + root = fromstring(generate_junit_xml(run_dir)) + assert int(root.get("errors")) == 1 + err = _find_testsuite(root, "v1").find("testcase").find("error") + assert err is not None + assert "unknown status" in (err.get("message") or "") + + +def test_empty_run_all_skipped(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + write_run_json(run_dir, [], skipped=[("tasks/a.yaml", "skip: true")]) + root = fromstring(generate_junit_xml(run_dir)) + assert int(root.get("tests")) == 1 # only the skipped case + assert int(root.get("skipped")) == 1 + assert int(root.get("failures")) == 0 + assert int(root.get("errors")) == 0 + + +def test_missing_run_json_raises(tmp_path: Path) -> None: + run_dir = tmp_path / "empty" + run_dir.mkdir() + with pytest.raises(FileNotFoundError, match=r"run\.json"): + generate_junit_xml(run_dir) + + +def test_write_junit_xml_creates_parents_and_returns_path(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("t", "SUCCESS")]) + out = tmp_path / "nested" / "deeper" / "junit.xml" + written = write_junit_xml(run_dir, out) + assert written == out + assert out.is_file() + fromstring(out.read_text(encoding="utf-8")) # parses + + +# -------------------------------------------------------------------------- +# Robustness against schema-skewed / crafted run.json rows (final-review findings) +# -------------------------------------------------------------------------- + + +def test_malformed_row_types_degrade_gracefully(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """Loose `task_results` dicts are untyped; a skewed row must not abort the report. + + Covers: bool replicate_index (bool is an int subclass), non-finite duration + (would emit an invalid time="nan"), non-str variant_id / task_path, and a + row with no `status` key at all. + """ + run_dir = tmp_path / "run" + rows: list[dict[str, Any]] = [ + {"task_id": "t_bool", "status": "SUCCESS", "replicate_index": True, "duration": 1.0}, + {"task_id": "t_nan", "status": "SUCCESS", "duration": float("nan")}, + {"task_id": "t_inf", "status": "SUCCESS", "duration": float("inf")}, + {"task_id": "t_neg", "status": "SUCCESS", "duration": -5.0}, + {"task_id": "t_variant", "status": "SUCCESS", "variant_id": 17}, + {"task_id": "t_path", "status": "SUCCESS", "task_path": 42}, + {"task_id": "t_nostatus"}, + ] + write_run_json(run_dir, rows) + + xml = generate_junit_xml(run_dir) + root = fromstring(xml) # must still be well-formed + + cases = [c for ts in root.findall("testsuite") for c in ts.findall("testcase")] + assert len(cases) == len(rows) + # Every time attribute must be a finite, non-negative float (JUnit requirement). + for case in cases: + t = float(case.get("time")) + assert t == t and t >= 0.0 # not NaN, not negative + # A bool replicate_index must NOT be formatted as a [NN] replicate suffix. + names = {c.get("name") for c in cases} + assert "t_bool" in names + # The missing-status row lands in the error bucket, not silently as a pass. + assert int(root.get("errors")) == 1 + + +def test_task_json_invalid_utf8_falls_back(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """A task.json with undecodable bytes must degrade, not raise (UnicodeDecodeError + is a ValueError but NOT a json.JSONDecodeError).""" + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("t_fail", "FAILURE", variant_id="v1", replicate_index=0)]) + task_dir = run_dir / "v1" / "t_fail" / "00" + task_dir.mkdir(parents=True, exist_ok=True) + (task_dir / "task.json").write_bytes(b"\xff\xfe\x00 not utf-8") + + root = fromstring(generate_junit_xml(run_dir)) + body = _find_testsuite(root, "v1").find("testcase").find("failure").text or "" + assert "FAILURE" in body # status-only fallback body + + +def test_task_json_lookup_cannot_escape_run_dir(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """A crafted variant_id/task_id must not make the writer read outside run_dir.""" + outside = tmp_path / "outside" + outside.mkdir() + (outside / "task.json").write_text( + json.dumps( + { + "success_criteria_results": [ + { + "criterion_type": "leaked", + "description": "SECRET", + "score": 0.0, + "pass_threshold": 0.9, + "details": "LEAKED_SECRET_DETAIL", + "error": None, + } + ] + } + ), + encoding="utf-8", + ) + run_dir = tmp_path / "run" + write_run_json(run_dir, [{"task_id": "..", "status": "FAILURE", "variant_id": "../outside"}]) + + root = fromstring(generate_junit_xml(run_dir)) + xml_text = generate_junit_xml(run_dir) + assert "LEAKED_SECRET_DETAIL" not in xml_text + body = root.find("testsuite").find("testcase").find("failure").text or "" + assert "FAILURE" in body # fell back, did not read the outside file + + +def test_ambiguous_replicate_does_not_misattribute(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """replicate_index=None with MULTIPLE replicate dirs must fall back rather than + attribute replicate 00's failure detail to the row.""" + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("task", "FAILURE", variant_id="v1", replicate_index=None)]) + for idx, marker in ((0, "REP0ONLY"), (1, "REP1ONLY")): + _write_task_json( + run_dir, + "v1", + "task", + idx, + [ + { + "criterion_type": "c", + "description": "d", + "score": 0.0, + "pass_threshold": 0.9, + "details": marker, + "error": None, + } + ], + ) + root = fromstring(generate_junit_xml(run_dir)) + body = _find_testsuite(root, "v1").find("testcase").find("failure").text or "" + assert "REP0ONLY" not in body and "REP1ONLY" not in body + assert "FAILURE" in body + + +def test_skipped_names_unique_for_same_stem(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """Two skipped tasks sharing a basename must get distinct testcase identities.""" + run_dir = tmp_path / "run" + write_run_json(run_dir, [], skipped=[("suiteA/task.yaml", "skip: true"), ("suiteB/task.yaml", "skip: true")]) + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "skipped") + names = {c.get("name") for c in ts.findall("testcase")} + assert len(names) == 2, f"skipped testcase names collided: {names}" + + +def test_skipped_name_is_platform_independent(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """A Windows-style path recorded in run.json must still emit a '/'-separated + name, so a report generated on Windows matches one generated on Linux.""" + run_dir = tmp_path / "run" + write_run_json(run_dir, [], skipped=[(r"tasks\win\task.yaml", "skip: true")]) + root = fromstring(generate_junit_xml(run_dir)) + name = _find_testsuite(root, "skipped").find("testcase").get("name") + assert name == "tasks/win/task" diff --git a/tests/test_run_command_junit.py b/tests/test_run_command_junit.py new file mode 100644 index 00000000..b5ac1de0 --- /dev/null +++ b/tests/test_run_command_junit.py @@ -0,0 +1,77 @@ +"""`coder-eval run --junit-xml` wiring at the ``_run_all_tasks`` seam. + +Fully hermetic: every side-effecting collaborator is patched (see the pattern in +``test_cli_telemetry.py``). The ``_run_with_experiment`` mock writes a minimal +``run.json`` to disk, which is exactly what the JUnit writer reads. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from unittest.mock import AsyncMock, Mock, patch + +import pytest +import typer +from defusedxml.ElementTree import fromstring + +from coder_eval.cli.run_command import _run_all_tasks + + +async def _invoke( + run_dir: Path, + junit_xml: Path | None, + write_run_json: Callable[..., Path], + *, + status: str, + failed: bool, +) -> None: + summary = Mock(tasks_failed=1 if failed else 0, tasks_error=0) + + async def _fake(*_args, **_kwargs): + # Mirror production: run.json is persisted inside _run_with_experiment. + write_run_json(run_dir, [{"task_id": "t", "status": status}]) + return summary, 0 + + with ( + patch("coder_eval.cli.run_command.prepare_run_directory", return_value=run_dir), + patch("coder_eval.cli.run_command.expand_task_files", return_value=[Path("a.yaml")]), + patch("coder_eval.cli.run_command._run_with_experiment", new=AsyncMock(side_effect=_fake)), + patch("coder_eval.logging_config.aggregate_task_logs"), + patch("coder_eval.cli.run_command.print_execution_summary"), + patch("coder_eval.telemetry.track_event"), + patch("coder_eval.telemetry.flush_telemetry"), + ): + await _run_all_tasks( + task_files=[Path("a.yaml")], + preservation_mode=None, + run_dir=run_dir, + max_parallel=1, + junit_xml=junit_xml, + ) + + +async def test_junit_written_on_success(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + junit = tmp_path / "j.xml" + await _invoke(run_dir, junit, write_run_json, status="SUCCESS", failed=False) + assert junit.is_file() + fromstring(junit.read_text(encoding="utf-8")) + + +async def test_junit_written_even_when_run_fails(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + # Ordering guarantee: the report is written BEFORE the failure exit-code gate. + run_dir = tmp_path / "run" + junit = tmp_path / "j.xml" + with pytest.raises(typer.Exit) as exc: + await _invoke(run_dir, junit, write_run_json, status="FAILURE", failed=True) + assert exc.value.exit_code == 1 + assert junit.is_file() + fromstring(junit.read_text(encoding="utf-8")) + + +async def test_no_junit_when_flag_absent(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + junit = tmp_path / "j.xml" + await _invoke(run_dir, None, write_run_json, status="SUCCESS", failed=False) + assert not junit.exists() diff --git a/uv.lock b/uv.lock index 6aaba0a0..214fc35f 100644 --- a/uv.lock +++ b/uv.lock @@ -480,6 +480,7 @@ codex = [ ] dev = [ { name = "bandit" }, + { name = "defusedxml" }, { name = "mcp" }, { name = "pip-audit" }, { name = "pre-commit" }, @@ -503,6 +504,7 @@ requires-dist = [ { name = "bandit", extras = ["toml"], marker = "extra == 'dev'", specifier = ">=1.9.4" }, { name = "claude-agent-sdk", specifier = ">=0.2.124" }, { name = "click", specifier = ">=8.3.3" }, + { name = "defusedxml", marker = "extra == 'dev'", specifier = ">=0.7.1" }, { name = "google-antigravity", marker = "extra == 'antigravity'", specifier = "==0.1.7" }, { name = "jmespath", specifier = ">=1.1.0" }, { name = "jsonschema", specifier = ">=4.26.0" },