UiPath · uipreliga · Jul 22, 2026 · Jul 22, 2026 · Jul 22, 2026 · Jul 22, 2026
@@ -831,3 +831,53 @@ jobs:
           name: byoa-live-output
           path: tmp/
           retention-days: 7
+
+  action-dogfood:
+    name: Action Dogfood (composite action, real API)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    # Skip on fork PRs where secrets aren't available (matches e2e-smoke).
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+
+      # The composite action is agent-agnostic and does NOT install a coding-agent
+      # runtime. The dogfood task uses the default claude-code agent, so provide
+      # Node + the Claude CLI here (as e2e-smoke does), before invoking the action.
+      - name: Set up Node.js 20
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020  # v4.4.0
+        with:
+          node-version: "20"
+      - name: Install Claude CLI
+        run: npm install -g @anthropic-ai/claude-code
+
+      - name: Run coder-eval via local action
+        id: dogfood
+        uses: ./
+        with:
+          version: local
+          tasks: tasks/hello_date.yaml
+          model: claude-haiku-4-5-20251001
+          run-dir: runs/ci-action-dogfood
+          junit-path: runs/ci-action-dogfood/junit.xml
+          anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
+
+      - name: Verify outputs and JUnit file
+        env:
+          JUNIT: ${{ steps.dogfood.outputs.junit-path }}
+          RUNDIR: ${{ steps.dogfood.outputs.run-dir }}
+        run: |
+          set -euo pipefail
+          test -n "$JUNIT" && test -f "$JUNIT" || { echo "junit output missing"; exit 1; }
+          # Well-formedness check on a file this job just generated (trusted input;
+          # our writer emits no DTDs/entities) — stdlib ET is fine here.
+          python3 -c "import sys, xml.etree.ElementTree as ET; ET.parse(sys.argv[1])" "$JUNIT"
+          test -f "$RUNDIR/run.json" || { echo "run.json missing"; exit 1; }
+
+      - name: Upload dogfood run on failure
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: action-dogfood-runs
+          path: runs/ci-action-dogfood/
+          retention-days: 7
@@ -182,23 +182,52 @@ jobs:
           echo "version=$V" >> "$GITHUB_OUTPUT"
           echo "Publishing version: $V"
 
-      - name: Regenerate uv.lock and amend release commit
+      - name: Regenerate uv.lock, bump action.yml pin, and amend release commit
         if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != ''
+        env:
+          # Passed via env (not interpolated into the script) per GitHub's
+          # injection guidance.
+          VERSION: ${{ steps.release.outputs.version }}
         run: |
+          set -euo pipefail
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git config user.name "github-actions[bot]"
+          # Bump the composite action's default `version:` pin to the just-released
+          # version so `UiPath/coder_eval@vX.Y.Z` installs `coder-eval==X.Y.Z`. The
+          # anchor is indentation-tolerant and keyed on the unique trailing
+          # "# <-- kept in sync" comment; the grep guard fails the release loudly
+          # if a reformat ever detaches it (rather than shipping a stale pin).
+          sed -i -E 's/^([[:space:]]*default: ")[0-9]+\.[0-9]+\.[0-9]+("   # <-- kept in sync)/\1'"${VERSION}"'\2/' action.yml
+          grep -q "default: \"${VERSION}\"" action.yml || { echo "action.yml version bump failed"; exit 1; }
+          git add action.yml
+          # Regenerate the lock too; stage it (a no-op if unchanged).
           uv lock
-          if ! git diff --quiet uv.lock; then
-            git config user.email "github-actions[bot]@users.noreply.github.com"
-            git config user.name "github-actions[bot]"
-            git add uv.lock
+          git add uv.lock
+          # Amend only if action.yml/uv.lock actually changed the tree.
+          if ! git diff --cached --quiet; then
             git commit --amend --no-edit
             # Amend replaced the commit the tag points at; re-point it before pushing.
-            git tag -f "v${{ steps.release.outputs.version }}"
+            git tag -f "v${VERSION}"
           fi
 
       - name: Push release commit and tags
         if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != ''
         run: git push origin main "v${{ steps.release.outputs.version }}"
 
+      - name: Move major action tag (vN -> this release)
+        # Real releases only: a prerelease never tags or pushes to main, so it must
+        # not move the tag consumers pin (mirrors `:latest` staying put).
+        if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != ''
+        env:
+          VERSION: ${{ steps.release.outputs.version }}
+        run: |
+          set -euo pipefail
+          # Consumers pin `UiPath/coder_eval@v0` (becomes `@v1` at 1.0.0). Force-move
+          # the moving major tag to this release. Force on a missing tag creates it.
+          MAJOR="v${VERSION%%.*}"
+          git tag -f "$MAJOR" "v${VERSION}"
+          git push -f origin "$MAJOR"
+
       - name: Build wheel + sdist
         if: steps.ver.outputs.version != ''
         run: uv build

@@ -20,6 +20,7 @@ coder_eval/
 ├── orchestrator.py                # Main evaluation loop
 ├── reports.py                     # Markdown/JSON report generation (run-level + per-suite rollup via write_suite_rollups)
 ├── reports_experiment.py          # Experiment/cross-variant report generation
+├── reports_junit.py               # JUnit XML report from a finalized run dir (run.json spine; for CI test-report ingestion)
 ├── analysis.py                    # Command statistics aggregation
 ├── logging_config.py              # Structured logging setup
 ├── path_utils.py                  # Run ID generation, path utilities
@@ -120,6 +121,7 @@ tasks/                             # Task definition YAML files
 tests/                             # Test suite
 docs/                              # Documentation
 templates/                         # Sandbox template directories
+action.yml                         # Published composite GitHub Action (coder-eval as a CI gate). release.yml maintains its `version:` default + the moving `v<major>` tag.
 ```
 
 ## Key Architectural Patterns

@@ -95,6 +95,53 @@ live in this repo — clone it or point the CLI at your own task files.) See
 [Tutorial 02 — Running coder_eval in CI](docs/tutorials/02-ci-pipeline.md) for
 the full setup.
 
+## Use as a GitHub Action
+
+A composite action at the repo root runs `coder-eval` as a CI gate — it installs
+the pinned CLI, runs your tasks, writes a JUnit XML report, appends `run.md` to
+the job summary, and fails the step on any task/gate failure:
+
+```yaml
+- uses: UiPath/coder_eval@v0    # becomes @v1 once 1.0.0 ships; @vX.Y.Z pins exactly
+  with:
+    tasks: tests/tasks/**/*.yaml
+    model: claude-sonnet-5
+    anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
+```
+
+| Input | Default | Purpose |
+| --- | --- | --- |
+| `tasks` | *(all `tasks/`)* | Task YAML path(s)/glob |
+| `tags` | — | `--tags` filter |
+| `model` | — | `--model` override |
+| `extra-args` | — | Verbatim extra args (`--experiment`, `-D …`, …) |
+| `version` | pinned release | PyPI version, or `local` to install from the checkout |
+| `run-dir` | `runs/ci` | Run directory |
+| `junit-path` | `coder-eval-junit.xml` | Where to write the JUnit report |
+| `step-summary` | `true` | Append `run.md` to the job summary |
+| `anthropic-api-key` | — | Exported as `ANTHROPIC_API_KEY` for the run |
+
+Outputs: `run-dir` and `junit-path`. Feed the JUnit file to your platform's
+test-report renderer — e.g. on GitHub Actions with
+[`mikepenz/action-junit-report`](https://github.com/mikepenz/action-junit-report):
+
+```yaml
+- uses: mikepenz/action-junit-report@v5
+  if: always()
+  with:
+    report_paths: coder-eval-junit.xml
+```
+
+> **Agent runtime is the caller's responsibility.** The action is agent-agnostic —
+> it installs `coder-eval` but no coding-agent runtime. Tasks using the default
+> `claude-code` agent need the `claude` CLI on `PATH` (`actions/setup-node` +
+> `npm install -g @anthropic-ai/claude-code`) in the job before the action runs.
+
+> **Security.** Evaluated tasks execute agent-generated code. Do **not** run this
+> action under `pull_request_target` with secrets exposed to untrusted fork PRs —
+> use `pull_request` and gate on the same-repo condition, as this repo's own
+> dogfood job does.
+
 ## Telemetry
 
 > 📊 **Usage telemetry is on by default.** `coder-eval` sends **anonymous** usage

@@ -0,0 +1,110 @@
+name: coder-eval
+description: Run coder-eval evaluation tasks as a CI gate, with JUnit XML output and a job-summary report.
+branding:
+  icon: check-circle
+  color: orange
+
+# This action installs and runs the `coder-eval` CLI. It is agent-agnostic: it
+# does NOT install any coding-agent runtime. Tasks that use the default
+# `claude-code` agent need the `claude` CLI on PATH (Node + the
+# `@anthropic-ai/claude-code` npm package) provided by the calling job before
+# this action runs. See the README "Use as a GitHub Action" section.
+#
+# SECURITY: evaluated tasks execute agent-generated code. Do NOT run this action
+# under `pull_request_target` with secrets exposed to untrusted fork PRs.
+
+inputs:
+  tasks:
+    description: Task YAML path(s)/glob passed to `coder-eval run` (empty = all tasks/ recursively)
+    required: false
+    default: ""
+  tags:
+    description: Only run tasks matching any of these comma-separated tags (--tags)
+    required: false
+    default: ""
+  model:
+    description: Override agent model for all tasks (--model)
+    required: false
+    default: ""
+  extra-args:
+    description: Extra arguments appended verbatim to `coder-eval run` (trusted caller input; covers --experiment, -D overrides, --tags exclusions, etc.)
+    required: false
+    default: ""
+  version:
+    description: coder-eval version to install from PyPI, or "local" to install from the action checkout
+    required: false
+    default: "0.8.7"   # <-- kept in sync with releases by release.yml
+  run-dir:
+    description: Run directory (--run-dir)
+    required: false
+    default: "runs/ci"
+  junit-path:
+    description: Where to write the JUnit XML report
+    required: false
+    default: "coder-eval-junit.xml"
+  step-summary:
+    description: Append run.md to the GitHub job summary ("true"/"false")
+    required: false
+    default: "true"
+  anthropic-api-key:
+    description: Anthropic API key (exported as ANTHROPIC_API_KEY for the run step)
+    required: false
+    default: ""
+
+outputs:
+  run-dir:
+    description: The run directory containing run.json/run.md
+    value: ${{ steps.run.outputs.run-dir }}
+  junit-path:
+    description: Path to the written JUnit XML report
+    value: ${{ steps.run.outputs.junit-path }}
+
+runs:
+  using: composite
+  steps:
+    - name: Install uv
+      uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a  # v4.2.0
+    - name: Install coder-eval
+      shell: bash
+      env:
+        CE_VERSION: ${{ inputs.version }}
+        CE_ACTION_PATH: ${{ github.action_path }}
+      run: |
+        set -euo pipefail
+        if [ "$CE_VERSION" = "local" ]; then
+          uv tool install "$CE_ACTION_PATH"
+        else
+          uv tool install "coder-eval==$CE_VERSION"
+        fi
+    - name: Run coder-eval
+      id: run
+      shell: bash
+      env:
+        ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }}
+        CE_TASKS: ${{ inputs.tasks }}
+        CE_TAGS: ${{ inputs.tags }}
+        CE_MODEL: ${{ inputs.model }}
+        CE_EXTRA_ARGS: ${{ inputs.extra-args }}
+        CE_RUN_DIR: ${{ inputs.run-dir }}
+        CE_JUNIT: ${{ inputs.junit-path }}
+        CE_SUMMARY: ${{ inputs.step-summary }}
+      run: |
+        set -uo pipefail
+        args=(run --run-dir "$CE_RUN_DIR" --junit-xml "$CE_JUNIT")
+        [ -n "$CE_TAGS" ] && args+=(--tags "$CE_TAGS")
+        [ -n "$CE_MODEL" ] && args+=(--model "$CE_MODEL")
+        # extra-args is a trusted caller input, split on whitespace intentionally
+        # shellcheck disable=SC2206
+        [ -n "$CE_EXTRA_ARGS" ] && args+=($CE_EXTRA_ARGS)
+        # shellcheck disable=SC2206
+        [ -n "$CE_TASKS" ] && args+=($CE_TASKS)
+        set +e
+        coder-eval "${args[@]}"
+        CODE=$?
+        set -e
+        echo "run-dir=$CE_RUN_DIR" >> "$GITHUB_OUTPUT"
+        echo "junit-path=$CE_JUNIT" >> "$GITHUB_OUTPUT"
+        if [ "$CE_SUMMARY" = "true" ] && [ -f "$CE_RUN_DIR/run.md" ]; then
+          head -c 1000000 "$CE_RUN_DIR/run.md" >> "$GITHUB_STEP_SUMMARY"
+        fi
+        exit $CODE
@@ -158,6 +158,77 @@ jobs:
 - **`if: always()`** on the verdict and upload steps means you still get results
   even when a task fails.
 
+## Shortcut: the packaged action
+
+The five steps above spell out the mechanics, but coder_eval also ships a
+composite action at the repo root that bundles install + run + JUnit report +
+job-summary + fail-on-failure into one step:
+
+```yaml
+      - uses: actions/setup-node@v4      # the claude-code agent needs the Claude CLI…
+        with: { node-version: '20' }
+      - run: npm install -g @anthropic-ai/claude-code
+
+      - uses: UiPath/coder_eval@v0       # …then run the gate (pin @vX.Y.Z in production)
+        with:
+          tasks: tests/tasks/**/*.yaml
+          model: claude-sonnet-5
+          anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
+```
+
+The action is agent-agnostic — it installs `coder-eval` but not the agent
+runtime, so provide the `claude` CLI (or your agent's runtime) in the job first.
+See the [README "Use as a GitHub Action"](../../README.md#use-as-a-github-action)
+section for the full inputs table and the fork/`pull_request_target` security
+caveat. The rest of this tutorial's hand-rolled workflow is still useful when you
+need finer control than the action's inputs expose.
+
+## Publishing test results (JUnit)
+
+The verdict step above parses `task.json` by hand. If your CI already ingests
+JUnit XML — for the per-test annotations, history, and flake tracking most CI
+systems render from it — let `coder-eval` emit it directly instead. Add
+`--junit-xml` to the run:
+
+```yaml
+      - name: Run evaluations
+        run: coder-eval run $TASK_GLOBS -j 4 --junit-xml coder-eval-junit.xml
+```
+
+The file is written **before** the exit-code decision, so a failing run still
+produces a report. You can also regenerate it from any existing run directory
+without re-running the suite:
+
+```bash
+coder-eval report runs/latest -f junit            # writes runs/latest/junit.xml
+coder-eval report runs/latest -f junit -o out.xml # custom path
+```
+
+The mapping: one `<testcase>` per task row (grouped into a `<testsuite>` per
+variant), failed/errored rows carry a `<failure>`/`<error>` with the per-criterion
+breakdown, skipped tasks and failing suite gates get their own synthetic suites.
+
+Feed the file to whatever your platform uses to render test results. On GitHub
+Actions, [`mikepenz/action-junit-report`](https://github.com/mikepenz/action-junit-report):
+
+```yaml
+      - name: Publish test report
+        if: always()
+        uses: mikepenz/action-junit-report@v5
+        with:
+          report_paths: coder-eval-junit.xml
+```
+
+On Azure DevOps, `PublishTestResults@2`:
+
+```yaml
+      - task: PublishTestResults@2
+        condition: always()
+        inputs:
+          testResultsFormat: 'JUnit'
+          testResultsFiles: 'coder-eval-junit.xml'
+```
+
 ## Secrets
 
 Add these under **Settings → Secrets and variables → Actions**:

@@ -70,6 +70,7 @@ dev = [
     "pip-audit>=2.10.0",
     "bandit[toml]>=1.9.4",
     "pre-commit>=4.5.1",
+    "defusedxml>=0.7.1",  # test-side JUnit XML round-trip parsing (defense-in-depth; production code never parses XML)
 ]
 # Optional extra that enables UiPath-specific features:
 #   - the `uipath` Python SDK on the host (handy for local sandbox parity with