diff --git a/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml b/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml index 54500910ac182..e7d2701937577 100644 --- a/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml +++ b/ci-operator/config/openshift-eng/ai-helpers/openshift-eng-ai-helpers-main.yaml @@ -67,6 +67,8 @@ tests: env: EVAL_CASES: case-003 EVAL_CONFIG: plugins/ci/evals/eval-payload-analysis.yaml + EVAL_HARNESS_REF: eval-stop-guard-hook + EVAL_HARNESS_REPO: https://github.com/stbenjam/agent-eval-harness.git EVAL_MAX_TURNS: "250" EVAL_MODEL: claude-opus-4-6 EVAL_SETUP_SCRIPT: plugins/ci/evals/scripts/extract-payload-analysis-snapshots.sh diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh index 0851c71909fad..e80bf104e0449 100755 --- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh +++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh @@ -109,8 +109,11 @@ fi echo "" echo "=== Installing plugins ===" EVAL_HARNESS_DIR="/tmp/agent-eval-harness" -git clone --depth 1 https://github.com/opendatahub-io/agent-eval-harness.git "${EVAL_HARNESS_DIR}" -echo "agent-eval-harness cloned." +# EVAL_HARNESS_REPO / EVAL_HARNESS_REF let a PR rehearse the eval against a +# not-yet-merged branch of the harness. They default to the released main. +EVAL_HARNESS_REPO="${EVAL_HARNESS_REPO:-https://github.com/opendatahub-io/agent-eval-harness.git}" +git clone --depth 1 ${EVAL_HARNESS_REF:+--branch "${EVAL_HARNESS_REF}"} "${EVAL_HARNESS_REPO}" "${EVAL_HARNESS_DIR}" +echo "agent-eval-harness cloned from ${EVAL_HARNESS_REPO} (ref: ${EVAL_HARNESS_REF:-default})." # ----------------------------------------------------------------------- # Run optional setup script (e.g. extract snapshots, populate fixtures) diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml index 7868e87b9d868..43c4c7cdd43e4 100644 --- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml +++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml @@ -10,6 +10,10 @@ ref: name: claude-payload-agent-github-token mount_path: /var/run/github-token env: + - name: EVAL_HARNESS_REPO + default: "https://github.com/opendatahub-io/agent-eval-harness.git" + - name: EVAL_HARNESS_REF + default: "" - name: EVAL_CONFIG default: "eval.yaml" - name: EVAL_MODEL