From 12414808d70ccc8cc98028af71d4a240a93274dc Mon Sep 17 00:00:00 2001 From: melmennaoui Date: Mon, 31 Aug 2026 17:30:23 +0200 Subject: [PATCH] feat: add assertions schema and verify field to EvalCriteria Add code-based assertion support to the evaluation criteria schema: - Assertion struct (Name, Type, Value) for declarative grading checks supporting types: contains, not_contains, equals, starts_with, ends_with, regex, cost_threshold, tool_called - Verify field on EvalCriteria for post-agent shell script verification - AssertionsCheck and VerifyCheck types on EvalResultChecks - AssertionResult type for per-assertion outcomes - Deep-clone support in cloneEvalCriteria and cloneEvalResultChecks - Tests for JSON unmarshaling of the new fields Closes #4082 --- pkg/session/branch.go | 10 +++++++ pkg/session/session.go | 56 +++++++++++++++++++++++++++++++------ pkg/session/session_test.go | 16 +++++++++++ 3 files changed, 74 insertions(+), 8 deletions(-) diff --git a/pkg/session/branch.go b/pkg/session/branch.go index 672aa3e73..3037e6d62 100644 --- a/pkg/session/branch.go +++ b/pkg/session/branch.go @@ -328,6 +328,7 @@ func cloneEvalCriteria(src *EvalCriteria) *EvalCriteria { } cp := *src cp.Relevance = cloneStringSlice(src.Relevance) + cp.Assertions = slices.Clone(src.Assertions) return &cp } @@ -361,6 +362,15 @@ func cloneEvalResultChecks(src EvalResultChecks) EvalResultChecks { relevance.Results = slices.Clone(src.Relevance.Results) cp.Relevance = &relevance } + if src.Assertions != nil { + assertions := *src.Assertions + assertions.Results = slices.Clone(src.Assertions.Results) + cp.Assertions = &assertions + } + if src.Verify != nil { + verify := *src.Verify + cp.Verify = &verify + } return cp } diff --git a/pkg/session/session.go b/pkg/session/session.go index 0ea600f42..953b836d2 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -638,9 +638,11 @@ type EvalResult struct { // EvalResultChecks groups the individual check results. // Only checks that were evaluated will be present (omitted if nil). type EvalResultChecks struct { - Size *SizeCheck `json:"size,omitempty"` - ToolCalls *ToolCallsCheck `json:"tool_calls,omitempty"` - Relevance *RelevanceCheck `json:"relevance,omitempty"` + Size *SizeCheck `json:"size,omitempty"` + ToolCalls *ToolCallsCheck `json:"tool_calls,omitempty"` + Relevance *RelevanceCheck `json:"relevance,omitempty"` + Assertions *AssertionsCheck `json:"assertions,omitempty"` + Verify *VerifyCheck `json:"verify,omitempty"` } // SizeCheck contains the result of the response size check. @@ -671,13 +673,51 @@ type RelevanceCriterionResult struct { Reason string `json:"reason,omitempty"` } +// AssertionsCheck contains the results of code-based assertion evaluations. +type AssertionsCheck struct { + Passed bool `json:"passed"` + PassedCount int `json:"passed_count"` + Total int `json:"total"` + Results []AssertionResult `json:"results"` +} + +// AssertionResult records the outcome of a single assertion. +type AssertionResult struct { + Name string `json:"name"` + Type string `json:"type"` + Passed bool `json:"passed"` + Reason string `json:"reason,omitempty"` +} + +// VerifyCheck contains the result of the post-agent verify script. +type VerifyCheck struct { + Passed bool `json:"passed"` + ExitCode int `json:"exit_code"` + Output string `json:"output,omitempty"` +} + // EvalCriteria contains the evaluation criteria for a session. type EvalCriteria struct { - Relevance []string `json:"relevance"` // Statements that should be true about the response - WorkingDir string `json:"working_dir,omitempty"` // Subdirectory under evals/working_dirs/ - Size string `json:"size,omitempty"` // Expected response size: S, M, L, XL - Setup string `json:"setup,omitempty"` // Optional sh script to run in the container before docker agent run --exec - Image string `json:"image,omitempty"` // Custom Docker image for this eval (overrides --base-image) + Relevance []string `json:"relevance"` // Statements that should be true about the response + Assertions []Assertion `json:"assertions,omitempty"` // Code-based assertions evaluated against the agent output + Verify string `json:"verify,omitempty"` // Shell script for post-agent outcome verification + WorkingDir string `json:"working_dir,omitempty"` // Subdirectory under evals/working_dirs/ + Size string `json:"size,omitempty"` // Expected response size: S, M, L, XL + Setup string `json:"setup,omitempty"` // Optional sh script to run in the container before docker agent run --exec + Image string `json:"image,omitempty"` // Custom Docker image for this eval (overrides --base-image) +} + +// Assertion defines a single code-based grading check evaluated against agent output. +type Assertion struct { + // Name identifies this assertion in logs and results. + Name string `json:"name"` + // Type selects the evaluator: "contains", "not_contains", "equals", + // "starts_with", "ends_with", "regex", "json_path", "cost_threshold", + // or "tool_called". + Type string `json:"type"` + // Value is the expected string, regex pattern, JSONPath expression, or + // threshold against which the agent output is checked. + Value string `json:"value"` } // UnmarshalJSON implements custom JSON unmarshaling for EvalCriteria that diff --git a/pkg/session/session_test.go b/pkg/session/session_test.go index 95604fb86..e45045ef6 100644 --- a/pkg/session/session_test.go +++ b/pkg/session/session_test.go @@ -512,6 +512,22 @@ func TestEvalCriteriaUnmarshalJSON(t *testing.T) { WorkingDir: "mydir", }, }, + { + name: "valid with assertions", + input: `{"relevance":[],"assertions":[{"name":"has greeting","type":"contains","value":"hello"}]}`, + want: EvalCriteria{ + Relevance: []string{}, + Assertions: []Assertion{{Name: "has greeting", Type: "contains", Value: "hello"}}, + }, + }, + { + name: "valid with verify", + input: `{"relevance":[],"verify":"test -f output.txt"}`, + want: EvalCriteria{ + Relevance: []string{}, + Verify: "test -f output.txt", + }, + }, { name: "empty object", input: `{}`,