feat: add verify script runner for post-agent outcome verification

pkg/evaluation/verify.go

@@ -0,0 +1,79 @@
+package evaluation
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+// verifyResult holds the outcome of running a verify script inside the
+// evaluation container.
+type verifyResult struct {
+	Passed   bool
+	ExitCode int
+	Output   string
+}
+
+// runVerifyScript executes a shell verify script and returns its outcome.
+// The script is run with sh -c; a zero exit code means pass. Output is
+// capped at maxVerifyOutputBytes to avoid unbounded memory from chatty
+// scripts.
+func runVerifyScript(ctx context.Context, script, containerRuntime, containerName string) (verifyResult, error) {
+	if script == "" {
+		return verifyResult{Passed: true}, nil
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
+	defer cancel()
+
+	args := []string{"exec", containerName, "sh", "-c", script}
+	cmd := exec.CommandContext(ctx, containerRuntime, args...)
+
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	err := cmd.Run()
+
+	output := strings.TrimSpace(stdout.String())
+	if errOutput := strings.TrimSpace(stderr.String()); errOutput != "" {
+		if output != "" {
+			output += "\n"
+		}
+		output += errOutput
+	}
+
+	// Cap output to avoid unbounded memory.
+	if len(output) > maxVerifyOutputBytes {
+		output = output[:maxVerifyOutputBytes] + "...(truncated)"
+	}
+
+	exitCode := 0
+	if err != nil {
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			exitCode = exitErr.ExitCode()
+		} else {
+			return verifyResult{ExitCode: -1, Output: output}, fmt.Errorf("running verify script: %w", err)
+		}
+	}
+
+	slog.DebugContext(ctx, "Verify script completed",
+		"exit_code", exitCode,
+		"output_length", len(output),
+	)
+
+	return verifyResult{
+		Passed:   exitCode == 0,
+		ExitCode: exitCode,
+		Output:   output,
+	}, nil
+}
+
+// maxVerifyOutputBytes caps the output captured from a verify script.
+const maxVerifyOutputBytes = 8192

pkg/evaluation/verify_test.go

@@ -0,0 +1,78 @@
+package evaluation
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRunVerifyScript_EmptyScriptPasses(t *testing.T) {
+	t.Parallel()
+	result, err := runVerifyScript(t.Context(), "", "docker", "c")
+	require.NoError(t, err)
+	assert.True(t, result.Passed)
+}
+
+func TestRunVerifyScript_ZeroExitCodePasses(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX shell required")
+	}
+	t.Parallel()
+
+	// Use a fake "container runtime" script that execs the verify command
+	// directly (no real container needed).
+	fake := writeFakeExecRuntime(t, "echo 'all good'; exit 0")
+
+	result, err := runVerifyScript(t.Context(), "verify", fake, "container-1")
+	require.NoError(t, err)
+	assert.True(t, result.Passed)
+	assert.Equal(t, 0, result.ExitCode)
+	assert.Contains(t, result.Output, "all good")
+}
+
+func TestRunVerifyScript_NonZeroExitCodeFails(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX shell required")
+	}
+	t.Parallel()
+
+	fake := writeFakeExecRuntime(t, "echo 'file missing'; exit 1")
+
+	result, err := runVerifyScript(t.Context(), "verify", fake, "container-1")
+	require.NoError(t, err)
+	assert.False(t, result.Passed)
+	assert.Equal(t, 1, result.ExitCode)
+	assert.Contains(t, result.Output, "file missing")
+}
+
+func TestRunVerifyScript_OutputCapped(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX shell required")
+	}
+	t.Parallel()
+
+	// Generate output larger than the cap.
+	fake := writeFakeExecRuntime(t, "dd if=/dev/zero bs=1 count=20000 2>/dev/null | tr '\\0' 'x'; exit 0")
+
+	result, err := runVerifyScript(t.Context(), "verify", fake, "container-1")
+	require.NoError(t, err)
+	assert.True(t, result.Passed)
+	assert.LessOrEqual(t, len(result.Output), maxVerifyOutputBytes+20) // +20 for truncation marker
+}
+
+// writeFakeExecRuntime creates a shell script that ignores docker exec args
+// and runs the given shell command instead, returning the script path.
+func writeFakeExecRuntime(t *testing.T, shCmd string) string {
+	t.Helper()
+	dir := t.TempDir()
+	path := filepath.Join(dir, "fake-runtime")
+	// The script ignores its arguments (exec container sh -c ...) and runs
+	// shCmd directly, simulating what the container would do.
+	script := "#!/bin/sh\n" + shCmd + "\n"
+	require.NoError(t, os.WriteFile(path, []byte(script), 0o755))
+	return path
+}