diff --git a/pkg/evaluation/verify.go b/pkg/evaluation/verify.go new file mode 100644 index 000000000..3bab27be7 --- /dev/null +++ b/pkg/evaluation/verify.go @@ -0,0 +1,79 @@ +package evaluation + +import ( + "bytes" + "context" + "errors" + "fmt" + "log/slog" + "os/exec" + "strings" + "time" +) + +// verifyResult holds the outcome of running a verify script inside the +// evaluation container. +type verifyResult struct { + Passed bool + ExitCode int + Output string +} + +// runVerifyScript executes a shell verify script and returns its outcome. +// The script is run with sh -c; a zero exit code means pass. Output is +// capped at maxVerifyOutputBytes to avoid unbounded memory from chatty +// scripts. +func runVerifyScript(ctx context.Context, script, containerRuntime, containerName string) (verifyResult, error) { + if script == "" { + return verifyResult{Passed: true}, nil + } + + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + + args := []string{"exec", containerName, "sh", "-c", script} + cmd := exec.CommandContext(ctx, containerRuntime, args...) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + + output := strings.TrimSpace(stdout.String()) + if errOutput := strings.TrimSpace(stderr.String()); errOutput != "" { + if output != "" { + output += "\n" + } + output += errOutput + } + + // Cap output to avoid unbounded memory. + if len(output) > maxVerifyOutputBytes { + output = output[:maxVerifyOutputBytes] + "...(truncated)" + } + + exitCode := 0 + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + exitCode = exitErr.ExitCode() + } else { + return verifyResult{ExitCode: -1, Output: output}, fmt.Errorf("running verify script: %w", err) + } + } + + slog.DebugContext(ctx, "Verify script completed", + "exit_code", exitCode, + "output_length", len(output), + ) + + return verifyResult{ + Passed: exitCode == 0, + ExitCode: exitCode, + Output: output, + }, nil +} + +// maxVerifyOutputBytes caps the output captured from a verify script. +const maxVerifyOutputBytes = 8192 diff --git a/pkg/evaluation/verify_test.go b/pkg/evaluation/verify_test.go new file mode 100644 index 000000000..0c5b6c355 --- /dev/null +++ b/pkg/evaluation/verify_test.go @@ -0,0 +1,78 @@ +package evaluation + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRunVerifyScript_EmptyScriptPasses(t *testing.T) { + t.Parallel() + result, err := runVerifyScript(t.Context(), "", "docker", "c") + require.NoError(t, err) + assert.True(t, result.Passed) +} + +func TestRunVerifyScript_ZeroExitCodePasses(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX shell required") + } + t.Parallel() + + // Use a fake "container runtime" script that execs the verify command + // directly (no real container needed). + fake := writeFakeExecRuntime(t, "echo 'all good'; exit 0") + + result, err := runVerifyScript(t.Context(), "verify", fake, "container-1") + require.NoError(t, err) + assert.True(t, result.Passed) + assert.Equal(t, 0, result.ExitCode) + assert.Contains(t, result.Output, "all good") +} + +func TestRunVerifyScript_NonZeroExitCodeFails(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX shell required") + } + t.Parallel() + + fake := writeFakeExecRuntime(t, "echo 'file missing'; exit 1") + + result, err := runVerifyScript(t.Context(), "verify", fake, "container-1") + require.NoError(t, err) + assert.False(t, result.Passed) + assert.Equal(t, 1, result.ExitCode) + assert.Contains(t, result.Output, "file missing") +} + +func TestRunVerifyScript_OutputCapped(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX shell required") + } + t.Parallel() + + // Generate output larger than the cap. + fake := writeFakeExecRuntime(t, "dd if=/dev/zero bs=1 count=20000 2>/dev/null | tr '\\0' 'x'; exit 0") + + result, err := runVerifyScript(t.Context(), "verify", fake, "container-1") + require.NoError(t, err) + assert.True(t, result.Passed) + assert.LessOrEqual(t, len(result.Output), maxVerifyOutputBytes+20) // +20 for truncation marker +} + +// writeFakeExecRuntime creates a shell script that ignores docker exec args +// and runs the given shell command instead, returning the script path. +func writeFakeExecRuntime(t *testing.T, shCmd string) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "fake-runtime") + // The script ignores its arguments (exec container sh -c ...) and runs + // shCmd directly, simulating what the container would do. + script := "#!/bin/sh\n" + shCmd + "\n" + require.NoError(t, os.WriteFile(path, []byte(script), 0o755)) + return path +}