diff --git a/internal/codex/benchmark.go b/internal/codex/benchmark.go index 61d6380..1a6d4de 100644 --- a/internal/codex/benchmark.go +++ b/internal/codex/benchmark.go @@ -1198,27 +1198,34 @@ func applyBenchmarkMeasurements(result *BenchmarkResult, telemetry *benchmarkTel telemetry.apply(result) if result.ToolUsed { result.CostIssue = "tool use prohibited: " + result.ToolType - } else if result.UsageKnown { - if result.UsageSource == BenchmarkUsageRawResponses { - result.CostKnown = true - for _, response := range result.ResponseUsage { - model := response.Model - if model == "" { - model = result.ActualModel - } - cost, known, issue := estimateSingleResponseAPICostWithIssue(model, response.Usage) - if !known { - result.CostUSD, result.CostKnown, result.CostIssue = 0, false, issue - break - } - result.CostUSD += cost - } - } else { - result.CostUSD, result.CostKnown, result.CostIssue = estimateAPICostWithIssue(result.ActualModel, result.Usage) + return + } + result.CostUSD, result.CostKnown, result.CostIssue = benchmarkMeasurementCost(result.ActualModel, result) +} + +// benchmarkMeasurementCost keeps per-response attribution and long-context +// pricing identical for built-in benchmarks and DigBench. Run-specific tool +// policies remain the caller's responsibility. +func benchmarkMeasurementCost(model string, measurements *BenchmarkResult) (float64, bool, string) { + if !measurements.UsageKnown { + return 0, false, measurements.UsageIssue + } + if measurements.UsageSource != BenchmarkUsageRawResponses { + return estimateAPICostWithIssue(model, measurements.Usage) + } + var total float64 + for _, response := range measurements.ResponseUsage { + responseModel := response.Model + if responseModel == "" { + responseModel = model } - } else { - result.CostIssue = result.UsageIssue + cost, known, issue := estimateSingleResponseAPICostWithIssue(responseModel, response.Usage) + if !known { + return 0, false, issue + } + total += cost } + return total, true, "" } type benchmarkTelemetry struct { diff --git a/internal/codex/benchmark_measurements_test.go b/internal/codex/benchmark_measurements_test.go new file mode 100644 index 0000000..3853b13 --- /dev/null +++ b/internal/codex/benchmark_measurements_test.go @@ -0,0 +1,100 @@ +package codex + +import ( + "math" + "strings" + "testing" +) + +func TestBenchmarkMeasurementCostsAgreeAcrossRunners(t *testing.T) { + cached := BenchmarkUsage{InputTokens: 1_000, CachedInputTokens: 400, OutputTokens: 100, TotalTokens: 1_100} + cumulative := func(usage BenchmarkUsage) func(*benchmarkTelemetry) { + return func(telemetry *benchmarkTelemetry) { telemetry.recordCumulative(usage) } + } + for _, tc := range []struct { + name, model, issue string + observe func(*benchmarkTelemetry) + source BenchmarkUsageSource + usageKnown, priced bool + total int64 + cost float64 + }{ + {name: "missing telemetry", model: "gpt-5.2", source: BenchmarkUsageUnavailable, issue: "matching usage event was not observed"}, + {name: "observed zero", model: "gpt-5.2", observe: cumulative(BenchmarkUsage{}), source: BenchmarkUsageCumulative, usageKnown: true, priced: true}, + {name: "cumulative cached input", model: "gpt-5.2", observe: cumulative(cached), source: BenchmarkUsageCumulative, usageKnown: true, priced: true, total: 1_100, cost: 0.00252}, + {name: "raw model attribution and fallback", model: "gpt-5.2", source: BenchmarkUsageRawResponses, usageKnown: true, priced: true, total: 1_650, cost: 0.00912, + observe: func(telemetry *benchmarkTelemetry) { + telemetry.recordRawResponse("fallback", &cached) + other := BenchmarkUsage{InputTokens: 500, CachedInputTokens: 100, OutputTokens: 50, TotalTokens: 550} + telemetry.recordRawResponseForModel("rerouted", &other, "gpt-6-astra") + telemetry.recordCumulative(BenchmarkUsage{TotalTokens: 128_000}) // Synthetic context snapshot must not replace exact usage. + }}, + {name: "long context priced per response", model: "gpt-5.6-terra", source: BenchmarkUsageRawResponses, usageKnown: true, priced: true, total: 402_000, cost: 0.824, + observe: func(telemetry *benchmarkTelemetry) { + response := BenchmarkUsage{InputTokens: 200_000, OutputTokens: 1_000, TotalTokens: 201_000} + telemetry.recordRawResponse("one", &response) + telemetry.recordRawResponse("two", &response) + }}, + {name: "ambiguous cumulative long context", model: "gpt-5.6-terra", source: BenchmarkUsageCumulative, usageKnown: true, total: 402_000, issue: "requires per-response usage", + observe: cumulative(BenchmarkUsage{InputTokens: 400_000, OutputTokens: 2_000, TotalTokens: 402_000})}, + {name: "unknown response discards partial price", model: "gpt-5.2", source: BenchmarkUsageRawResponses, usageKnown: true, total: 2_200, issue: "no maintained standard API price", + observe: func(telemetry *benchmarkTelemetry) { + telemetry.recordRawResponse("priced", &cached) + telemetry.recordRawResponseForModel("unpriced", &cached, "unpriced-test-model") + }}, + {name: "unknown cache write price", model: "gpt-5.2", source: BenchmarkUsageCumulative, usageKnown: true, total: 1_100, issue: "no published cache-write price", + observe: cumulative(BenchmarkUsage{InputTokens: 1_000, CacheWriteInputTokens: 100, OutputTokens: 100, TotalTokens: 1_100})}, + {name: "missing raw usage stays unknown", model: "gpt-5.2", source: BenchmarkUsageRawResponses, issue: "raw response omitted usage", + observe: func(telemetry *benchmarkTelemetry) { + telemetry.recordRawResponse("priced", &cached) + telemetry.recordRawResponse("missing", nil) + telemetry.recordCumulative(cached) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + telemetry := newBenchmarkTelemetry() + if tc.observe != nil { + tc.observe(telemetry) + } + builtin := BenchmarkResult{ActualModel: tc.model, UsageKnown: true, UsageIssue: "stale", CostUSD: 99, CostKnown: true, CostIssue: "stale"} + dig := DigBenchResult{ActualModel: tc.model, UsageKnown: true, UsageIssue: "stale", CostUSD: 99, CostKnown: true, CostIssue: "stale"} + for range 2 { // Repeated progress snapshots must not accumulate cost or retain stale errors. + applyBenchmarkMeasurements(&builtin, telemetry) + applyDigBenchMeasurements(&dig, telemetry) + for _, got := range []struct { + name string + usageKnown, priced bool + source BenchmarkUsageSource + total int64 + cost float64 + issue string + }{ + {"built-in", builtin.UsageKnown, builtin.CostKnown, builtin.UsageSource, builtin.Usage.TotalTokens, builtin.CostUSD, builtin.CostIssue}, + {"DigBench", dig.UsageKnown, dig.CostKnown, dig.UsageSource, dig.Usage.TotalTokens, dig.CostUSD, dig.CostIssue}, + } { + if got.usageKnown != tc.usageKnown || got.priced != tc.priced || got.source != tc.source || got.total != tc.total || math.Abs(got.cost-tc.cost) > 1e-12 { + t.Fatalf("%s measurements = %+v; want source=%s usage=%v total=%d priced=%v cost=%g", got.name, got, tc.source, tc.usageKnown, tc.total, tc.priced, tc.cost) + } + if tc.issue == "" && got.issue != "" || tc.issue != "" && !strings.Contains(got.issue, tc.issue) { + t.Fatalf("%s issue = %q; want %q", got.name, got.issue, tc.issue) + } + } + } + }) + } +} + +func TestSharedMeasurementPricingKeepsBenchmarkToolPoliciesSeparate(t *testing.T) { + telemetry := newBenchmarkTelemetry() + telemetry.recordCumulative(BenchmarkUsage{InputTokens: 1_000, CachedInputTokens: 400, OutputTokens: 100, TotalTokens: 1_100}) + builtin := BenchmarkResult{ActualModel: "gpt-5.2", ToolUsed: true, ToolType: "exec", CostUSD: 99, CostKnown: true} + dig := DigBenchResult{ActualModel: "gpt-5.2"} + applyBenchmarkMeasurements(&builtin, telemetry) + applyDigBenchMeasurements(&dig, telemetry) + if !builtin.UsageKnown || builtin.CostKnown || builtin.CostUSD != 0 || builtin.CostIssue != "tool use prohibited: exec" { + t.Fatalf("built-in tool restriction changed: %+v", builtin) + } + if !dig.CostKnown || math.Abs(dig.CostUSD-0.00252) > 1e-12 || dig.CostIssue != "" { + t.Fatalf("DigBench's permitted tool use was incorrectly rejected: %+v", dig) + } +} diff --git a/internal/codex/digbench.go b/internal/codex/digbench.go index d9d917f..4a47d65 100644 --- a/internal/codex/digbench.go +++ b/internal/codex/digbench.go @@ -1008,7 +1008,6 @@ func digBenchFinalResponse(params json.RawMessage, threadID, turnID string, sens } func applyDigBenchMeasurements(result *DigBenchResult, telemetry *benchmarkTelemetry) { - result.CostUSD, result.CostKnown, result.CostIssue = 0, false, "" var measurements BenchmarkResult telemetry.apply(&measurements) result.Usage = measurements.Usage @@ -1017,25 +1016,5 @@ func applyDigBenchMeasurements(result *DigBenchResult, telemetry *benchmarkTelem result.UsageIssue = measurements.UsageIssue result.UsageSource = measurements.UsageSource result.ResponseUsage = measurements.ResponseUsage - if !result.UsageKnown { - result.CostIssue = result.UsageIssue - return - } - if result.UsageSource != BenchmarkUsageRawResponses { - result.CostUSD, result.CostKnown, result.CostIssue = estimateAPICostWithIssue(result.ActualModel, result.Usage) - return - } - result.CostKnown = true - for _, response := range result.ResponseUsage { - model := response.Model - if model == "" { - model = result.ActualModel - } - cost, known, issue := estimateSingleResponseAPICostWithIssue(model, response.Usage) - if !known { - result.CostUSD, result.CostKnown, result.CostIssue = 0, false, issue - return - } - result.CostUSD += cost - } + result.CostUSD, result.CostKnown, result.CostIssue = benchmarkMeasurementCost(result.ActualModel, &measurements) } diff --git a/internal/ui/benchmark_reset_test.go b/internal/ui/benchmark_reset_test.go new file mode 100644 index 0000000..ccc29cc --- /dev/null +++ b/internal/ui/benchmark_reset_test.go @@ -0,0 +1,49 @@ +package ui + +import ( + "reflect" + "testing" + "time" + + "github.com/merefield/codexometer/internal/codex" +) + +func TestBenchmarkStartAndClearResetPresentationWithoutLosingPreferences(t *testing.T) { + for _, start := range []bool{false, true} { + t.Run(map[bool]string{false: "clear", true: "start"}[start], func(t *testing.T) { + old := codex.BenchmarkResult{TaskID: codex.BenchmarkLRUCache, TaskName: "LRU CACHE", Model: "saved-model", Effort: "high"} + scope := codex.BenchmarkScope{Models: []string{"saved-model"}, Efforts: []string{"high"}} + model := Model{ + benchmarkRunner: benchmarkCaptureFetcher{}, benchmarkCombinations: 2, + benchmarkState: benchmarkFinished, benchmarkResults: []codex.BenchmarkResult{old}, + benchmarkTotal: 99, benchmarkCompleted: 42, benchmarkCurrentModel: "stale", benchmarkCurrentEffort: "stale", benchmarkCurrentTask: "stale", benchmarkError: "stale", + benchmarkScroll: 7, benchmarkActive: &old, benchmarkActiveSince: time.Now(), benchmarkSelectedRun: "stale", benchmarkHoveredRun: "stale", benchmarkRunHovered: true, + benchmarkDetail: &old, benchmarkDetailActive: true, benchmarkDetailScroll: 9, benchmarkDetailCache: benchmarkDetailTranscriptCache{valid: true, lines: []string{"cached"}}, + benchmarkAllArmed: true, benchmarkSelectedArmed: true, benchmarkScopeOpen: true, benchmarkScopeKeyboard: true, + benchmarkScope: scope, benchmarkFilter: benchmarkFilterFail, benchmarkRankMode: benchmarkRankCost, benchmarkSort: benchmarkSortTokens, benchmarkSortDescending: true, + } + if start { + updated, cmd := model.startBenchmark([]codex.BenchmarkTaskID{codex.BenchmarkMergeRanges}, scope, false) + model = updated + defer model.cancelBenchmark() + if cmd == nil || model.benchmarkState != benchmarkRunning || model.benchmarkTotal != 2 || len(model.benchmarkResults) != 1 || model.benchmarkResults[0].TaskID != old.TaskID || model.benchmarkRunResults == nil || model.benchmarkRunIndexes == nil { + t.Fatal("start changed run/history lifecycle") + } + } else { + model.clearBenchmarkResults() + if model.benchmarkState != benchmarkIdle || model.benchmarkTotal != 0 || model.benchmarkResults != nil || model.benchmarkResultIndexes != nil || model.benchmarkRunResults != nil || model.benchmarkRunIndexes != nil { + t.Fatal("clear did not discard results and run indexes") + } + } + if model.benchmarkCompleted != 0 || model.benchmarkCurrentModel != "" || model.benchmarkCurrentEffort != "" || model.benchmarkCurrentTask != "" || model.benchmarkError != "" || model.benchmarkScroll != 0 || model.benchmarkActive != nil || !model.benchmarkActiveSince.IsZero() || model.benchmarkSelectedRun != "" || model.benchmarkHoveredRun != "" || model.benchmarkRunHovered || model.benchmarkDetail != nil || model.benchmarkDetailActive || model.benchmarkDetailScroll != 0 || !reflect.DeepEqual(model.benchmarkDetailCache, benchmarkDetailTranscriptCache{}) || model.benchmarkAllArmed || model.benchmarkSelectedArmed { + t.Fatal("stale benchmark presentation survived") + } + if !reflect.DeepEqual(model.benchmarkScope, scope) || model.benchmarkFilter != benchmarkFilterFail || model.benchmarkRankMode != benchmarkRankCost || model.benchmarkSort != benchmarkSortTokens || !model.benchmarkSortDescending { + t.Fatal("reset changed scope/filter/ranking preferences") + } + if model.benchmarkScopeOpen == start || model.benchmarkScopeKeyboard == start { + t.Fatal("start/clear scope-form behaviours were conflated") + } + }) + } +} diff --git a/internal/ui/model.go b/internal/ui/model.go index d362806..5ddc855 100644 --- a/internal/ui/model.go +++ b/internal/ui/model.go @@ -1765,27 +1765,11 @@ func (m Model) startBenchmark(tasks []codex.BenchmarkTaskID, scope codex.Benchma if len(tasks) == 1 && tasks[0] == codex.BenchmarkDigBench { m.benchmarkTotal = combinations * len(scope.Games) } - m.benchmarkCompleted = 0 - m.benchmarkCurrentModel = "" - m.benchmarkCurrentEffort = "" - m.benchmarkCurrentTask = "" - m.benchmarkError = "" - m.benchmarkScroll = 0 - m.benchmarkActive = nil - m.benchmarkActiveSince = time.Time{} - m.benchmarkSelectedRun = "" - m.benchmarkHoveredRun = "" - m.benchmarkRunHovered = false - m.benchmarkDetail = nil - m.benchmarkDetailActive = false - m.benchmarkDetailScroll = 0 - m.benchmarkDetailCache = benchmarkDetailTranscriptCache{} + m.resetBenchmarkPresentation() m.benchmarkScopeOpen = false m.benchmarkScopeKeyboard = false m.benchmarkState = benchmarkRunning m.benchmarkQuotaAccounting.start() - m.benchmarkAllArmed = false - m.benchmarkSelectedArmed = false ctx, cancel := context.WithCancel(context.Background()) m.benchmarkCancel = cancel events := make(chan codex.BenchmarkEvent, 2) @@ -1800,6 +1784,12 @@ func (m *Model) clearBenchmarkResults() { m.benchmarkRunResults = nil m.benchmarkRunIndexes = nil m.benchmarkTotal = 0 + m.resetBenchmarkPresentation() +} + +// resetBenchmarkPresentation clears transient run/detail state without +// changing results, scope, filters or the caller's run lifecycle. +func (m *Model) resetBenchmarkPresentation() { m.benchmarkCompleted = 0 m.benchmarkCurrentModel = "" m.benchmarkCurrentEffort = "" diff --git a/internal/ui/monitor_approval.go b/internal/ui/monitor_approval.go index 4ed77d5..b438d19 100644 --- a/internal/ui/monitor_approval.go +++ b/internal/ui/monitor_approval.go @@ -233,13 +233,16 @@ func (m Model) monitorApprovalControlRows(width, height int) int { } func (m Model) renderMonitorApprovalControls(width, height int, colors palette) string { - buttons := m.monitorApprovalButtons(width, height) + return renderMonitorButtonRows(m.monitorApprovalButtons(width, height), colors, m.renderMonitorApprovalAction) +} + +func renderMonitorButtonRows(buttons []monitorApprovalButton, colors palette, render func(string, string, palette) string) string { if len(buttons) == 0 { return "" } rows := make([]string, buttons[len(buttons)-1].y+1) for _, b := range buttons { - rows[b.y] += strings.Repeat(" ", max(b.x-lipgloss.Width(rows[b.y]), 0)) + m.renderMonitorApprovalAction(b.action, b.label, colors) + rows[b.y] += strings.Repeat(" ", max(b.x-lipgloss.Width(rows[b.y]), 0)) + render(b.action, b.label, colors) } return strings.Join(rows, "\n") } diff --git a/internal/ui/monitor_buttons_test.go b/internal/ui/monitor_buttons_test.go new file mode 100644 index 0000000..6cad868 --- /dev/null +++ b/internal/ui/monitor_buttons_test.go @@ -0,0 +1,57 @@ +package ui + +import ( + "slices" + "strings" + "testing" +) + +func TestMonitorButtonRowsPreservePositionsAndVisibleWidths(t *testing.T) { + if got := renderMonitorButtonRows(nil, palette{}, nil); got != "" { + t.Fatalf("empty controls = %q", got) + } + buttons := []monitorApprovalButton{ + {action: "first", label: "ē•Œ", x: 1, y: 0}, + {action: "second", label: "B", x: 6, y: 0}, + {action: "third", label: "C", x: 2, y: 2}, + } + var actions []string + got := renderMonitorButtonRows(buttons, palette{}, func(action, label string, _ palette) string { + actions = append(actions, action) + return "\x1b[31m" + label + "\x1b[0m" + }) + want := " \x1b[31mē•Œ\x1b[0m \x1b[31mB\x1b[0m\n\n \x1b[31mC\x1b[0m" + if got != want || !slices.Equal(actions, []string{"first", "second", "third"}) { + t.Fatalf("positioned controls = %q, actions = %v; want %q", got, actions, want) + } +} + +func TestMonitorButtonRowsKeepApprovalAndProfileRenderers(t *testing.T) { + approval := approvalTestModel() + profile, _ := profileTestModel(t) + for _, theme := range []themeID{themeHacker, themeRust, themeBlueSteel, themeUltraviolet, themeNightshade} { + colors := paletteFor(theme) + for _, width := range []int{20, 50, 120} { + for _, tc := range []struct { + buttons []monitorApprovalButton + render func(string, string, palette) string + output string + }{ + {approval.monitorApprovalButtons(width, 40), approval.renderMonitorApprovalAction, approval.renderMonitorApprovalControls(width, 40, colors)}, + {profile.profileButtons(width, 40, profile.monitorSessionData[0]), profile.renderContextAction, profile.renderProfileControls(profile.profileButtons(width, 40, profile.monitorSessionData[0]), colors)}, + } { + if width == 120 && len(tc.buttons) == 0 { + t.Fatal("wide fixture must exercise actual controls") + } + if len(tc.buttons) == 0 && tc.output != "" { + t.Fatal("hidden controls rendered") + } + for _, button := range tc.buttons { + if !strings.Contains(tc.output, tc.render(button.action, button.label, colors)) { + t.Fatal("button lost its original renderer", theme, width, button.action) + } + } + } + } + } +} diff --git a/internal/ui/monitor_profile.go b/internal/ui/monitor_profile.go index fccc8a9..cdbf7a0 100644 --- a/internal/ui/monitor_profile.go +++ b/internal/ui/monitor_profile.go @@ -135,14 +135,7 @@ func (m Model) profileButtons(width, height int, s monitorSession) []monitorAppr } func (m Model) renderProfileControls(buttons []monitorApprovalButton, colors palette) string { - if len(buttons) == 0 { - return "" - } - rows := make([]string, buttons[len(buttons)-1].y+1) - for _, b := range buttons { - rows[b.y] += strings.Repeat(" ", max(b.x-lipgloss.Width(rows[b.y]), 0)) + m.renderContextAction(b.action, b.label, colors) - } - return strings.Join(rows, "\n") + return renderMonitorButtonRows(buttons, colors, m.renderContextAction) } func (m Model) renderSessionProfile(width, height int, s monitorSession, colors palette) string {