diff --git a/.changeset/shadow-costed-by-core.md b/.changeset/shadow-costed-by-core.md new file mode 100644 index 000000000..4188984e7 --- /dev/null +++ b/.changeset/shadow-costed-by-core.md @@ -0,0 +1,37 @@ +--- +"ftw": patch +--- + +The Python shadow's verdict is now Core's price for both plans. Core walks +the challenger's own action sequence through its forward pass and costs it +with the same arithmetic the DP uses on itself, instead of reading the total +the challenger arrived with — and it does the same for its own plan, so the +subtraction stays symmetric if the DP's bookkeeping ever moves. The +difference keeps the key `python_minus_core_ore_terminal_corrected`. + +What the challenger reported is kept beside the verdict rather than inside +it: `self_reported_ore` is the cost it claimed for its own plan, and +`self_reported_objective_ore` is the value it actually minimized — +scenario-weighted and CVaR-shaped, so not comparable with any cost. On the +owner's box those two differ by 450 öre on a single plan. + +A plan Core cannot cost honestly — wrong length, non-finite output, a +battery driven outside its operating band or past its power limits — is +recorded as `evaluation_refused_reason` with no difference number at all. +Core costing its own plan differently from what it reported is a bug in +Core, and now says so with a warning and `active_evaluation_drift_ore`. + +The replay bench measures the same way, prints the challenger's own figure +in its own column, and takes `FTW_MPC_BENCH_CVAR_WEIGHT`, +`FTW_MPC_BENCH_CVAR_ALPHA` and `FTW_MPC_BENCH_MIP_GAP`. Those knobs matter: +a snapshot records the planning inputs but not the site's solver settings, +so replaying one with the bench's defaults asks the challenger a different +question than the box asked — worth more than 130 öre per plan on a summer +day with PV uncertainty, which is larger than the gap being measured. + +A second bench asks what the price twin is worth. It re-solves each snapshot +three ways — the confidences the box used, those same forecast slots +flattened to the horizon mean, and the forecast slots deleted — and reports +the only number that reaches hardware: the first slot's battery power. On +every snapshot recorded so far the three agree exactly, and they keep +agreeing until the known window falls under four hours. diff --git a/go/internal/mpc/forecast_value_bench_test.go b/go/internal/mpc/forecast_value_bench_test.go new file mode 100644 index 000000000..03de970dd --- /dev/null +++ b/go/internal/mpc/forecast_value_bench_test.go @@ -0,0 +1,798 @@ +package mpc + +// Forecast-value bench: does guessing tomorrow's price change what the box +// does RIGHT NOW? +// +// Spot prices publish around 13:00 local. Before that the box holds perhaps +// 12 hours of real day-ahead prices; after, up to 36. The rest of the 48 h +// horizon is an ML price twin, marked Confidence < 1.0, and the DP blends it +// toward the horizon mean: +// +// effPrice(slot) = confidence × rawPrice + (1 − confidence) × horizonMean +// +// MPC dispatches only the FIRST slot, so the decisive question is not whether +// the guess moves a plan 30 hours out. It is whether it moves the watt that +// reaches hardware in the next 15 minutes. This bench re-solves each recorded +// snapshot three ways on one grid and reports that watt: +// +// A as recorded — the confidences the box actually used +// B flat guess — every forecast slot pinned to the horizon mean +// C truncated — the guessed slots deleted, solve the known window only +// +// Two honest limits on what B and C mean, both worth stating before reading +// any number below: +// +// - B does not erase the guess, it flattens it. horizonMeans() averages +// over every slot, forecast rows included, so B still inherits the LEVEL +// the twin predicted — it discards only the SHAPE. That is exactly what +// Confidence → 0 does inside the DP today, which is the thing being +// measured. +// - C deletes the guessed slots from the solve but still runs at the +// terminal price the box derived from the FULL horizon. A box that truly +// knew nothing past day-ahead would price stored energy off the known +// window alone, so the sweep below also reports C at a terminal price +// rederived from the known slots, using Core's own mode-dependent +// formula. +// +// And the limit on the whole exercise: these snapshots carry no ground truth +// about what prices turned out to be. This bench can say whether the guess +// changes the action. It cannot say which action earned more money. That +// needs a closed-loop backtest against realised prices. +// +// Skipped without FTW_MPC_SNAPSHOT_DIR; the snapshot directory stays outside +// the repository because real blobs carry a household's load traces. + +import ( + "fmt" + "math" + "os" + "path/filepath" + "slices" + "sort" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/state" +) + +// benchNoTrustConfidence is the "trust nothing" confidence for variant B. +// +// It is not 0. Confidence ≤ 0 means "caller did not fill this in" to Core and +// is coerced to 1.0 — twice, once in sanitizeOptimizeSlots and again in +// Optimize's own defaulting loop — so asking for zero trust with a zero would +// silently ask for TOTAL trust, the exact opposite of the variant. The +// smallest value that survives both coercions is any positive float; 1e-9 +// puts effPrice within ~1e-7 öre of the horizon mean, which is far below the +// öre the DP can act on. +const benchNoTrustConfidence = 1e-9 + +// benchActionDiffW is how far two plans' battery power must part before the +// slot counts as a disagreement. 50 W is well under the resolution any +// inverter tracks and well over DP grid residue. +const benchActionDiffW = 50.0 + +// benchLastKnownSlot reports the index of the last confidence-1.0 slot and +// whether the known slots form a contiguous prefix. A non-contiguous known +// window would make "truncate to what we know" ill-defined, so the caller +// says so rather than quietly truncating at a hole. +func benchLastKnownSlot(slots []Slot) (last int, contiguous bool, count int) { + last = -1 + for i, s := range slots { + if s.Confidence >= 1.0 { + last = i + count++ + } + } + if last < 0 { + return -1, false, 0 + } + return last, count == last+1, count +} + +// benchFlattenForecast returns a copy of slots with every slot the box was +// unsure about pinned to no-trust confidence. Known slots are untouched. +func benchFlattenForecast(slots []Slot) []Slot { + out := make([]Slot, len(slots)) + copy(out, slots) + for i := range out { + if out[i].Confidence < 1.0 { + out[i].Confidence = benchNoTrustConfidence + } + } + return out +} + +// benchHoursTo is the wall-clock offset of slot index i, so a divergence +// index can be read as "the guess stops mattering after N hours". +func benchHoursTo(slots []Slot, i int) float64 { + var min float64 + for k := 0; k < i && k < len(slots); k++ { + min += float64(slots[k].LenMin) + } + return min / 60.0 +} + +// benchFirstDivergence finds the first slot where two plans' battery power +// parts by more than benchActionDiffW, comparing only as far as the shorter +// plan reaches. ok is false when they agree the whole way. +func benchFirstDivergence(a, b *Plan) (idx int, ok bool) { + n := len(a.Actions) + if len(b.Actions) < n { + n = len(b.Actions) + } + for i := 0; i < n; i++ { + if math.Abs(a.Actions[i].BatteryW-b.Actions[i].BatteryW) > benchActionDiffW { + return i, true + } + } + return n, false +} + +// benchFirstSlotW is the only number in this file that reaches hardware. +func benchFirstSlotW(p *Plan) float64 { + if p == nil || len(p.Actions) == 0 { + return math.NaN() + } + return p.Actions[0].BatteryW +} + +// benchKnownWindowCost costs a plan over the KNOWN slots only, under Core's +// model, terminal-corrected at the scoring price. +// +// All three variants must be scored on the same real-price slots. Costing A +// over 48 h of guessed prices and C over 13 h of real ones compares two +// different questions and would make the shorter horizon look cheap for no +// reason but being shorter. +func benchKnownWindowCost(plan *Plan, slots []Slot, n int, solveP, scoreP Params) (corrected, raw, endSoC float64, err error) { + if plan == nil || len(plan.Actions) < n { + return 0, 0, 0, fmt.Errorf("plan has %d actions, need %d", len(plan.Actions), n) + } + trunc := Plan{Actions: plan.Actions[:n]} + eval, err := evaluatePlan(trunc, slots[:n], solveP) + if err != nil { + return 0, 0, 0, err + } + return terminalCorrectedOre(eval.CostOre, eval.EndSoC, scoreP), eval.CostOre, eval.EndSoC, nil +} + +// benchTerminalPrice rederives the terminal credit from a given set of slots +// using Core's own mode-dependent formula (service.go), so "C, knowing +// nothing beyond day-ahead" can price stored energy off the known window +// instead of inheriting a number computed from the guess. +func benchTerminalPrice(mode Mode, slots []Slot) float64 { + prices := make([]state.PricePoint, 0, len(slots)) + for _, s := range slots { + prices = append(prices, state.PricePoint{ + SlotTsMs: s.StartMs, + SlotLenMin: s.LenMin, + SpotOreKwh: s.SpotOre, + TotalOreKwh: s.PriceOre, + }) + } + switch mode { + case ModeSelfConsumption, ModeCheapCharge, ModePassiveArbitrage: + return selfConsumptionTerminalPrice(prices, 0, 0) + default: + return upperHalfMeanPrice(prices) + } +} + +type forecastVariant struct { + name string + plan *Plan + firstW float64 + corrected float64 + raw float64 + endSoC float64 + costErr error + solveMs int64 +} + +// TestForecastValueBench answers the owner's question with numbers: does the +// price twin change the watt dispatched now, and for how many hours does the +// guess keep mattering? Run with -v; the verdict is the table. +// +// Knobs: +// +// FTW_MPC_SNAPSHOT_DIR — directory of /api/mpc/diagnose blobs (required) +// FTW_MPC_FORECAST_SOC — SoC grid (default 201) +// FTW_MPC_FORECAST_ACTIONS — action grid (default 401) +// +// Both grids are forced so every variant answers on one resolution: a blob +// carries the recorded replan's grid, and the older blobs were recorded at a +// coarser one under a different planner. +func TestForecastValueBench(t *testing.T) { + dir := os.Getenv("FTW_MPC_SNAPSHOT_DIR") + if dir == "" { + t.Skip("FTW_MPC_SNAPSHOT_DIR not set") + } + paths, err := filepath.Glob(filepath.Join(dir, "*.json")) + if err != nil || len(paths) == 0 { + t.Fatalf("no snapshots in %q (err=%v)", dir, err) + } + sort.Strings(paths) + + socLevels, actionLevels := 201, 401 + if v := os.Getenv("FTW_MPC_FORECAST_SOC"); v != "" { + fmt.Sscanf(v, "%d", &socLevels) + } + if v := os.Getenv("FTW_MPC_FORECAST_ACTIONS"); v != "" { + fmt.Sscanf(v, "%d", &actionLevels) + } + t.Logf("grid forced to SoCLevels=%d ActionLevels=%d for every variant", socLevels, actionLevels) + t.Logf("A = as recorded; B = forecast slots flattened to the horizon mean (confidence %g);"+ + " C = forecast slots deleted, known window only", benchNoTrustConfidence) + + terminalScales := []float64{0.5, 1.0, 1.5} + + t.Logf("") + t.Logf("=== decision table: the first slot is the only number that reaches hardware ===") + t.Logf("%-22s %-18s %7s %9s %9s %9s %9s %9s %8s %8s", + "snapshot", "mode", "known_h", "rec_w", "A_w", "B_w", "C_w", "B-A_w", "C-A_w", "A_soc0") + + type row struct { + name string + live bool + mode Mode + knownH float64 + knownN int + recW, aW, bW, cW float64 + divABIdx, divACIdx int + divABOK, divACOK bool + divABH, divACH float64 + aCorr, bCorr, cCorr float64 + aRaw, bRaw, cRaw float64 + aSoC, bSoC, cSoC float64 + terminalOre, knownTermOre float64 + knownMeanOre float64 + sweepW map[float64]float64 + sweepCorr map[float64]float64 + sweepSoC map[float64]float64 + sweepDivIdx int + sweepDivOK bool + knownTermW, knownTermCorr float64 + knownTermSoC float64 + diffSlotsAB, diffSlotsAC int + costable bool + } + var rows []row + + for _, path := range paths { + name := filepath.Base(path) + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + d, err := loadDiagnosticBlob(data) + if err != nil { + t.Logf("%-22s SKIP: %v", name, err) + continue + } + recorded, slots, params, _, ok := planFromDiagnostic(d) + if !ok { + t.Logf("%-22s SKIP: not rehydratable", name) + continue + } + params.SoCLevels = socLevels + params.ActionLevels = actionLevels + + lastKnown, contiguous, knownCount := benchLastKnownSlot(slots) + if lastKnown < 0 { + t.Logf("%-22s SKIP: no confidence-1.0 slot, nothing is known", name) + continue + } + if !contiguous { + t.Logf("%-22s WARN: known slots are not a contiguous prefix (%d known, last at %d);"+ + " C truncates at the last one anyway", name, knownCount, lastKnown) + } + n := lastKnown + 1 + + // Scoring price never moves: every variant, every terminal scale, is + // costed with the terminal price the box actually derived. + scoreParams := params + + r := row{ + name: name, + live: name == "live-2026-08-31.json", + mode: params.Mode, + knownN: n, + knownH: benchHoursTo(slots, n), + recW: benchFirstSlotW(recorded), + terminalOre: params.TerminalSoCPrice, + sweepW: map[float64]float64{}, + sweepCorr: map[float64]float64{}, + sweepSoC: map[float64]float64{}, + costable: true, + } + { + var sum, w float64 + for _, s := range slots[:n] { + sum += s.PriceOre * float64(s.LenMin) + w += float64(s.LenMin) + } + if w > 0 { + r.knownMeanOre = sum / w + } + } + + solve := func(label string, in []Slot, p Params) forecastVariant { + start := time.Now() + plan := Optimize(in, p) + ms := time.Since(start).Milliseconds() + v := forecastVariant{name: label, plan: &plan, firstW: benchFirstSlotW(&plan), solveMs: ms} + v.corrected, v.raw, v.endSoC, v.costErr = benchKnownWindowCost(&plan, slots, n, p, scoreParams) + if v.costErr != nil { + t.Logf("%-22s %s: known-window cost refused: %v", name, label, v.costErr) + } + return v + } + + a := solve("A", slots, params) + b := solve("B", benchFlattenForecast(slots), params) + c := solve("C", slots[:n], params) + + r.aW, r.bW, r.cW = a.firstW, b.firstW, c.firstW + r.aCorr, r.bCorr, r.cCorr = a.corrected, b.corrected, c.corrected + r.aRaw, r.bRaw, r.cRaw = a.raw, b.raw, c.raw + r.aSoC, r.bSoC, r.cSoC = a.endSoC, b.endSoC, c.endSoC + if a.costErr != nil || b.costErr != nil || c.costErr != nil { + r.costable = false + } + r.divABIdx, r.divABOK = benchFirstDivergence(a.plan, b.plan) + r.divACIdx, r.divACOK = benchFirstDivergence(a.plan, c.plan) + r.divABH = benchHoursTo(slots, r.divABIdx) + r.divACH = benchHoursTo(slots, r.divACIdx) + // How much of the KNOWN window the variants disagree about at all. + // "The plans differ" and "the dispatched watt differs" are separate + // facts, and only the second one reaches hardware. + for i := 0; i < n; i++ { + if math.Abs(a.plan.Actions[i].BatteryW-b.plan.Actions[i].BatteryW) > benchActionDiffW { + r.diffSlotsAB++ + } + if math.Abs(a.plan.Actions[i].BatteryW-c.plan.Actions[i].BatteryW) > benchActionDiffW { + r.diffSlotsAC++ + } + } + + // Terminal sweep on C. Short horizon plus a terminal price above the + // window's own mean is the combination that would make C bank energy + // it has no measured reason to bank, so the end-of-window SoC is + // reported next to the first-slot watt: the terminal price can move + // the plan without moving the decision. + sweepPlans := map[float64]*Plan{} + for _, scale := range terminalScales { + ps := params + ps.TerminalSoCPrice = params.TerminalSoCPrice * scale + plan := Optimize(slots[:n], ps) + sweepPlans[scale] = &plan + r.sweepW[scale] = benchFirstSlotW(&plan) + corr, _, endSoC, err := benchKnownWindowCost(&plan, slots, n, ps, scoreParams) + if err == nil { + r.sweepCorr[scale] = corr + r.sweepSoC[scale] = endSoC + } else { + r.sweepCorr[scale] = math.NaN() + r.sweepSoC[scale] = math.NaN() + } + } + if lo, hi := sweepPlans[0.5], sweepPlans[1.5]; lo != nil && hi != nil { + r.sweepDivIdx, r.sweepDivOK = benchFirstDivergence(lo, hi) + } + // And C as a box that knows nothing past day-ahead would really run + // it: terminal price rederived from the known window alone. + r.knownTermOre = benchTerminalPrice(params.Mode, slots[:n]) + pk := params + pk.TerminalSoCPrice = r.knownTermOre + planK := Optimize(slots[:n], pk) + r.knownTermW = benchFirstSlotW(&planK) + if corr, _, endSoC, err := benchKnownWindowCost(&planK, slots, n, pk, scoreParams); err == nil { + r.knownTermCorr = corr + r.knownTermSoC = endSoC + } else { + r.knownTermCorr = math.NaN() + r.knownTermSoC = math.NaN() + } + + rows = append(rows, r) + t.Logf("%-22s %-18s %7.2f %9.1f %9.1f %9.1f %9.1f %9.1f %8.1f %8.4f", + r.name, string(r.mode), r.knownH, r.recW, r.aW, r.bW, r.cW, + r.bW-r.aW, r.cW-r.aW, params.InitialSoC) + } + if len(rows) == 0 { + t.Fatal("no snapshot produced a comparison") + } + + t.Logf("") + t.Logf("=== how long the guess keeps mattering (first slot where |ΔbatteryW| > %.0f W) ===", benchActionDiffW) + t.Logf("%-22s %10s %10s %10s %10s %12s %12s", "snapshot", + "A_vs_B_idx", "A_vs_B_h", "A_vs_C_idx", "A_vs_C_h", "AB_diff/known", "AC_diff/known") + for _, r := range rows { + ab, ac := fmt.Sprintf("%d", r.divABIdx), fmt.Sprintf("%d", r.divACIdx) + abh, ach := fmt.Sprintf("%.2f", r.divABH), fmt.Sprintf("%.2f", r.divACH) + if !r.divABOK { + ab, abh = "none", ">"+fmt.Sprintf("%.2f", r.divABH) + } + if !r.divACOK { + ac, ach = "none", ">"+fmt.Sprintf("%.2f", r.divACH) + } + t.Logf("%-22s %10s %10s %10s %10s %12s %12s", r.name, ab, abh, ac, ach, + fmt.Sprintf("%d/%d", r.diffSlotsAB, r.knownN), + fmt.Sprintf("%d/%d", r.diffSlotsAC, r.knownN)) + } + + t.Logf("") + t.Logf("=== cost over the KNOWN window only (%s), öre, terminal-corrected at the recorded price ===", + "real day-ahead slots, identical for all three") + t.Logf("%-22s %6s %10s %10s %10s %9s %9s %8s %8s %8s", + "snapshot", "known_n", "A_corr", "B_corr", "C_corr", "B-A", "C-A", "A_soc", "B_soc", "C_soc") + for _, r := range rows { + if !r.costable { + t.Logf("%-22s %6d REFUSED", r.name, r.knownN) + continue + } + t.Logf("%-22s %6d %10.1f %10.1f %10.1f %9.1f %9.1f %8.4f %8.4f %8.4f", + r.name, r.knownN, r.aCorr, r.bCorr, r.cCorr, + r.bCorr-r.aCorr, r.cCorr-r.aCorr, r.aSoC, r.bSoC, r.cSoC) + } + + t.Logf("") + t.Logf("=== C's dependence on the terminal price ===") + t.Logf("term_ore is the price the box derived from the FULL horizon; known_mean is the known window's own mean.") + t.Logf("knownT_* rederives the terminal price from the known slots alone, with Core's mode-dependent formula.") + t.Logf("%-22s %9s %9s %9s %9s %9s %9s %9s %9s %9s %8s", + "snapshot", "term_ore", "known_mean", "C@0.5_w", "C@1.0_w", "C@1.5_w", + "knownT_ore", "knownT_w", "A_w", "sweep_div", "C_w_sprd") + for _, r := range rows { + div := fmt.Sprintf("%d", r.sweepDivIdx) + if !r.sweepDivOK { + div = "none" + } + t.Logf("%-22s %9.1f %9.1f %9.1f %9.1f %9.1f %9.1f %9.1f %9.1f %9s %8.1f", + r.name, r.terminalOre, r.knownMeanOre, + r.sweepW[0.5], r.sweepW[1.0], r.sweepW[1.5], r.knownTermOre, r.knownTermW, r.aW, + div, math.Abs(r.sweepW[1.5]-r.sweepW[0.5])) + } + t.Logf("--- and what the terminal price DOES move: SoC parked at the end of the known window ---") + t.Logf("%-22s %9s %9s %9s %9s %11s %11s %11s", + "snapshot", "C@0.5_soc", "C@1.0_soc", "C@1.5_soc", "knownT_soc", "C@0.5_ore", "C@1.0_ore", "C@1.5_ore") + for _, r := range rows { + t.Logf("%-22s %9.4f %9.4f %9.4f %9.4f %11.1f %11.1f %11.1f", + r.name, r.sweepSoC[0.5], r.sweepSoC[1.0], r.sweepSoC[1.5], r.knownTermSoC, + r.sweepCorr[0.5], r.sweepCorr[1.0], r.sweepCorr[1.5]) + } + + summarize := func(label string, sel func(row) bool) { + var n, sameAB, sameAC, exactAB, exactAC int + var sumAbsAB, sumAbsAC, maxAbsAB, maxAbsAC float64 + var sumCostAB, sumCostAC float64 + var sumDivAB, sumDivAC float64 + var sumDiffAB, sumDiffAC, sumKnownN float64 + var sumSoCAC, sumSweepSoC float64 + var sweepSpread float64 + for _, r := range rows { + if !sel(r) { + continue + } + n++ + dAB, dAC := math.Abs(r.bW-r.aW), math.Abs(r.cW-r.aW) + sumAbsAB += dAB + sumAbsAC += dAC + maxAbsAB = math.Max(maxAbsAB, dAB) + maxAbsAC = math.Max(maxAbsAC, dAC) + if dAB <= benchActionDiffW { + sameAB++ + } + if dAC <= benchActionDiffW { + sameAC++ + } + if dAB < 1 { + exactAB++ + } + if dAC < 1 { + exactAC++ + } + sumCostAB += r.bCorr - r.aCorr + sumCostAC += r.cCorr - r.aCorr + sumDivAB += r.divABH + sumDivAC += r.divACH + sumDiffAB += float64(r.diffSlotsAB) + sumDiffAC += float64(r.diffSlotsAC) + sumKnownN += float64(r.knownN) + sumSoCAC += r.cSoC - r.aSoC + sumSweepSoC += math.Abs(r.sweepSoC[1.5] - r.sweepSoC[0.5]) + lo, hi := r.sweepW[0.5], r.sweepW[1.5] + sweepSpread = math.Max(sweepSpread, math.Abs(hi-lo)) + } + if n == 0 { + return + } + t.Logf("") + t.Logf("SUMMARY %s (n=%d)", label, n) + t.Logf(" first-slot W, B vs A: identical(<1W) %d/%d, within %.0f W %d/%d, mean |Δ| %.1f W, max |Δ| %.1f W", + exactAB, n, benchActionDiffW, sameAB, n, sumAbsAB/float64(n), maxAbsAB) + t.Logf(" first-slot W, C vs A: identical(<1W) %d/%d, within %.0f W %d/%d, mean |Δ| %.1f W, max |Δ| %.1f W", + exactAC, n, benchActionDiffW, sameAC, n, sumAbsAC/float64(n), maxAbsAC) + t.Logf(" mean hours before the plans part: A/B %.2f h, A/C %.2f h", sumDivAB/float64(n), sumDivAC/float64(n)) + t.Logf(" slots of the known window the plans disagree about: A/B %.1f of %.1f, A/C %.1f of %.1f", + sumDiffAB/float64(n), sumKnownN/float64(n), sumDiffAC/float64(n), sumKnownN/float64(n)) + t.Logf(" known-window cost, mean B−A %+.1f öre, mean C−A %+.1f öre (positive = the variant costs more)", + sumCostAB/float64(n), sumCostAC/float64(n)) + t.Logf(" SoC parked at the end of the known window, mean C−A %+.4f (%+.0f Wh on a 9.6 kWh pack)", + sumSoCAC/float64(n), sumSoCAC/float64(n)*9600) + t.Logf(" C first-slot W across terminal scales 0.5…1.5: worst spread %.1f W;"+ + " mean end-of-window SoC spread %.4f", sweepSpread, sumSweepSoC/float64(n)) + } + summarize("all snapshots", func(r row) bool { return true }) + summarize("live snapshot only (today's Core)", func(r row) bool { return r.live }) + summarize("older Python-era snapshots only", func(r row) bool { return !r.live }) + t.Logf("") + t.Logf("NOTE: no snapshot carries realised prices, so nothing above says which variant EARNS more." + + " It says only whether the guess changes the action.") +} + +// benchRelabelKnown returns a copy of slots where the first k count as real +// day-ahead and everything after is marked forecast at forecastConf. The +// PRICES do not move — only how far the box is told to trust them. +// +// That is the point: it isolates the confidence mechanism from forecast +// error. Whatever the twin would have got wrong is held at zero here, so any +// difference the sweep finds is caused by the blending rule alone, and a real +// twin with real error can only differ by more. +func benchRelabelKnown(slots []Slot, k int, forecastConf float64) []Slot { + out := make([]Slot, len(slots)) + copy(out, slots) + for i := range out { + if i < k { + out[i].Confidence = 1.0 + } else { + out[i].Confidence = forecastConf + } + } + return out +} + +// benchForecastConfidence is the confidence the box actually stamped on its +// guessed slots — 0.6 on these sites. Falls back to 0.6 when a snapshot has +// no forecast row to read it from. +func benchForecastConfidence(slots []Slot) float64 { + for _, s := range slots { + if s.Confidence > 0 && s.Confidence < 1.0 { + return s.Confidence + } + } + return 0.6 +} + +// TestForecastValueKnownWindowSweep asks the same question in the regime +// where the guess has the best chance of mattering: a SHORT known window. +// +// Every snapshot here happens to carry 11.75–13.5 h of real day-ahead price, +// which is the comfortable end of the daily cycle. Just before publication a +// box can be down to a few hours. The sweep shortens the known window by +// hand — 2, 4, 6, 8, 12 h and as recorded — and re-asks whether A (trust the +// guess as the box does), B (flatten it) and C (delete it) dispatch the same +// watt. +// +// Prices are left at their recorded values throughout, so this measures the +// blending rule, not the twin's accuracy. Skipped without +// FTW_MPC_SNAPSHOT_DIR. +func TestForecastValueKnownWindowSweep(t *testing.T) { + dir := os.Getenv("FTW_MPC_SNAPSHOT_DIR") + if dir == "" { + t.Skip("FTW_MPC_SNAPSHOT_DIR not set") + } + paths, err := filepath.Glob(filepath.Join(dir, "*.json")) + if err != nil || len(paths) == 0 { + t.Fatalf("no snapshots in %q (err=%v)", dir, err) + } + sort.Strings(paths) + + socLevels, actionLevels := 201, 401 + if v := os.Getenv("FTW_MPC_FORECAST_SOC"); v != "" { + fmt.Sscanf(v, "%d", &socLevels) + } + if v := os.Getenv("FTW_MPC_FORECAST_ACTIONS"); v != "" { + fmt.Sscanf(v, "%d", &actionLevels) + } + + // 15-minute slots on every snapshot here, so k is hours × 4. + windows := []int{8, 16, 24, 32, 48} + t.Logf("grid forced to SoCLevels=%d ActionLevels=%d", socLevels, actionLevels) + t.Logf("prices are unchanged at every k — only the trust label moves, so this isolates") + t.Logf("the confidence rule from forecast error") + t.Logf("%-22s %6s %7s %9s %9s %9s %9s %8s", "snapshot", "known_k", "known_h", "A_w", "B_w", "C_w", "B-A_w", "C-A_w") + + agree := map[int][2]int{} + total := map[int]int{} + for _, path := range paths { + name := filepath.Base(path) + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + d, err := loadDiagnosticBlob(data) + if err != nil { + continue + } + _, slots, params, _, ok := planFromDiagnostic(d) + if !ok { + continue + } + params.SoCLevels = socLevels + params.ActionLevels = actionLevels + fConf := benchForecastConfidence(slots) + recordedK, _, _ := benchLastKnownSlot(slots) + ks := append([]int{}, windows...) + if !slices.Contains(ks, recordedK+1) { + ks = append(ks, recordedK+1) + } + for _, k := range ks { + if k < 2 || k > len(slots) { + continue + } + aPlan := Optimize(benchRelabelKnown(slots, k, fConf), params) + bPlan := Optimize(benchRelabelKnown(slots, k, benchNoTrustConfidence), params) + cPlan := Optimize(slots[:k], params) + aW, bW, cW := benchFirstSlotW(&aPlan), benchFirstSlotW(&bPlan), benchFirstSlotW(&cPlan) + label := name + if k == recordedK+1 { + label = name + " (as recorded)" + } + t.Logf("%-22s %6d %7.2f %9.1f %9.1f %9.1f %9.1f %8.1f", + label, k, benchHoursTo(slots, k), aW, bW, cW, bW-aW, cW-aW) + counts := agree[k] + if math.Abs(bW-aW) <= benchActionDiffW { + counts[0]++ + } + if math.Abs(cW-aW) <= benchActionDiffW { + counts[1]++ + } + agree[k] = counts + total[k]++ + } + } + ks := make([]int, 0, len(total)) + for k := range total { + ks = append(ks, k) + } + sort.Ints(ks) + t.Logf("") + t.Logf("SUMMARY first-slot agreement within %.0f W, by assumed known window", benchActionDiffW) + t.Logf("%6s %7s %14s %14s", "known_k", "known_h", "B_matches_A", "C_matches_A") + for _, k := range ks { + t.Logf("%6d %7.2f %14s %14s", k, float64(k)*15/60, + fmt.Sprintf("%d/%d", agree[k][0], total[k]), + fmt.Sprintf("%d/%d", agree[k][1], total[k])) + } +} + +// ---- CI-runnable fixture tests (no external data) ---- + +// TestForecastBenchZeroConfidenceMeansTotalTrust pins the trap the bench's +// epsilon exists for: asking for zero confidence gets full confidence, so a +// "trust nothing" variant written with Confidence = 0 would silently measure +// the opposite of what it claims. +func TestForecastBenchZeroConfidenceMeansTotalTrust(t *testing.T) { + got := sanitizeOptimizeSlots([]Slot{{LenMin: 15, PriceOre: 100, Confidence: 0}}) + if len(got) != 1 || got[0].Confidence != 1.0 { + t.Fatalf("Confidence 0 sanitized to %v, want 1.0 (the coercion the bench works around)", got) + } + eps := sanitizeOptimizeSlots([]Slot{{LenMin: 15, PriceOre: 100, Confidence: benchNoTrustConfidence}}) + if len(eps) != 1 || eps[0].Confidence != benchNoTrustConfidence { + t.Fatalf("epsilon confidence did not survive sanitisation: %v", eps) + } +} + +// TestForecastBenchFlattenLeavesKnownSlotsAlone pins that variant B touches +// only the slots the box was unsure about. +func TestForecastBenchFlattenLeavesKnownSlotsAlone(t *testing.T) { + in := []Slot{ + {LenMin: 15, PriceOre: 100, Confidence: 1.0}, + {LenMin: 15, PriceOre: 300, Confidence: 0.6}, + {LenMin: 15, PriceOre: 200, Confidence: 1.0}, + } + out := benchFlattenForecast(in) + if out[0].Confidence != 1.0 || out[2].Confidence != 1.0 { + t.Fatalf("known slots were flattened: %v", out) + } + if out[1].Confidence != benchNoTrustConfidence { + t.Fatalf("forecast slot not flattened: %v", out[1]) + } + if in[1].Confidence != 0.6 { + t.Fatal("benchFlattenForecast mutated the caller's slice") + } +} + +// TestForecastBenchRelabelKnownMovesTrustNotPrice pins that the known-window +// sweep shortens what the box is told it knows without touching a single +// price — the property that lets the sweep isolate the confidence rule from +// the twin's accuracy. +func TestForecastBenchRelabelKnownMovesTrustNotPrice(t *testing.T) { + in := []Slot{ + {LenMin: 15, PriceOre: 100, Confidence: 1.0}, + {LenMin: 15, PriceOre: 200, Confidence: 1.0}, + {LenMin: 15, PriceOre: 300, Confidence: 1.0}, + } + out := benchRelabelKnown(in, 1, 0.6) + for i := range out { + if out[i].PriceOre != in[i].PriceOre { + t.Fatalf("slot %d price moved: %v -> %v", i, in[i].PriceOre, out[i].PriceOre) + } + } + if out[0].Confidence != 1.0 || out[1].Confidence != 0.6 || out[2].Confidence != 0.6 { + t.Fatalf("trust labels wrong: %v", []float64{out[0].Confidence, out[1].Confidence, out[2].Confidence}) + } + if in[1].Confidence != 1.0 { + t.Fatal("benchRelabelKnown mutated the caller's slice") + } + if got := benchForecastConfidence(out); got != 0.6 { + t.Fatalf("forecast confidence read back as %v, want 0.6", got) + } +} + +// TestForecastBenchKnownWindow pins the prefix detection and the hours +// conversion that turn a slot index into "the guess stops mattering after N +// hours". +func TestForecastBenchKnownWindow(t *testing.T) { + slots := []Slot{ + {LenMin: 15, Confidence: 1.0}, + {LenMin: 15, Confidence: 1.0}, + {LenMin: 15, Confidence: 0.6}, + {LenMin: 15, Confidence: 0.6}, + } + last, contiguous, count := benchLastKnownSlot(slots) + if last != 1 || !contiguous || count != 2 { + t.Fatalf("last=%d contiguous=%v count=%d, want 1/true/2", last, contiguous, count) + } + if h := benchHoursTo(slots, 2); h != 0.5 { + t.Fatalf("hours to slot 2 = %v, want 0.5", h) + } + slots[3].Confidence = 1.0 + if _, contiguous, _ := benchLastKnownSlot(slots); contiguous { + t.Fatal("a hole in the known window must not report contiguous") + } +} + +// TestForecastBenchFlatForecastRemovesArbitrageBeyondKnown is the property +// the whole bench rests on: flattening a forecast slot's confidence removes +// the price SHAPE the DP could arbitrage against, so a plan that only charges +// because of a guessed evening peak stops charging. +func TestForecastBenchFlatForecastRemovesArbitrageBeyondKnown(t *testing.T) { + // Known window: four flat, unremarkable slots. Guessed window: a deep + // cheap trough then an expensive peak — pure invented arbitrage. + slots := make([]Slot, 12) + for i := range slots { + price := 200.0 + conf := 1.0 + switch { + case i >= 4 && i < 8: + price, conf = 20, 0.6 + case i >= 8: + price, conf = 600, 0.6 + } + slots[i] = Slot{ + StartMs: 1_700_000_000_000 + int64(i)*3_600_000, LenMin: 60, + PriceOre: price, SpotOre: price / 2, Confidence: conf, LoadW: 1000, + } + } + p := Params{ + Mode: ModeArbitrage, InitialSoC: 0.5, SoCMin: 0.1, SoCMax: 1.0, + SoCLevels: 41, ActionLevels: 41, MaxChargeW: 3000, MaxDischargeW: 3000, + ChargeEfficiency: 0.95, DischargeEfficiency: 0.95, CapacityWh: 9600, + TerminalSoCPrice: 200, + } + trusted := Optimize(slots, p) + flat := Optimize(benchFlattenForecast(slots), p) + // Under the guess, the cheap trough is worth charging into; flattened, + // those four slots all price at the horizon mean and there is nothing to + // arbitrage, so the trough charging must weaken. + var trustedTrough, flatTrough float64 + for i := 4; i < 8; i++ { + trustedTrough += trusted.Actions[i].BatteryW + flatTrough += flat.Actions[i].BatteryW + } + if trustedTrough <= flatTrough { + t.Fatalf("flattening did not remove the invented arbitrage: trusted charges %.0f W, flat %.0f W", + trustedTrough, flatTrough) + } +} diff --git a/go/internal/mpc/mpc.go b/go/internal/mpc/mpc.go index 9f1ae7c1a..b04a3c632 100644 --- a/go/internal/mpc/mpc.go +++ b/go/internal/mpc/mpc.go @@ -404,6 +404,30 @@ type ShadowPlan struct { ActiveTerminalCorrectedOre float64 `json:"active_terminal_corrected_ore,omitempty"` TerminalCorrectedOre float64 `json:"terminal_corrected_ore,omitempty"` ActiveMinusShadowTerminalCorrectedOre float64 `json:"active_minus_shadow_terminal_corrected_ore,omitempty"` + + // SelfReportedOre is the grid cost the challenger claimed for its own + // plan, kept beside Core's valuation of it rather than in place of it. + // The two agree only as long as the challenger prices the meter the way + // Core does; the gap is the diagnostic. + SelfReportedOre float64 `json:"self_reported_ore,omitempty"` + // SelfReportedObjectiveOre is the value the challenger actually + // minimized — scenario-weighted, CVaR-shaped, carrying its own + // penalties. It is NOT comparable with any cost above and exists to + // show how differently the two objectives are shaped. + SelfReportedObjectiveOre float64 `json:"self_reported_objective_ore,omitempty"` + // EvaluationRefusedReason is set when Core declined to cost one of the + // plans. The comparison fields are then left empty: a refusal that gets + // recorded beats a plausible number nobody can tell from a measurement. + EvaluationRefusedReason string `json:"evaluation_refused_reason,omitempty"` + // ActiveEvaluationDriftOre is Core's valuation of its OWN plan minus the + // cost that plan reported. Anything but ~0 is a bug in Core's + // bookkeeping, not a property of the challenger. + ActiveEvaluationDriftOre float64 `json:"active_evaluation_drift_ore,omitempty"` + // PV curtailment counts. Core's cost model prices every slot at full + // forecast generation on both sides, so slots either plan wants to + // curtail are slots where the comparison is blind to a real difference. + ActivePVCurtailmentSlots int `json:"active_pv_curtailment_slots,omitempty"` + PVCurtailmentSlots int `json:"pv_curtailment_slots,omitempty"` } // terminalCorrectedOre nets the terminal-SoC credit out of a raw grid cost so @@ -493,6 +517,48 @@ func SlotExportPriceOre(slot Slot, p Params) float64 { return gridcost.ExportPriceOre(slot.SpotOre, exportPricingFromParams(p)) } +// planSlotOutcome is one slot of a forward walk: the SoC the battery +// reaches, the power the meter sees, and what that flow costs under Core's +// cost model. +type planSlotOutcome struct { + SoC float64 + GridW float64 + CostOre float64 +} + +// stepPlanSlot advances one slot from socIn under a battery and loadpoint +// power. The DP's own forward pass and evaluatePlan both drive it, so the +// cost arithmetic cannot be changed for one and not the other. That sharing +// is the point: a foreign plan can only be judged on Core's terms while +// "Core's terms" is a single piece of code, and a second copy of these three +// lines would silently invalidate every comparison the day one of them moved. +// +// SoC comes back unclamped. The DP pins it to the operating band afterwards +// because its feasibility screen already rejected out-of-band transitions; +// evaluatePlan needs to see the raw excursion, because a foreign plan has had +// no such screen. +func stepPlanSlot(slot Slot, p Params, socIn, batteryW, loadpointW float64) planSlotOutcome { + dtH := float64(slot.LenMin) / 60.0 + var deltaWh float64 + if batteryW >= 0 { + deltaWh = +batteryW * dtH * p.ChargeEfficiency + } else { + deltaWh = +batteryW * dtH / p.DischargeEfficiency + } + gridW := slot.LoadW + slot.PVW + batteryW + loadpointW + return planSlotOutcome{ + SoC: socIn + deltaWh/p.CapacityWh, + GridW: gridW, + CostOre: SlotGridCostOre(slot, gridW*dtH/1000.0, p), + } +} + +// stepLoadpointSoC advances one flex load over a slot. Charging only, so the +// result never falls; the caller enforces the ceiling. +func stepLoadpointSoC(socIn, powerW, dtH, capacityWh, chargeEfficiency float64) float64 { + return socIn + powerW*dtH*chargeEfficiency/capacityWh +} + func exportPricingFromParams(p Params) gridcost.ExportPricing { return gridcost.ExportPricing{ BonusOreKwh: p.ExportBonusOreKwh, @@ -1077,14 +1143,12 @@ func Optimize(slots []Slot, p Params) Plan { ea := pol % EA actW := actionAt(ba) evW := evActionW(ea) - // Battery SoC transition. - var dSoCWh float64 - if actW >= 0 { - dSoCWh = +actW * dtH * p.ChargeEfficiency - } else { - dSoCWh = +actW * dtH / p.DischargeEfficiency - } - soc2 := soc + dSoCWh/p.CapacityWh + // Battery SoC transition and slot cost. Report the ACTUAL + // expected cost using the raw (un-blended) prices so the UI + // summary reflects "what we'd actually pay if prices hold". + // Blending is a decision lens only. + step := stepPlanSlot(slot, p, soc, actW, evW) + soc2 := step.SoC if soc2 < p.SoCMin { soc2 = p.SoCMin } @@ -1094,18 +1158,13 @@ func Optimize(slots []Slot, p Params) Plan { // EV SoC transition (no-op when !evActive since evW = 0). var evSoc2 float64 if evActive { - dEvWh := evW * dtH * evChargeEff - evSoc2 = evSoc + dEvWh/lp.CapacityWh + evSoc2 = stepLoadpointSoC(evSoc, evW, dtH, lp.CapacityWh, evChargeEff) if evSoc2 > lp.SoCMax { evSoc2 = lp.SoCMax } } - gridW := slot.LoadW + slot.PVW + actW + evW - gridKWh := gridW * dtH / 1000.0 - // Report the ACTUAL expected cost using the raw (un-blended) - // prices so the UI summary reflects "what we'd actually pay - // if prices hold". Blending is a decision lens only. - cost := SlotGridCostOre(slot, gridKWh, p) + gridW := step.GridW + cost := step.CostOre totalCost += cost a := Action{ SlotStartMs: slot.StartMs, diff --git a/go/internal/mpc/plan_evaluation.go b/go/internal/mpc/plan_evaluation.go new file mode 100644 index 000000000..b45b279d7 --- /dev/null +++ b/go/internal/mpc/plan_evaluation.go @@ -0,0 +1,211 @@ +package mpc + +import ( + "errors" + "fmt" + "math" +) + +// Costing a foreign plan is not the same thing as reading the number that +// plan came with. A solver reports the value of ITS objective — scenario +// weights, a CVaR tail term, its own penalties and bonuses — and subtracting +// that from Core's cost measures the two objectives rather than the two +// plans. evaluatePlan closes that gap: it walks somebody else's action +// sequence through Core's own forward pass and reports what Core says it +// costs, so both sides of a comparison are priced by one piece of code. +// +// Refusal is a first-class result. A plan that cannot be costed honestly — +// wrong length, non-finite output, state outside the operating band Core +// enforces — yields no number at all, because a logged refusal is worth more +// than a plausible fabrication nobody can tell apart from a measurement. + +// planEvaluation is Core's valuation of a plan's actions. +type planEvaluation struct { + // CostOre is the raw grid cost over the horizon — the same quantity + // Plan.TotalCostOre carries for a Core plan. Terminal-correct it before + // comparing two plans that park the horizon at different SoC. + CostOre float64 + // EndSoC is the SoC the walk reached, not the SoC the plan claimed. + // Trusting a foreign plan's own SoC column would put the terminal + // credit back on the challenger's honour system. + EndSoC float64 + // CurtailedSlots counts slots whose action asks to cap PV. Core's cost + // model prices every slot at full forecast generation, for its own plans + // and foreign ones alike, so this is not a mispricing — it is the count + // of slots where the two plans could still differ in a way the model + // does not resolve. + CurtailedSlots int +} + +var errPlanEvaluationCapacity = errors.New("capacity_wh is not positive") + +// evaluatePlanOre reports what a plan's actions cost under Core's cost +// model, so a foreign plan can be judged on the same terms as Core's own. +// ok is false when the plan cannot be costed; evaluatePlan carries the +// reason for callers that log it. +func evaluatePlanOre(plan Plan, slots []Slot, p Params) (ore float64, ok bool) { + eval, err := evaluatePlan(plan, slots, p) + if err != nil { + return 0, false + } + return eval.CostOre, true +} + +// evaluatePlan walks plan.Actions across slots under Core's cost model and +// returns Core's valuation, or the reason it refuses to produce one. +// +// Storage is walked in aggregate — one SoC against p.CapacityWh and the +// aggregate efficiencies — because that is the state the champion plans in +// and the state its own reported cost describes. Cost never reads SoC, so a +// fleet of batteries with unequal efficiencies can only move the terminal +// correction and the band check, never the öre per slot. ValidatePlan +// already replays a foreign plan per storage on the way in. +func evaluatePlan(plan Plan, slots []Slot, p Params) (planEvaluation, error) { + // Optimize sanitizes and defaults before it costs anything. Judging a + // plan under other inputs than the ones Core would have used is not + // judging it under Core's model. + slots = sanitizeOptimizeSlots(slots) + if len(slots) == 0 { + return planEvaluation{}, errors.New("no slots to evaluate against") + } + if p.CapacityWh <= 0 { + return planEvaluation{}, errPlanEvaluationCapacity + } + if len(plan.Actions) != len(slots) { + return planEvaluation{}, fmt.Errorf("action count %d, want %d", + len(plan.Actions), len(slots)) + } + if p.ChargeEfficiency <= 0 { + p.ChargeEfficiency = 0.95 + } + if p.DischargeEfficiency <= 0 { + p.DischargeEfficiency = 0.95 + } + + bandActive := p.SoCMax > p.SoCMin + bandTol := socBandTolerance(p) + soc := p.InitialSoC + if bandActive { + // Same clamp the DP's forward pass opens with: the site may report + // a SoC outside the band, and Core plans from the band edge. + soc = math.Max(p.SoCMin, math.Min(p.SoCMax, soc)) + } + + loadpoints := p.activeLoadpoints() + lpSoC := make(map[string]float64, len(loadpoints)) + for _, lp := range loadpoints { + lpSoC[lp.ID] = lp.InitialSoC + } + + eval := planEvaluation{EndSoC: soc} + for i, slot := range slots { + a := plan.Actions[i] + if a.SlotStartMs != slot.StartMs || a.SlotLenMin != slot.LenMin { + return planEvaluation{}, fmt.Errorf("slot %d is not the slot the action was planned for", i) + } + loadpointW := 0.0 + for idx := range loadpoints { + powerW := evaluationLoadpointStepW(a, loadpoints, idx) + if !finite(powerW) { + return planEvaluation{}, fmt.Errorf("slot %d loadpoint %s has non-finite power", + i, loadpoints[idx].ID) + } + loadpointW += powerW + } + if !finite(a.BatteryW) || !finite(a.PVLimitW) { + return planEvaluation{}, fmt.Errorf("slot %d contains non-finite output", i) + } + if a.BatteryW > p.MaxChargeW+planEvaluationPowerTolW || + a.BatteryW < -p.MaxDischargeW-planEvaluationPowerTolW { + return planEvaluation{}, fmt.Errorf( + "slot %d battery_w %.1f outside charge/discharge limits %.1f…%.1f", + i, a.BatteryW, -p.MaxDischargeW, p.MaxChargeW) + } + + step := stepPlanSlot(slot, p, soc, a.BatteryW, loadpointW) + if !finite(step.SoC) || !finite(step.CostOre) { + return planEvaluation{}, fmt.Errorf("slot %d costs to a non-finite value", i) + } + if bandActive { + if step.SoC < p.SoCMin-bandTol || step.SoC > p.SoCMax+bandTol { + return planEvaluation{}, fmt.Errorf( + "slot %d drives soc to %.4f, outside %.4f…%.4f", i, step.SoC, p.SoCMin, p.SoCMax) + } + // The DP pins its own walk to the band; matching it keeps a Core + // plan's evaluation identical to the cost the DP reported. + soc = math.Max(p.SoCMin, math.Min(p.SoCMax, step.SoC)) + } else { + soc = step.SoC + } + for idx, lp := range loadpoints { + ceiling := lp.SoCMax + if ceiling <= lp.SoCMin { + ceiling = 1.0 // same normalization the DP applies + } + eff := lp.ChargeEfficiency + if eff <= 0 { + eff = 0.9 + } + next := stepLoadpointSoC(lpSoC[lp.ID], evaluationLoadpointStepW(a, loadpoints, idx), + float64(slot.LenMin)/60.0, lp.CapacityWh, eff) + if !finite(next) { + return planEvaluation{}, fmt.Errorf("slot %d loadpoint %s reaches a non-finite soc", i, lp.ID) + } + if next > ceiling+planEvaluationLoadpointSoCTol { + return planEvaluation{}, fmt.Errorf( + "slot %d drives loadpoint %s to soc %.4f, above %.4f", i, lp.ID, next, ceiling) + } + // The DP pins the EV walk to its ceiling too. + lpSoC[lp.ID] = math.Min(next, ceiling) + } + + eval.CostOre += step.CostOre + if a.PVLimitW > 0 { + eval.CurtailedSlots++ + } + } + eval.EndSoC = soc + if !finite(eval.CostOre) { + return planEvaluation{}, errors.New("horizon cost is not finite") + } + return eval, nil +} + +const ( + // planEvaluationPowerTolW matches the slack ValidatePlan already grants a + // continuous solver riding a bound: float residue lands a plan a + // fraction of a watt over its own limit, and rejecting the measurement + // for that would only hide it. + planEvaluationPowerTolW = 2.0 + // planEvaluationLoadpointSoCTol mirrors ValidatePlan's EV replay + // tolerance. + planEvaluationLoadpointSoCTol = 0.0005 +) + +// socBandTolerance is how far past soc_min…soc_max one slot may land before +// the walk refuses. The DP looks its policy up on a discretized SoC grid +// while propagating a continuous SoC, so a legitimate Core plan can overshoot +// by up to a grid step; a plan that leaves the band by more than the DP's own +// resolution allows is asking for state the band does not permit. +func socBandTolerance(p Params) float64 { + levels := p.SoCLevels + if levels < 3 { + levels = 3 + } + step := (p.SoCMax - p.SoCMin) / float64(levels-1) + return math.Max(0.001, step) +} + +// evaluationLoadpointStepW reads one action's power for one active flex +// load, using the same per-ID map with a first-loadpoint scalar fallback +// ValidatePlan reads — the DP writes only the scalar, the external optimizer +// writes both. +func evaluationLoadpointStepW(a Action, loadpoints []*LoadpointSpec, idx int) float64 { + if len(a.LoadpointPowerW) == 0 { + if idx == 0 { + return a.LoadpointW + } + return 0 + } + return a.LoadpointPowerW[loadpoints[idx].ID] +} diff --git a/go/internal/mpc/plan_evaluation_test.go b/go/internal/mpc/plan_evaluation_test.go new file mode 100644 index 000000000..474d33bdc --- /dev/null +++ b/go/internal/mpc/plan_evaluation_test.go @@ -0,0 +1,315 @@ +package mpc + +import ( + "math" + "strings" + "testing" +) + +// evaluationSlots builds a cheap-then-expensive horizon: enough spread that +// every mode below actually moves the battery. +func evaluationSlots(n int) []Slot { + slots := make([]Slot, n) + for i := range slots { + price := 80.0 + pv := 0.0 + if i >= n/2 { + price = 320.0 + } + if i >= n/4 && i < n/2 { + pv = -2500 // midday surplus + } + slots[i] = Slot{ + StartMs: 1_700_000_000_000 + int64(i)*900_000, + LenMin: 15, + PriceOre: price, + SpotOre: price / 2, + Confidence: 1, + PVW: pv, + LoadW: 900, + } + } + return slots +} + +func evaluationParams(mode Mode) Params { + return Params{ + Mode: mode, CapacityWh: 9600, InitialSoC: 0.5, + SoCMin: 0.1, SoCMax: 1.0, SoCLevels: 61, ActionLevels: 41, + MaxChargeW: 4000, MaxDischargeW: 4000, + ChargeEfficiency: 0.95, DischargeEfficiency: 0.95, + TerminalSoCPrice: 150, + } +} + +// TestEvaluatePlanCostsTheDPsOwnPlanIdentically is the sharing proof. The DP +// reports a total; the evaluator re-derives it from the same actions. They +// agree only while both drive the one piece of cost arithmetic — fork it and +// this test is what fails, before a measurement quietly stops meaning +// anything. +func TestEvaluatePlanCostsTheDPsOwnPlanIdentically(t *testing.T) { + cases := map[string]func() (Params, []Slot){ + "arbitrage": func() (Params, []Slot) { + return evaluationParams(ModeArbitrage), evaluationSlots(24) + }, + "passive arbitrage": func() (Params, []Slot) { + return evaluationParams(ModePassiveArbitrage), evaluationSlots(24) + }, + "self consumption": func() (Params, []Slot) { + return evaluationParams(ModeSelfConsumption), evaluationSlots(24) + }, + "arbitrage with a minimum spread": func() (Params, []Slot) { + p := evaluationParams(ModeArbitrage) + p.MinArbitrageSpreadOreKwh = 30 + return p, evaluationSlots(24) + }, + "arbitrage starting at the floor": func() (Params, []Slot) { + p := evaluationParams(ModeArbitrage) + p.InitialSoC = p.SoCMin + return p, evaluationSlots(24) + }, + "arbitrage starting below the floor": func() (Params, []Slot) { + // The DP plans from the clamped band edge; so must the + // evaluator, or every recovery replan would be unscoreable. + p := evaluationParams(ModeArbitrage) + p.InitialSoC = 0.05 + return p, evaluationSlots(24) + }, + "with an active loadpoint": func() (Params, []Slot) { + p := evaluationParams(ModeArbitrage) + p.Loadpoint = &LoadpointSpec{ + ID: "ev", CapacityWh: 60000, InitialSoC: 0.3, SoCMin: 0, + SoCMax: 0.9, TargetSoC: 0.8, TargetSlotIdx: 20, Levels: 11, + MaxChargeW: 11000, ChargeEfficiency: 0.9, + } + return p, evaluationSlots(24) + }, + "loadpoint on surplus only": func() (Params, []Slot) { + p := evaluationParams(ModePassiveArbitrage) + p.Loadpoint = &LoadpointSpec{ + ID: "ev", CapacityWh: 60000, InitialSoC: 0.3, SoCMin: 0, + SoCMax: 0.9, TargetSoC: 0.8, TargetSlotIdx: 20, Levels: 11, + MaxChargeW: 11000, ChargeEfficiency: 0.9, SurplusOnly: true, + } + return p, evaluationSlots(24) + }, + "export priced flat": func() (Params, []Slot) { + p := evaluationParams(ModeArbitrage) + p.ExportOrePerKWh = 60 + return p, evaluationSlots(24) + }, + "fifteen and sixty minute slots mixed": func() (Params, []Slot) { + slots := evaluationSlots(24) + for i := range slots { + if i%3 == 0 { + slots[i].LenMin = 60 + } + } + return evaluationParams(ModeArbitrage), slots + }, + } + for name, build := range cases { + t.Run(name, func(t *testing.T) { + p, slots := build() + plan := Optimize(slots, p) + if len(plan.Actions) == 0 { + t.Fatal("the DP produced no plan to evaluate") + } + ore, ok := evaluatePlanOre(plan, slots, p) + if !ok { + _, err := evaluatePlan(plan, slots, p) + t.Fatalf("core refused to cost its own plan: %v", err) + } + if diff := math.Abs(ore - plan.TotalCostOre); diff > 1e-9 { + t.Fatalf("evaluated %.9f öre, the DP reported %.9f (%.9f apart)", + ore, plan.TotalCostOre, diff) + } + // The end SoC drives the terminal correction, so it has to come + // from the same walk rather than the plan's own column. + eval, err := evaluatePlan(plan, slots, p) + if err != nil { + t.Fatal(err) + } + if diff := math.Abs(eval.EndSoC - planEndSoC(&plan)); diff > 1e-9 { + t.Fatalf("walked end soc %.9f, plan reported %.9f", eval.EndSoC, planEndSoC(&plan)) + } + }) + } +} + +// TestEvaluatePlanRefusesRatherThanGuessing — every refusal here is a plan +// Core cannot honestly price. A number would be indistinguishable from a +// measurement; the absence of one is not. +func TestEvaluatePlanRefusesRatherThanGuessing(t *testing.T) { + base := evaluationParams(ModeArbitrage) + slots := evaluationSlots(12) + good := Optimize(slots, base) + + cases := map[string]struct { + mutate func(Plan) Plan + params func(Params) Params + reason string + }{ + "one action short": { + mutate: func(p Plan) Plan { p.Actions = p.Actions[:len(p.Actions)-1]; return p }, + reason: "action count", + }, + "one action too many": { + mutate: func(p Plan) Plan { p.Actions = append(p.Actions, p.Actions[0]); return p }, + reason: "action count", + }, + "a NaN battery action": { + mutate: func(p Plan) Plan { p.Actions[3].BatteryW = math.NaN(); return p }, + reason: "non-finite", + }, + "an infinite battery action": { + mutate: func(p Plan) Plan { p.Actions[3].BatteryW = math.Inf(1); return p }, + reason: "non-finite", + }, + "discharging straight through the floor": { + mutate: func(p Plan) Plan { + for i := range p.Actions { + p.Actions[i].BatteryW = -4000 + } + return p + }, + reason: "outside", + }, + "charging straight through the ceiling": { + mutate: func(p Plan) Plan { + for i := range p.Actions { + p.Actions[i].BatteryW = 4000 + } + return p + }, + reason: "outside", + }, + "a battery action beyond the discharge limit": { + mutate: func(p Plan) Plan { p.Actions[2].BatteryW = -9000; return p }, + reason: "charge/discharge limits", + }, + "a battery action beyond the charge limit": { + mutate: func(p Plan) Plan { p.Actions[2].BatteryW = 9000; return p }, + reason: "charge/discharge limits", + }, + "actions planned for other slots": { + mutate: func(p Plan) Plan { p.Actions[5].SlotStartMs += 1000; return p }, + reason: "not the slot", + }, + "no battery to evaluate": { + params: func(p Params) Params { p.CapacityWh = 0; return p }, + reason: "capacity_wh", + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + plan := clonePlan(&good) + if tc.mutate != nil { + *plan = tc.mutate(*plan) + } + p := base + if tc.params != nil { + p = tc.params(p) + } + if _, ok := evaluatePlanOre(*plan, slots, p); ok { + t.Fatal("core costed a plan it should have refused") + } + _, err := evaluatePlan(*plan, slots, p) + if err == nil { + t.Fatal("refused without a reason") + } + if !strings.Contains(err.Error(), tc.reason) { + t.Fatalf("refusal reason %q does not mention %q", err, tc.reason) + } + }) + } + + // The unmutated plan still evaluates — otherwise the table above proves + // nothing about the mutations. + if _, ok := evaluatePlanOre(good, slots, base); !ok { + t.Fatal("the control plan was refused too") + } +} + +// TestEvaluatePlanRanksPlans — the evaluator has to order plans, not merely +// reproduce one number. Two hand-built action sequences over the same two +// slots: buy cheap and sell dear, or the reverse. +func TestEvaluatePlanRanksPlans(t *testing.T) { + slots := []Slot{ + {StartMs: 1_000_000, LenMin: 60, PriceOre: 100, SpotOre: 100, Confidence: 1, LoadW: 1000}, + {StartMs: 1_000_000 + 3_600_000, LenMin: 60, PriceOre: 400, SpotOre: 400, Confidence: 1, LoadW: 1000}, + } + p := Params{ + Mode: ModeArbitrage, CapacityWh: 10000, InitialSoC: 0.5, + SoCMin: 0.1, SoCMax: 1.0, SoCLevels: 101, ActionLevels: 41, + MaxChargeW: 2000, MaxDischargeW: 2000, + ChargeEfficiency: 1, DischargeEfficiency: 1, + } + build := func(first, second float64) Plan { + return Plan{Actions: []Action{ + {SlotStartMs: slots[0].StartMs, SlotLenMin: 60, BatteryW: first}, + {SlotStartMs: slots[1].StartMs, SlotLenMin: 60, BatteryW: second}, + }} + } + + // Charge 2 kW at 100 öre, discharge 2 kW at 400 öre. + // slot 0: grid = 1000 + 2000 = 3 kWh × 100 = 300 öre + // slot 1: grid = 1000 − 2000 = −1 kWh × 400 = −400 öre + smart, ok := evaluatePlanOre(build(2000, -2000), slots, p) + if !ok { + t.Fatal("the sensible plan was refused") + } + if math.Abs(smart-(-100)) > 1e-9 { + t.Fatalf("buy-low-sell-high costs %.6f öre, want −100", smart) + } + + // The same energy moved the wrong way round. + // slot 0: grid = 1000 − 2000 = −1 kWh × 100 = −100 öre + // slot 1: grid = 1000 + 2000 = 3 kWh × 400 = 1200 öre + silly, ok := evaluatePlanOre(build(-2000, 2000), slots, p) + if !ok { + t.Fatal("the wasteful plan was refused") + } + if math.Abs(silly-1100) > 1e-9 { + t.Fatalf("sell-low-buy-high costs %.6f öre, want 1100", silly) + } + if silly <= smart { + t.Fatalf("the wasteful plan (%v) did not cost more than the sensible one (%v)", silly, smart) + } + + // Doing nothing sits between them: 2 kWh of house load, 100 + 400. + idle, ok := evaluatePlanOre(build(0, 0), slots, p) + if !ok { + t.Fatal("the idle plan was refused") + } + if math.Abs(idle-500) > 1e-9 { + t.Fatalf("idling costs %.6f öre, want 500", idle) + } + if !(smart < idle && idle < silly) { + t.Fatalf("ranking broken: smart=%v idle=%v silly=%v", smart, idle, silly) + } +} + +// TestEvaluatePlanIgnoresTheCostAPlanClaims — the whole point. A challenger +// that reports a flattering total gets no credit for it. +func TestEvaluatePlanIgnoresTheCostAPlanClaims(t *testing.T) { + p := evaluationParams(ModeArbitrage) + slots := evaluationSlots(16) + plan := Optimize(slots, p) + honest, ok := evaluatePlanOre(plan, slots, p) + if !ok { + t.Fatal("refused") + } + plan.TotalCostOre = -999999 + for i := range plan.Actions { + plan.Actions[i].CostOre = -1 + } + plan.Actions[len(plan.Actions)-1].SoC = 1.0 + flattered, ok := evaluatePlanOre(plan, slots, p) + if !ok { + t.Fatal("refused after the totals were rewritten") + } + if flattered != honest { + t.Fatalf("rewriting the reported cost moved the evaluation: %v then %v", honest, flattered) + } +} diff --git a/go/internal/mpc/python_shadow.go b/go/internal/mpc/python_shadow.go index c5a7a4529..75e3b4919 100644 --- a/go/internal/mpc/python_shadow.go +++ b/go/internal/mpc/python_shadow.go @@ -4,6 +4,7 @@ import ( "context" "errors" "log/slog" + "math" "time" ) @@ -34,8 +35,24 @@ const ( // shadowErrWindowLimit bounds the suppression map. Errors carrying a // request id or timestamp would otherwise make every message distinct. shadowErrWindowLimit = 32 + // championEvalDriftOre is how far Core's valuation of its own plan may + // sit from the cost that plan reported before it counts as a bug rather + // than float residue. A 192-slot horizon accumulates far less than this. + championEvalDriftOre = 0.5 ) +// shadowRefusalReason names which side Core declined to cost, and why. +func shadowRefusalReason(championErr, shadowErr error) string { + switch { + case championErr != nil && shadowErr != nil: + return "core plan: " + championErr.Error() + "; python plan: " + shadowErr.Error() + case championErr != nil: + return "core plan: " + championErr.Error() + default: + return "python plan: " + shadowErr.Error() + } +} + type shadowErrWindow struct { openedAt time.Time suppressed int @@ -98,34 +115,75 @@ func (s *Service) runPythonShadow(ctx context.Context, cancel context.CancelFunc block := compareDPShadow(champion, shadow) block.ForecastBasis = "same downside input, python challenger" block.Solver = shadow.Solver - block.TotalCostOre = shadow.TotalCostOre - block.ActiveMinusShadowOre = champion.TotalCostOre - shadow.TotalCostOre + block.SelfReportedOre = shadow.TotalCostOre + if shadow.Solver != nil { + block.SelfReportedObjectiveOre = shadow.Solver.ObjectiveOre + } if block.FirstAction != nil { mode, _, _ := actionToSlot(*block.FirstAction, p.Mode) block.FirstAction.EMSMode = mode } - // Raw totals do not compare: a plan that ends the horizon fuller looks - // expensive while it is merely storing value. Correct both sides before - // the difference is written anywhere a human will read it. - championOre := terminalCorrectedOre(champion.TotalCostOre, planEndSoC(&champion), p) - shadowOre := terminalCorrectedOre(shadow.TotalCostOre, planEndSoC(&shadow), p) - block.ActiveTerminalCorrectedOre = championOre - block.TerminalCorrectedOre = shadowOre - block.ActiveMinusShadowTerminalCorrectedOre = championOre - shadowOre if block.Solver != nil && block.Solver.SolveMs == 0 { block.Solver.SolveMs = solveMs } - slog.Info("mpc: core champion vs python shadow", + // Both plans are costed by Core, not by whoever produced them. Doing it + // for the champion too is not ceremony: it keeps the subtraction + // symmetric, so the verdict cannot drift if the DP's own bookkeeping + // ever changes, and it is the only way the cross-check below exists. + championEval, championErr := evaluatePlan(champion, slots, p) + shadowEval, shadowErr := evaluatePlan(shadow, slots, p) + + base := []any{ "decision_id", champion.DecisionID, "reason", reason, - "core_cost_ore", champion.TotalCostOre, - "python_cost_ore", shadow.TotalCostOre, - "python_minus_core_ore_terminal_corrected", shadowOre-championOre, + "python_self_reported_ore", shadow.TotalCostOre, + "python_objective_ore", block.SelfReportedObjectiveOre, "python_solve_ms", solveMs, "mean_abs_battery_delta_w", block.MeanAbsBatteryDeltaW, "direction_disagreements", block.DirectionDisagreements, - "compared_slots", block.ComparedSlots) + "compared_slots", block.ComparedSlots, + } + if championErr != nil || shadowErr != nil { + block.EvaluationRefusedReason = shadowRefusalReason(championErr, shadowErr) + slog.Warn("mpc: core champion vs python shadow not scored", + append(base, "evaluation_refused", block.EvaluationRefusedReason)...) + s.recordPythonShadow(champion, slots, p, reason, replanAtMs, block) + return + } + + block.ActiveEvaluationDriftOre = championEval.CostOre - champion.TotalCostOre + if math.Abs(block.ActiveEvaluationDriftOre) > championEvalDriftOre { + slog.Warn("mpc: core's own plan does not cost what core reported", + "decision_id", champion.DecisionID, + "reported_ore", champion.TotalCostOre, + "evaluated_ore", championEval.CostOre, + "drift_ore", block.ActiveEvaluationDriftOre) + } + block.TotalCostOre = shadowEval.CostOre + block.ActiveMinusShadowOre = championEval.CostOre - shadowEval.CostOre + block.ActivePVCurtailmentSlots = championEval.CurtailedSlots + block.PVCurtailmentSlots = shadowEval.CurtailedSlots + + // Raw totals do not compare: a plan that ends the horizon fuller looks + // expensive while it is merely storing value. Correct both sides before + // the difference is written anywhere a human will read it. + championOre := terminalCorrectedOre(championEval.CostOre, championEval.EndSoC, p) + shadowOre := terminalCorrectedOre(shadowEval.CostOre, shadowEval.EndSoC, p) + block.ActiveTerminalCorrectedOre = championOre + block.TerminalCorrectedOre = shadowOre + block.ActiveMinusShadowTerminalCorrectedOre = championOre - shadowOre + + slog.Info("mpc: core champion vs python shadow", + append(base, + "core_cost_ore", championEval.CostOre, + "python_cost_ore", shadowEval.CostOre, + "core_cost_ore_terminal_corrected", championOre, + "python_cost_ore_terminal_corrected", shadowOre, + "python_minus_core_ore_terminal_corrected", shadowOre-championOre, + "core_evaluation_drift_ore", block.ActiveEvaluationDriftOre, + "pv_curtailment_slots_core", championEval.CurtailedSlots, + "pv_curtailment_slots_python", shadowEval.CurtailedSlots)...) s.recordPythonShadow(champion, slots, p, reason, replanAtMs, block) } diff --git a/go/internal/mpc/python_shadow_test.go b/go/internal/mpc/python_shadow_test.go index 660ce28fa..70528c05c 100644 --- a/go/internal/mpc/python_shadow_test.go +++ b/go/internal/mpc/python_shadow_test.go @@ -6,6 +6,7 @@ import ( "errors" "math" "path/filepath" + "strings" "sync/atomic" "testing" "time" @@ -13,29 +14,59 @@ import ( "github.com/srcfl/ftw/go/internal/state" ) -// costShadowOptimizer answers with the DP's own plan re-labelled as the -// external solver, then overrides the two numbers the comparison is made of. -// That keeps the plan structurally valid while the cost difference stays a -// hand-computable constant. +// costShadowOptimizer answers with the DP's own plan, optionally scaled into +// a genuinely different action sequence, and then claims whatever cost and +// end SoC the test asks it to. The claims are the point: a challenger's own +// numbers must not reach the verdict, because they are the value of ITS +// objective, not Core's price for its plan. type costShadowOptimizer struct { totalCostOre float64 + objectiveOre float64 endSoC float64 - calls atomic.Int32 + // chargeLastSlotW rewrites the final slot's battery power, making the + // challenger a genuinely different — and here deliberately worse — plan + // rather than the champion's own actions handed back. + chargeLastSlotW float64 + calls atomic.Int32 } func (o *costShadowOptimizer) Optimize(_ context.Context, slots []Slot, p Params) (Plan, error) { o.calls.Add(1) plan := Optimize(slots, p) + if n := len(plan.Actions); n > 0 && o.chargeLastSlotW != 0 { + plan.Actions[n-1].BatteryW = o.chargeLastSlotW + } plan.TotalCostOre = o.totalCostOre if n := len(plan.Actions); n > 0 { plan.Actions[n-1].SoC = o.endSoC } - plan.Solver = &SolverInfo{Engine: "cvxpy", Backend: "highs", Status: "optimal", SolveMs: 42} + plan.Solver = &SolverInfo{ + Engine: "cvxpy", Backend: "highs", Status: "optimal", SolveMs: 42, + ObjectiveOre: o.objectiveOre, + } return plan, nil } func (o *costShadowOptimizer) Close() error { return nil } +// refusedShadowOptimizer answers with a plan Core cannot cost: the battery +// drives straight through the floor. A real challenger doing this is a +// finding, not a number. +type refusedShadowOptimizer struct{ calls atomic.Int32 } + +func (o *refusedShadowOptimizer) Optimize(_ context.Context, slots []Slot, p Params) (Plan, error) { + o.calls.Add(1) + plan := Optimize(slots, p) + for i := range plan.Actions { + plan.Actions[i].BatteryW = -p.MaxDischargeW + } + plan.TotalCostOre = -100000 // and a flattering total to go with it + plan.Solver = &SolverInfo{Engine: "cvxpy", Backend: "highs", Status: "optimal"} + return plan, nil +} + +func (o *refusedShadowOptimizer) Close() error { return nil } + type failingShadowOptimizer struct{ calls atomic.Int32 } func (o *failingShadowOptimizer) Optimize(context.Context, []Slot, Params) (Plan, error) { @@ -175,10 +206,15 @@ func TestPrimaryFailureStillMarksTheDPPlanAsFallback(t *testing.T) { // TestPythonShadowRecordsTerminalCorrectedComparison is the soak instrument: // same inputs, one number per replan, on the Diagnostic that -// /api/mpc/diagnose/at hands out. +// /api/mpc/diagnose/at hands out. Both plans are costed by Core, so a +// challenger cannot move the verdict by reporting a flattering total of its +// own — which is exactly how a −97 öre field verdict once stood beside a +// +32 öre bench result on the same day's data. func TestPythonShadowRecordsTerminalCorrectedComparison(t *testing.T) { svc := shadowTestService(t) - shadow := &costShadowOptimizer{totalCostOre: 1234, endSoC: 0.75} + shadow := &costShadowOptimizer{ + totalCostOre: 1234, objectiveOre: 4321, endSoC: 0.75, chargeLastSlotW: 1000, + } svc.ShadowOptimizer = shadow var saved atomic.Int32 var withShadow atomic.Int32 @@ -210,30 +246,47 @@ func TestPythonShadowRecordsTerminalCorrectedComparison(t *testing.T) { if block.ComparedSlots != len(plan.Actions) || block.FirstAction == nil { t.Fatalf("comparison incomplete: %+v", block) } - if block.TotalCostOre != 1234 { - t.Fatalf("shadow raw cost = %v, want 1234", block.TotalCostOre) + if block.EvaluationRefusedReason != "" { + t.Fatalf("a feasible challenger was refused: %q", block.EvaluationRefusedReason) } - if got, want := block.ActiveMinusShadowOre, plan.TotalCostOre-1234; got != want { - t.Fatalf("raw core − python = %v, want %v", got, want) + + // What the challenger claimed is kept, and kept apart. + if block.SelfReportedOre != 1234 || block.SelfReportedObjectiveOre != 4321 { + t.Fatalf("self-reported figures = %v / %v, want 1234 / 4321", + block.SelfReportedOre, block.SelfReportedObjectiveOre) + } + // ...but none of it reaches the comparison. + if block.TotalCostOre == 1234 { + t.Fatal("the challenger's own total became the comparison") + } + svcParams := Params{TerminalSoCPrice: 200, CapacityWh: 10000} + if terminalCorrectedOre(1234, 0.75, svcParams) == block.TerminalCorrectedOre { + t.Fatal("the challenger's own end SoC and total set the corrected figure") } - // Hand value: corrected = raw − price·(SoC·capacity)/1000 - // = 1234 − 200·(0.75·10000)/1000 = 1234 − 1500 = −266. - if d.Params.TerminalSoCPrice != 200 || d.Params.CapacityWh != 10000 { - t.Fatalf("terminal economics moved: price=%v capacity=%v", - d.Params.TerminalSoCPrice, d.Params.CapacityWh) + // Core's own plan must cost what Core said it costs. + if math.Abs(block.ActiveEvaluationDriftOre) > 1e-9 { + t.Fatalf("core's plan drifted from its own bookkeeping by %v öre", block.ActiveEvaluationDriftOre) + } + if got, want := block.ActiveTerminalCorrectedOre, + terminalCorrectedOre(plan.TotalCostOre, planEndSoC(plan), svcParams); math.Abs(got-want) > 1e-9 { + t.Fatalf("champion corrected = %v, want %v", got, want) } - wantShadow := -266.0 - wantChampion := terminalCorrectedOre(plan.TotalCostOre, - plan.Actions[len(plan.Actions)-1].SoC, Params{TerminalSoCPrice: 200, CapacityWh: 10000}) - if block.TerminalCorrectedOre != wantShadow { - t.Fatalf("shadow corrected = %v, want %v", block.TerminalCorrectedOre, wantShadow) + + // The challenger really is a different plan, so the verdict is not zero. + if block.MeanAbsBatteryDeltaW == 0 { + t.Fatal("the challenger returned the champion's own actions") } - if block.ActiveTerminalCorrectedOre != wantChampion { - t.Fatalf("champion corrected = %v, want %v", block.ActiveTerminalCorrectedOre, wantChampion) + wantDiff := block.ActiveTerminalCorrectedOre - block.TerminalCorrectedOre + if got := block.ActiveMinusShadowTerminalCorrectedOre; math.Abs(got-wantDiff) > 1e-9 { + t.Fatalf("corrected difference = %v, want %v", got, wantDiff) } - if got := block.ActiveMinusShadowTerminalCorrectedOre; got != wantChampion-wantShadow { - t.Fatalf("corrected difference = %v, want %v", got, wantChampion-wantShadow) + if got, want := block.ActiveMinusShadowOre, plan.TotalCostOre-block.TotalCostOre; math.Abs(got-want) > 1e-9 { + t.Fatalf("raw core − python = %v, want %v", got, want) + } + // The DP is optimal on these inputs, so a halved challenger costs more. + if block.ActiveMinusShadowTerminalCorrectedOre >= 0 { + t.Fatalf("a worse challenger scored as cheap: %v", block.ActiveMinusShadowTerminalCorrectedOre) } if saved.Load() != 2 || withShadow.Load() != 1 { t.Fatalf("diagnostic writes = %d (%d carrying the shadow), want the replan's write plus one rewrite", @@ -241,6 +294,47 @@ func TestPythonShadowRecordsTerminalCorrectedComparison(t *testing.T) { } } +// TestPythonShadowRefusesToScoreAnInfeasiblePlan — a challenger Core cannot +// cost produces a recorded reason and no difference at all. Half a +// measurement is worse than none: it looks exactly like a whole one. +func TestPythonShadowRefusesToScoreAnInfeasiblePlan(t *testing.T) { + svc := shadowTestService(t) + svc.ShadowOptimizer = &refusedShadowOptimizer{} + svc.SaveDiag = func(*Diagnostic, string) error { return nil } + + plan := svc.Replan(context.Background()) + if plan == nil { + t.Fatal("no champion plan") + } + waitFor(t, "the refused shadow to land", func() bool { + d := svc.Diagnose() + return d != nil && d.PythonShadow != nil + }) + + block := svc.Diagnose().PythonShadow + if block.EvaluationRefusedReason == "" { + t.Fatalf("an unscoreable plan was scored anyway: %+v", block) + } + if !strings.Contains(block.EvaluationRefusedReason, "python plan") { + t.Fatalf("refusal does not name the side that failed: %q", block.EvaluationRefusedReason) + } + for name, got := range map[string]float64{ + "total_cost_ore": block.TotalCostOre, + "active_minus_shadow_ore": block.ActiveMinusShadowOre, + "terminal_corrected_ore": block.TerminalCorrectedOre, + "active_terminal_corrected_ore": block.ActiveTerminalCorrectedOre, + "active_minus_shadow_terminal_corrected_ore": block.ActiveMinusShadowTerminalCorrectedOre, + } { + if got != 0 { + t.Fatalf("%s = %v on a refused comparison, want no number at all", name, got) + } + } + // The challenger's own claim is still recorded — it is the evidence. + if block.SelfReportedOre != -100000 { + t.Fatalf("self-reported cost = %v, want the claim that was refused", block.SelfReportedOre) + } +} + // TestPythonShadowFailureLeavesTheChampionAlone — a broken challenger costs // the site nothing but a log line. func TestPythonShadowFailureLeavesTheChampionAlone(t *testing.T) { diff --git a/go/internal/mpc/replay_bench_test.go b/go/internal/mpc/replay_bench_test.go index c53e3e5fe..70345a0ac 100644 --- a/go/internal/mpc/replay_bench_test.go +++ b/go/internal/mpc/replay_bench_test.go @@ -61,15 +61,57 @@ func loadDiagnosticBlob(data []byte) (*Diagnostic, error) { // field shadow reports the same correction every replan, so the bench // and the running planner must not drift apart on the formula. +// benchEvaluate costs one plan under Core's model. A plan Core cannot cost +// fails the bench outright rather than being silently replaced by the number +// it came with — the same rule the field shadow follows. The caller +// terminal-corrects; the raw cost and the reached end SoC are reported +// alongside so a reader can see whether an advantage is cheaper decisions or +// merely more energy left in the pack. +func benchEvaluate(t *testing.T, leg, snapshot string, plan *Plan, slots []Slot, p Params) planEvaluation { + t.Helper() + eval, err := evaluatePlan(*plan, slots, p) + if err != nil { + t.Fatalf("%s: core cannot cost the %s plan: %v", snapshot, leg, err) + } + return eval +} + // TestReplayBenchSnapshots is the A/B instrument. Skipped without // FTW_MPC_SNAPSHOT_DIR. Optional knobs: // -// FTW_TEST_OPTIMIZER_PYTHON — adds the Python champion leg +// FTW_TEST_OPTIMIZER_PYTHON — adds the Python challenger leg // FTW_MPC_BENCH_SPREAD_ORE — MinArbitrageSpreadOreKwh fallback for // blobs written before the diagnostic // persisted it. A spread carried by the // snapshot always wins: it is what the // replan actually solved under. +// FTW_MPC_BENCH_CVAR_WEIGHT — the site's optimizer.cvar_weight +// FTW_MPC_BENCH_CVAR_ALPHA — the site's optimizer.cvar_alpha +// FTW_MPC_BENCH_MIP_GAP — the site's optimizer.mip_rel_gap +// FTW_MPC_BENCH_TERMINAL_SCALE — multiplier on the snapshot's +// TerminalSoCPrice: 1.0 as recorded, 0 for no +// terminal credit at all. +// +// The three optimizer knobs exist because the diagnostic records the +// PLANNING inputs, not the challenger's solver configuration: replaying a +// box's snapshot with the bench's defaults asks Python a different question +// than the box asked it, and the answer differs by more than the gap being +// measured. Set them to the site's values before reading a py column as "what +// that box's shadow would have said". +// +// The terminal knob answers a different question: whether a measured gap is a +// gap between plans or an artifact of how stored energy is priced. The +// terminal price is both an input the solvers optimize against and the rate +// the comparison credits leftover charge at, so the scale moves both together +// — every leg still solves and is scored under one consistent price of stored +// energy. If a challenger's lead melts as the scale goes to 0, the lead was +// bought with end-of-horizon SoC, not with cheaper decisions. +// +// py_self is what Python reported for its own plan; py_corr is what that same +// plan costs under Core's model, terminal-corrected. Only py_corr may be +// subtracted from dp_corr. The *_raw columns are the same plans before the +// correction and *_soc the SoC each parks the horizon at, which is what makes +// the difference between the two readable. // // Run with -v; the verdict is the table, not a pass/fail. func TestReplayBenchSnapshots(t *testing.T) { @@ -90,6 +132,15 @@ func TestReplayBenchSnapshots(t *testing.T) { } } + benchFloat := func(key string, fallback float64) float64 { + if v := os.Getenv(key); v != "" { + if f, err := strconv.ParseFloat(v, 64); err == nil { + return f + } + } + return fallback + } + var ext *ExternalOptimizer if python := os.Getenv("FTW_TEST_OPTIMIZER_PYTHON"); python != "" { _, file, _, ok := runtime.Caller(0) @@ -97,13 +148,19 @@ func TestReplayBenchSnapshots(t *testing.T) { t.Fatal("runtime.Caller failed") } moduleDir := filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", "..", "optimizer")) + cvarWeight := benchFloat("FTW_MPC_BENCH_CVAR_WEIGHT", 0) + cvarAlpha := benchFloat("FTW_MPC_BENCH_CVAR_ALPHA", 0.9) + mipGap := benchFloat("FTW_MPC_BENCH_MIP_GAP", 0.001) + t.Logf("python leg: cvar_weight=%g cvar_alpha=%g mip_rel_gap=%g", cvarWeight, cvarAlpha, mipGap) ext, err = NewExternalOptimizer(ExternalOptimizerConfig{ Command: []string{python, "-m", "ftw_optimizer.worker"}, ModuleDir: moduleDir, Timeout: 60 * time.Second, Solver: "HIGHS", Formulation: "auto", - MIPRelGap: 0.001, + MIPRelGap: mipGap, + CVaRWeight: cvarWeight, + CVaRAlpha: cvarAlpha, IdleTimeout: 5 * time.Second, }) if err != nil { @@ -112,9 +169,16 @@ func TestReplayBenchSnapshots(t *testing.T) { defer ext.Close() } - t.Logf("%-15s %-18s %10s %10s %10s %10s %10s %10s", - "snapshot", "mode", "rec_corr", "dp_corr", "dp-rec", "py_corr", "py-dp", "dp_ms") - var sumDPvsRec, sumPYvsDP float64 + terminalScale := benchFloat("FTW_MPC_BENCH_TERMINAL_SCALE", 1) + if terminalScale != 1 { + t.Logf("terminal price scaled by %g (solve and scoring alike)", terminalScale) + } + + t.Logf("%-15s %-18s %9s %9s %6s %9s %9s %9s %9s %6s %9s %9s %9s %8s", + "snapshot", "mode", "rec_corr", + "dp_raw", "dp_soc", "dp_corr", "dp-rec", + "py_self", "py_raw", "py_soc", "py_corr", "py-dp", "py-dp_raw", "dp_ms") + var sumDPvsRec, sumPYvsDP, sumPYvsDPRaw, sumSoCDiff float64 var nDP, nPY int for _, path := range paths { data, err := os.ReadFile(path) @@ -134,6 +198,9 @@ func TestReplayBenchSnapshots(t *testing.T) { if params.MinArbitrageSpreadOreKwh == 0 { params.MinArbitrageSpreadOreKwh = fallbackSpreadOre } + // Applied before anything solves, so the DP, Python and the + // correction all price stored energy the same way. + params.TerminalSoCPrice *= terminalScale // A blob carries the RECORDED replan's grid resolution, so a // default bump would be invisible here without an override — // judging a resolution change is exactly what the knobs exist @@ -149,38 +216,61 @@ func TestReplayBenchSnapshots(t *testing.T) { } } - recCorr := terminalCorrectedOre(recorded.TotalCostOre, planEndSoC(recorded), params) + // Every leg is costed by Core, on Core's terms. A solver's own + // total is its objective's opinion of its own plan; subtracting + // one solver's objective from another's measures the objectives, + // not the plans (#1020 follow-up). + recEval := benchEvaluate(t, "recorded", filepath.Base(path), recorded, slots, params) + recCorr := terminalCorrectedOre(recEval.CostOre, recEval.EndSoC, params) dpStart := time.Now() dpPlan := Optimize(slots, params) dpMs := time.Since(dpStart).Milliseconds() - dpCorr := terminalCorrectedOre(dpPlan.TotalCostOre, planEndSoC(&dpPlan), params) + dpEval := benchEvaluate(t, "dp", filepath.Base(path), &dpPlan, slots, params) + dpCorr := terminalCorrectedOre(dpEval.CostOre, dpEval.EndSoC, params) sumDPvsRec += dpCorr - recCorr nDP++ - pyCol, pyDelta := "-", "-" + pySelf, pyRaw, pySoC, pyCol, pyDelta, pyDeltaRaw := "-", "-", "-", "-", "-", "-" if ext != nil { pyPlan, err := ext.Optimize(t.Context(), slots, params) - if err != nil { + switch { + case err != nil: pyCol = "ERR" t.Logf("%-15s python: %v", filepath.Base(path), err) - } else { - pyCorr := terminalCorrectedOre(pyPlan.TotalCostOre, planEndSoC(&pyPlan), params) - pyCol = fmt.Sprintf("%10.1f", pyCorr) - pyDelta = fmt.Sprintf("%10.1f", pyCorr-dpCorr) + default: + pySelf = fmt.Sprintf("%9.1f", + terminalCorrectedOre(pyPlan.TotalCostOre, planEndSoC(&pyPlan), params)) + pyEval, evalErr := evaluatePlan(pyPlan, slots, params) + if evalErr != nil { + pyCol = "REFUSED" + t.Logf("%-15s python plan not costable by core: %v", filepath.Base(path), evalErr) + break + } + pyCorr := terminalCorrectedOre(pyEval.CostOre, pyEval.EndSoC, params) + pyRaw = fmt.Sprintf("%9.1f", pyEval.CostOre) + pySoC = fmt.Sprintf("%6.3f", pyEval.EndSoC) + pyCol = fmt.Sprintf("%9.1f", pyCorr) + pyDelta = fmt.Sprintf("%9.1f", pyCorr-dpCorr) + pyDeltaRaw = fmt.Sprintf("%9.1f", pyEval.CostOre-dpEval.CostOre) sumPYvsDP += pyCorr - dpCorr + sumPYvsDPRaw += pyEval.CostOre - dpEval.CostOre + sumSoCDiff += pyEval.EndSoC - dpEval.EndSoC nPY++ } } - t.Logf("%-15s %-18s %10.1f %10.1f %10.1f %10s %10s %8dms", - filepath.Base(path), string(params.Mode), - recCorr, dpCorr, dpCorr-recCorr, pyCol, pyDelta, dpMs) + t.Logf("%-15s %-18s %9.1f %9.1f %6.3f %9.1f %9.1f %9s %9s %6s %9s %9s %9s %6dms", + filepath.Base(path), string(params.Mode), recCorr, + dpEval.CostOre, dpEval.EndSoC, dpCorr, dpCorr-recCorr, + pySelf, pyRaw, pySoC, pyCol, pyDelta, pyDeltaRaw, dpMs) } if nDP > 0 { t.Logf("SUMMARY dp_vs_recorded: n=%d mean=%.1f öre/plan", nDP, sumDPvsRec/float64(nDP)) } if nPY > 0 { - t.Logf("SUMMARY python_vs_dp (terminal-corrected, positive = python costs more): n=%d mean=%.1f öre/plan", nPY, sumPYvsDP/float64(nPY)) + t.Logf("SUMMARY python_vs_dp (both costed by core, terminal-corrected, positive = python costs more): n=%d mean=%.1f öre/plan", nPY, sumPYvsDP/float64(nPY)) + t.Logf("SUMMARY python_vs_dp_raw (no terminal correction): n=%d mean=%.1f öre/plan; mean end_soc diff (py-dp) = %+.4f (%+.1f Wh)", + nPY, sumPYvsDPRaw/float64(nPY), sumSoCDiff/float64(nPY), sumSoCDiff/float64(nPY)*9600) } }