From c3393c6a853413487b6993fc743447aa84314a65 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 31 Aug 2026 13:45:28 +0200 Subject: [PATCH 1/3] fix(mpc): cost every challenger with Core's own model, and let the bench ask Python the same question MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shadow's verdict could not be checked. Production said Python beat Core by ~97 öre on 55 of 55 replans; the replay bench, on a snapshot pulled from that same box the same day, said Python cost 32 öre MORE. Same plans, opposite signs, and no way to tell which number to believe. The bench was wrong. Its ExternalOptimizerConfig omits CVaRWeight while the box runs cvar_weight 0.15 at alpha 0.9, so the two were asking Python different questions. Setting the box's value in the bench reproduces the recorded figure exactly: -101.14 öre against the box's -101.14. The field measurement was right all along. So the numbers stand, but nothing forced them to. plan.TotalCostOre arrives from the optimizer and is only checked by ValidatePlan; the comparison never recomputed it from Core's own arithmetic, and Python's real objective (Solver.ObjectiveOre, 5003 öre against the 5453 the plan costs) was never in the comparison at all, which is what made the mismatch so hard to read. evaluatePlanOre now costs a given action sequence with the DP's own forward pass -- one copy of the arithmetic, shared through stepPlanSlot and stepLoadpointSoC, so a later change cannot silently invalidate the measurements. It refuses rather than guesses: wrong action count, non-finite values, slot misalignment, SoC out of band, power past limits. The shadow costs both plans this way, warns when Core's plan disagrees with Core's own bookkeeping, and records the self-reported figures beside the honest one. The bench takes the box's solver settings through FTW_MPC_BENCH_CVAR_WEIGHT / _CVAR_ALPHA / _MIP_GAP, so it can be pointed at the question production actually asks. Measured across 13 snapshots: the evaluator's drift from the plans' recorded totals is 0.000000000 öre, freshly solved and box-recorded alike, so the production verdict is unchanged to within 0.05 öre. Also surfaces a curtailment asymmetry found on the way: annotateCurtailment sets PVLimitW after TotalCostOre is summed, so a curtailing DP plan is costed as if it exported everything while ValidatePlan costs Python's with its curtailment. None of these snapshots has a negative spot slot, so it is not behind the gap above. Both sides are now costed at full PV and pv_curtailment_slots counts the case instead of leaving it silent. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C1d4uknzs7QUv7nE9rW4fi --- .changeset/shadow-costed-by-core.md | 30 +++ go/internal/mpc/mpc.go | 91 +++++-- go/internal/mpc/plan_evaluation.go | 211 ++++++++++++++++ go/internal/mpc/plan_evaluation_test.go | 315 ++++++++++++++++++++++++ go/internal/mpc/python_shadow.go | 88 +++++-- go/internal/mpc/python_shadow_test.go | 146 +++++++++-- go/internal/mpc/replay_bench_test.go | 81 +++++- 7 files changed, 892 insertions(+), 70 deletions(-) create mode 100644 .changeset/shadow-costed-by-core.md create mode 100644 go/internal/mpc/plan_evaluation.go create mode 100644 go/internal/mpc/plan_evaluation_test.go diff --git a/.changeset/shadow-costed-by-core.md b/.changeset/shadow-costed-by-core.md new file mode 100644 index 000000000..652199890 --- /dev/null +++ b/.changeset/shadow-costed-by-core.md @@ -0,0 +1,30 @@ +--- +"ftw": patch +--- + +The Python shadow's verdict is now Core's price for both plans. Core walks +the challenger's own action sequence through its forward pass and costs it +with the same arithmetic the DP uses on itself, instead of reading the total +the challenger arrived with — and it does the same for its own plan, so the +subtraction stays symmetric if the DP's bookkeeping ever moves. The +difference keeps the key `python_minus_core_ore_terminal_corrected`. + +What the challenger reported is kept beside the verdict rather than inside +it: `self_reported_ore` is the cost it claimed for its own plan, and +`self_reported_objective_ore` is the value it actually minimized — +scenario-weighted and CVaR-shaped, so not comparable with any cost. On the +owner's box those two differ by 450 öre on a single plan. + +A plan Core cannot cost honestly — wrong length, non-finite output, a +battery driven outside its operating band or past its power limits — is +recorded as `evaluation_refused_reason` with no difference number at all. +Core costing its own plan differently from what it reported is a bug in +Core, and now says so with a warning and `active_evaluation_drift_ore`. + +The replay bench measures the same way, prints the challenger's own figure +in its own column, and takes `FTW_MPC_BENCH_CVAR_WEIGHT`, +`FTW_MPC_BENCH_CVAR_ALPHA` and `FTW_MPC_BENCH_MIP_GAP`. Those knobs matter: +a snapshot records the planning inputs but not the site's solver settings, +so replaying one with the bench's defaults asks the challenger a different +question than the box asked — worth more than 130 öre per plan on a summer +day with PV uncertainty, which is larger than the gap being measured. diff --git a/go/internal/mpc/mpc.go b/go/internal/mpc/mpc.go index 9f1ae7c1a..b04a3c632 100644 --- a/go/internal/mpc/mpc.go +++ b/go/internal/mpc/mpc.go @@ -404,6 +404,30 @@ type ShadowPlan struct { ActiveTerminalCorrectedOre float64 `json:"active_terminal_corrected_ore,omitempty"` TerminalCorrectedOre float64 `json:"terminal_corrected_ore,omitempty"` ActiveMinusShadowTerminalCorrectedOre float64 `json:"active_minus_shadow_terminal_corrected_ore,omitempty"` + + // SelfReportedOre is the grid cost the challenger claimed for its own + // plan, kept beside Core's valuation of it rather than in place of it. + // The two agree only as long as the challenger prices the meter the way + // Core does; the gap is the diagnostic. + SelfReportedOre float64 `json:"self_reported_ore,omitempty"` + // SelfReportedObjectiveOre is the value the challenger actually + // minimized — scenario-weighted, CVaR-shaped, carrying its own + // penalties. It is NOT comparable with any cost above and exists to + // show how differently the two objectives are shaped. + SelfReportedObjectiveOre float64 `json:"self_reported_objective_ore,omitempty"` + // EvaluationRefusedReason is set when Core declined to cost one of the + // plans. The comparison fields are then left empty: a refusal that gets + // recorded beats a plausible number nobody can tell from a measurement. + EvaluationRefusedReason string `json:"evaluation_refused_reason,omitempty"` + // ActiveEvaluationDriftOre is Core's valuation of its OWN plan minus the + // cost that plan reported. Anything but ~0 is a bug in Core's + // bookkeeping, not a property of the challenger. + ActiveEvaluationDriftOre float64 `json:"active_evaluation_drift_ore,omitempty"` + // PV curtailment counts. Core's cost model prices every slot at full + // forecast generation on both sides, so slots either plan wants to + // curtail are slots where the comparison is blind to a real difference. + ActivePVCurtailmentSlots int `json:"active_pv_curtailment_slots,omitempty"` + PVCurtailmentSlots int `json:"pv_curtailment_slots,omitempty"` } // terminalCorrectedOre nets the terminal-SoC credit out of a raw grid cost so @@ -493,6 +517,48 @@ func SlotExportPriceOre(slot Slot, p Params) float64 { return gridcost.ExportPriceOre(slot.SpotOre, exportPricingFromParams(p)) } +// planSlotOutcome is one slot of a forward walk: the SoC the battery +// reaches, the power the meter sees, and what that flow costs under Core's +// cost model. +type planSlotOutcome struct { + SoC float64 + GridW float64 + CostOre float64 +} + +// stepPlanSlot advances one slot from socIn under a battery and loadpoint +// power. The DP's own forward pass and evaluatePlan both drive it, so the +// cost arithmetic cannot be changed for one and not the other. That sharing +// is the point: a foreign plan can only be judged on Core's terms while +// "Core's terms" is a single piece of code, and a second copy of these three +// lines would silently invalidate every comparison the day one of them moved. +// +// SoC comes back unclamped. The DP pins it to the operating band afterwards +// because its feasibility screen already rejected out-of-band transitions; +// evaluatePlan needs to see the raw excursion, because a foreign plan has had +// no such screen. +func stepPlanSlot(slot Slot, p Params, socIn, batteryW, loadpointW float64) planSlotOutcome { + dtH := float64(slot.LenMin) / 60.0 + var deltaWh float64 + if batteryW >= 0 { + deltaWh = +batteryW * dtH * p.ChargeEfficiency + } else { + deltaWh = +batteryW * dtH / p.DischargeEfficiency + } + gridW := slot.LoadW + slot.PVW + batteryW + loadpointW + return planSlotOutcome{ + SoC: socIn + deltaWh/p.CapacityWh, + GridW: gridW, + CostOre: SlotGridCostOre(slot, gridW*dtH/1000.0, p), + } +} + +// stepLoadpointSoC advances one flex load over a slot. Charging only, so the +// result never falls; the caller enforces the ceiling. +func stepLoadpointSoC(socIn, powerW, dtH, capacityWh, chargeEfficiency float64) float64 { + return socIn + powerW*dtH*chargeEfficiency/capacityWh +} + func exportPricingFromParams(p Params) gridcost.ExportPricing { return gridcost.ExportPricing{ BonusOreKwh: p.ExportBonusOreKwh, @@ -1077,14 +1143,12 @@ func Optimize(slots []Slot, p Params) Plan { ea := pol % EA actW := actionAt(ba) evW := evActionW(ea) - // Battery SoC transition. - var dSoCWh float64 - if actW >= 0 { - dSoCWh = +actW * dtH * p.ChargeEfficiency - } else { - dSoCWh = +actW * dtH / p.DischargeEfficiency - } - soc2 := soc + dSoCWh/p.CapacityWh + // Battery SoC transition and slot cost. Report the ACTUAL + // expected cost using the raw (un-blended) prices so the UI + // summary reflects "what we'd actually pay if prices hold". + // Blending is a decision lens only. + step := stepPlanSlot(slot, p, soc, actW, evW) + soc2 := step.SoC if soc2 < p.SoCMin { soc2 = p.SoCMin } @@ -1094,18 +1158,13 @@ func Optimize(slots []Slot, p Params) Plan { // EV SoC transition (no-op when !evActive since evW = 0). var evSoc2 float64 if evActive { - dEvWh := evW * dtH * evChargeEff - evSoc2 = evSoc + dEvWh/lp.CapacityWh + evSoc2 = stepLoadpointSoC(evSoc, evW, dtH, lp.CapacityWh, evChargeEff) if evSoc2 > lp.SoCMax { evSoc2 = lp.SoCMax } } - gridW := slot.LoadW + slot.PVW + actW + evW - gridKWh := gridW * dtH / 1000.0 - // Report the ACTUAL expected cost using the raw (un-blended) - // prices so the UI summary reflects "what we'd actually pay - // if prices hold". Blending is a decision lens only. - cost := SlotGridCostOre(slot, gridKWh, p) + gridW := step.GridW + cost := step.CostOre totalCost += cost a := Action{ SlotStartMs: slot.StartMs, diff --git a/go/internal/mpc/plan_evaluation.go b/go/internal/mpc/plan_evaluation.go new file mode 100644 index 000000000..b45b279d7 --- /dev/null +++ b/go/internal/mpc/plan_evaluation.go @@ -0,0 +1,211 @@ +package mpc + +import ( + "errors" + "fmt" + "math" +) + +// Costing a foreign plan is not the same thing as reading the number that +// plan came with. A solver reports the value of ITS objective — scenario +// weights, a CVaR tail term, its own penalties and bonuses — and subtracting +// that from Core's cost measures the two objectives rather than the two +// plans. evaluatePlan closes that gap: it walks somebody else's action +// sequence through Core's own forward pass and reports what Core says it +// costs, so both sides of a comparison are priced by one piece of code. +// +// Refusal is a first-class result. A plan that cannot be costed honestly — +// wrong length, non-finite output, state outside the operating band Core +// enforces — yields no number at all, because a logged refusal is worth more +// than a plausible fabrication nobody can tell apart from a measurement. + +// planEvaluation is Core's valuation of a plan's actions. +type planEvaluation struct { + // CostOre is the raw grid cost over the horizon — the same quantity + // Plan.TotalCostOre carries for a Core plan. Terminal-correct it before + // comparing two plans that park the horizon at different SoC. + CostOre float64 + // EndSoC is the SoC the walk reached, not the SoC the plan claimed. + // Trusting a foreign plan's own SoC column would put the terminal + // credit back on the challenger's honour system. + EndSoC float64 + // CurtailedSlots counts slots whose action asks to cap PV. Core's cost + // model prices every slot at full forecast generation, for its own plans + // and foreign ones alike, so this is not a mispricing — it is the count + // of slots where the two plans could still differ in a way the model + // does not resolve. + CurtailedSlots int +} + +var errPlanEvaluationCapacity = errors.New("capacity_wh is not positive") + +// evaluatePlanOre reports what a plan's actions cost under Core's cost +// model, so a foreign plan can be judged on the same terms as Core's own. +// ok is false when the plan cannot be costed; evaluatePlan carries the +// reason for callers that log it. +func evaluatePlanOre(plan Plan, slots []Slot, p Params) (ore float64, ok bool) { + eval, err := evaluatePlan(plan, slots, p) + if err != nil { + return 0, false + } + return eval.CostOre, true +} + +// evaluatePlan walks plan.Actions across slots under Core's cost model and +// returns Core's valuation, or the reason it refuses to produce one. +// +// Storage is walked in aggregate — one SoC against p.CapacityWh and the +// aggregate efficiencies — because that is the state the champion plans in +// and the state its own reported cost describes. Cost never reads SoC, so a +// fleet of batteries with unequal efficiencies can only move the terminal +// correction and the band check, never the öre per slot. ValidatePlan +// already replays a foreign plan per storage on the way in. +func evaluatePlan(plan Plan, slots []Slot, p Params) (planEvaluation, error) { + // Optimize sanitizes and defaults before it costs anything. Judging a + // plan under other inputs than the ones Core would have used is not + // judging it under Core's model. + slots = sanitizeOptimizeSlots(slots) + if len(slots) == 0 { + return planEvaluation{}, errors.New("no slots to evaluate against") + } + if p.CapacityWh <= 0 { + return planEvaluation{}, errPlanEvaluationCapacity + } + if len(plan.Actions) != len(slots) { + return planEvaluation{}, fmt.Errorf("action count %d, want %d", + len(plan.Actions), len(slots)) + } + if p.ChargeEfficiency <= 0 { + p.ChargeEfficiency = 0.95 + } + if p.DischargeEfficiency <= 0 { + p.DischargeEfficiency = 0.95 + } + + bandActive := p.SoCMax > p.SoCMin + bandTol := socBandTolerance(p) + soc := p.InitialSoC + if bandActive { + // Same clamp the DP's forward pass opens with: the site may report + // a SoC outside the band, and Core plans from the band edge. + soc = math.Max(p.SoCMin, math.Min(p.SoCMax, soc)) + } + + loadpoints := p.activeLoadpoints() + lpSoC := make(map[string]float64, len(loadpoints)) + for _, lp := range loadpoints { + lpSoC[lp.ID] = lp.InitialSoC + } + + eval := planEvaluation{EndSoC: soc} + for i, slot := range slots { + a := plan.Actions[i] + if a.SlotStartMs != slot.StartMs || a.SlotLenMin != slot.LenMin { + return planEvaluation{}, fmt.Errorf("slot %d is not the slot the action was planned for", i) + } + loadpointW := 0.0 + for idx := range loadpoints { + powerW := evaluationLoadpointStepW(a, loadpoints, idx) + if !finite(powerW) { + return planEvaluation{}, fmt.Errorf("slot %d loadpoint %s has non-finite power", + i, loadpoints[idx].ID) + } + loadpointW += powerW + } + if !finite(a.BatteryW) || !finite(a.PVLimitW) { + return planEvaluation{}, fmt.Errorf("slot %d contains non-finite output", i) + } + if a.BatteryW > p.MaxChargeW+planEvaluationPowerTolW || + a.BatteryW < -p.MaxDischargeW-planEvaluationPowerTolW { + return planEvaluation{}, fmt.Errorf( + "slot %d battery_w %.1f outside charge/discharge limits %.1f…%.1f", + i, a.BatteryW, -p.MaxDischargeW, p.MaxChargeW) + } + + step := stepPlanSlot(slot, p, soc, a.BatteryW, loadpointW) + if !finite(step.SoC) || !finite(step.CostOre) { + return planEvaluation{}, fmt.Errorf("slot %d costs to a non-finite value", i) + } + if bandActive { + if step.SoC < p.SoCMin-bandTol || step.SoC > p.SoCMax+bandTol { + return planEvaluation{}, fmt.Errorf( + "slot %d drives soc to %.4f, outside %.4f…%.4f", i, step.SoC, p.SoCMin, p.SoCMax) + } + // The DP pins its own walk to the band; matching it keeps a Core + // plan's evaluation identical to the cost the DP reported. + soc = math.Max(p.SoCMin, math.Min(p.SoCMax, step.SoC)) + } else { + soc = step.SoC + } + for idx, lp := range loadpoints { + ceiling := lp.SoCMax + if ceiling <= lp.SoCMin { + ceiling = 1.0 // same normalization the DP applies + } + eff := lp.ChargeEfficiency + if eff <= 0 { + eff = 0.9 + } + next := stepLoadpointSoC(lpSoC[lp.ID], evaluationLoadpointStepW(a, loadpoints, idx), + float64(slot.LenMin)/60.0, lp.CapacityWh, eff) + if !finite(next) { + return planEvaluation{}, fmt.Errorf("slot %d loadpoint %s reaches a non-finite soc", i, lp.ID) + } + if next > ceiling+planEvaluationLoadpointSoCTol { + return planEvaluation{}, fmt.Errorf( + "slot %d drives loadpoint %s to soc %.4f, above %.4f", i, lp.ID, next, ceiling) + } + // The DP pins the EV walk to its ceiling too. + lpSoC[lp.ID] = math.Min(next, ceiling) + } + + eval.CostOre += step.CostOre + if a.PVLimitW > 0 { + eval.CurtailedSlots++ + } + } + eval.EndSoC = soc + if !finite(eval.CostOre) { + return planEvaluation{}, errors.New("horizon cost is not finite") + } + return eval, nil +} + +const ( + // planEvaluationPowerTolW matches the slack ValidatePlan already grants a + // continuous solver riding a bound: float residue lands a plan a + // fraction of a watt over its own limit, and rejecting the measurement + // for that would only hide it. + planEvaluationPowerTolW = 2.0 + // planEvaluationLoadpointSoCTol mirrors ValidatePlan's EV replay + // tolerance. + planEvaluationLoadpointSoCTol = 0.0005 +) + +// socBandTolerance is how far past soc_min…soc_max one slot may land before +// the walk refuses. The DP looks its policy up on a discretized SoC grid +// while propagating a continuous SoC, so a legitimate Core plan can overshoot +// by up to a grid step; a plan that leaves the band by more than the DP's own +// resolution allows is asking for state the band does not permit. +func socBandTolerance(p Params) float64 { + levels := p.SoCLevels + if levels < 3 { + levels = 3 + } + step := (p.SoCMax - p.SoCMin) / float64(levels-1) + return math.Max(0.001, step) +} + +// evaluationLoadpointStepW reads one action's power for one active flex +// load, using the same per-ID map with a first-loadpoint scalar fallback +// ValidatePlan reads — the DP writes only the scalar, the external optimizer +// writes both. +func evaluationLoadpointStepW(a Action, loadpoints []*LoadpointSpec, idx int) float64 { + if len(a.LoadpointPowerW) == 0 { + if idx == 0 { + return a.LoadpointW + } + return 0 + } + return a.LoadpointPowerW[loadpoints[idx].ID] +} diff --git a/go/internal/mpc/plan_evaluation_test.go b/go/internal/mpc/plan_evaluation_test.go new file mode 100644 index 000000000..474d33bdc --- /dev/null +++ b/go/internal/mpc/plan_evaluation_test.go @@ -0,0 +1,315 @@ +package mpc + +import ( + "math" + "strings" + "testing" +) + +// evaluationSlots builds a cheap-then-expensive horizon: enough spread that +// every mode below actually moves the battery. +func evaluationSlots(n int) []Slot { + slots := make([]Slot, n) + for i := range slots { + price := 80.0 + pv := 0.0 + if i >= n/2 { + price = 320.0 + } + if i >= n/4 && i < n/2 { + pv = -2500 // midday surplus + } + slots[i] = Slot{ + StartMs: 1_700_000_000_000 + int64(i)*900_000, + LenMin: 15, + PriceOre: price, + SpotOre: price / 2, + Confidence: 1, + PVW: pv, + LoadW: 900, + } + } + return slots +} + +func evaluationParams(mode Mode) Params { + return Params{ + Mode: mode, CapacityWh: 9600, InitialSoC: 0.5, + SoCMin: 0.1, SoCMax: 1.0, SoCLevels: 61, ActionLevels: 41, + MaxChargeW: 4000, MaxDischargeW: 4000, + ChargeEfficiency: 0.95, DischargeEfficiency: 0.95, + TerminalSoCPrice: 150, + } +} + +// TestEvaluatePlanCostsTheDPsOwnPlanIdentically is the sharing proof. The DP +// reports a total; the evaluator re-derives it from the same actions. They +// agree only while both drive the one piece of cost arithmetic — fork it and +// this test is what fails, before a measurement quietly stops meaning +// anything. +func TestEvaluatePlanCostsTheDPsOwnPlanIdentically(t *testing.T) { + cases := map[string]func() (Params, []Slot){ + "arbitrage": func() (Params, []Slot) { + return evaluationParams(ModeArbitrage), evaluationSlots(24) + }, + "passive arbitrage": func() (Params, []Slot) { + return evaluationParams(ModePassiveArbitrage), evaluationSlots(24) + }, + "self consumption": func() (Params, []Slot) { + return evaluationParams(ModeSelfConsumption), evaluationSlots(24) + }, + "arbitrage with a minimum spread": func() (Params, []Slot) { + p := evaluationParams(ModeArbitrage) + p.MinArbitrageSpreadOreKwh = 30 + return p, evaluationSlots(24) + }, + "arbitrage starting at the floor": func() (Params, []Slot) { + p := evaluationParams(ModeArbitrage) + p.InitialSoC = p.SoCMin + return p, evaluationSlots(24) + }, + "arbitrage starting below the floor": func() (Params, []Slot) { + // The DP plans from the clamped band edge; so must the + // evaluator, or every recovery replan would be unscoreable. + p := evaluationParams(ModeArbitrage) + p.InitialSoC = 0.05 + return p, evaluationSlots(24) + }, + "with an active loadpoint": func() (Params, []Slot) { + p := evaluationParams(ModeArbitrage) + p.Loadpoint = &LoadpointSpec{ + ID: "ev", CapacityWh: 60000, InitialSoC: 0.3, SoCMin: 0, + SoCMax: 0.9, TargetSoC: 0.8, TargetSlotIdx: 20, Levels: 11, + MaxChargeW: 11000, ChargeEfficiency: 0.9, + } + return p, evaluationSlots(24) + }, + "loadpoint on surplus only": func() (Params, []Slot) { + p := evaluationParams(ModePassiveArbitrage) + p.Loadpoint = &LoadpointSpec{ + ID: "ev", CapacityWh: 60000, InitialSoC: 0.3, SoCMin: 0, + SoCMax: 0.9, TargetSoC: 0.8, TargetSlotIdx: 20, Levels: 11, + MaxChargeW: 11000, ChargeEfficiency: 0.9, SurplusOnly: true, + } + return p, evaluationSlots(24) + }, + "export priced flat": func() (Params, []Slot) { + p := evaluationParams(ModeArbitrage) + p.ExportOrePerKWh = 60 + return p, evaluationSlots(24) + }, + "fifteen and sixty minute slots mixed": func() (Params, []Slot) { + slots := evaluationSlots(24) + for i := range slots { + if i%3 == 0 { + slots[i].LenMin = 60 + } + } + return evaluationParams(ModeArbitrage), slots + }, + } + for name, build := range cases { + t.Run(name, func(t *testing.T) { + p, slots := build() + plan := Optimize(slots, p) + if len(plan.Actions) == 0 { + t.Fatal("the DP produced no plan to evaluate") + } + ore, ok := evaluatePlanOre(plan, slots, p) + if !ok { + _, err := evaluatePlan(plan, slots, p) + t.Fatalf("core refused to cost its own plan: %v", err) + } + if diff := math.Abs(ore - plan.TotalCostOre); diff > 1e-9 { + t.Fatalf("evaluated %.9f öre, the DP reported %.9f (%.9f apart)", + ore, plan.TotalCostOre, diff) + } + // The end SoC drives the terminal correction, so it has to come + // from the same walk rather than the plan's own column. + eval, err := evaluatePlan(plan, slots, p) + if err != nil { + t.Fatal(err) + } + if diff := math.Abs(eval.EndSoC - planEndSoC(&plan)); diff > 1e-9 { + t.Fatalf("walked end soc %.9f, plan reported %.9f", eval.EndSoC, planEndSoC(&plan)) + } + }) + } +} + +// TestEvaluatePlanRefusesRatherThanGuessing — every refusal here is a plan +// Core cannot honestly price. A number would be indistinguishable from a +// measurement; the absence of one is not. +func TestEvaluatePlanRefusesRatherThanGuessing(t *testing.T) { + base := evaluationParams(ModeArbitrage) + slots := evaluationSlots(12) + good := Optimize(slots, base) + + cases := map[string]struct { + mutate func(Plan) Plan + params func(Params) Params + reason string + }{ + "one action short": { + mutate: func(p Plan) Plan { p.Actions = p.Actions[:len(p.Actions)-1]; return p }, + reason: "action count", + }, + "one action too many": { + mutate: func(p Plan) Plan { p.Actions = append(p.Actions, p.Actions[0]); return p }, + reason: "action count", + }, + "a NaN battery action": { + mutate: func(p Plan) Plan { p.Actions[3].BatteryW = math.NaN(); return p }, + reason: "non-finite", + }, + "an infinite battery action": { + mutate: func(p Plan) Plan { p.Actions[3].BatteryW = math.Inf(1); return p }, + reason: "non-finite", + }, + "discharging straight through the floor": { + mutate: func(p Plan) Plan { + for i := range p.Actions { + p.Actions[i].BatteryW = -4000 + } + return p + }, + reason: "outside", + }, + "charging straight through the ceiling": { + mutate: func(p Plan) Plan { + for i := range p.Actions { + p.Actions[i].BatteryW = 4000 + } + return p + }, + reason: "outside", + }, + "a battery action beyond the discharge limit": { + mutate: func(p Plan) Plan { p.Actions[2].BatteryW = -9000; return p }, + reason: "charge/discharge limits", + }, + "a battery action beyond the charge limit": { + mutate: func(p Plan) Plan { p.Actions[2].BatteryW = 9000; return p }, + reason: "charge/discharge limits", + }, + "actions planned for other slots": { + mutate: func(p Plan) Plan { p.Actions[5].SlotStartMs += 1000; return p }, + reason: "not the slot", + }, + "no battery to evaluate": { + params: func(p Params) Params { p.CapacityWh = 0; return p }, + reason: "capacity_wh", + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + plan := clonePlan(&good) + if tc.mutate != nil { + *plan = tc.mutate(*plan) + } + p := base + if tc.params != nil { + p = tc.params(p) + } + if _, ok := evaluatePlanOre(*plan, slots, p); ok { + t.Fatal("core costed a plan it should have refused") + } + _, err := evaluatePlan(*plan, slots, p) + if err == nil { + t.Fatal("refused without a reason") + } + if !strings.Contains(err.Error(), tc.reason) { + t.Fatalf("refusal reason %q does not mention %q", err, tc.reason) + } + }) + } + + // The unmutated plan still evaluates — otherwise the table above proves + // nothing about the mutations. + if _, ok := evaluatePlanOre(good, slots, base); !ok { + t.Fatal("the control plan was refused too") + } +} + +// TestEvaluatePlanRanksPlans — the evaluator has to order plans, not merely +// reproduce one number. Two hand-built action sequences over the same two +// slots: buy cheap and sell dear, or the reverse. +func TestEvaluatePlanRanksPlans(t *testing.T) { + slots := []Slot{ + {StartMs: 1_000_000, LenMin: 60, PriceOre: 100, SpotOre: 100, Confidence: 1, LoadW: 1000}, + {StartMs: 1_000_000 + 3_600_000, LenMin: 60, PriceOre: 400, SpotOre: 400, Confidence: 1, LoadW: 1000}, + } + p := Params{ + Mode: ModeArbitrage, CapacityWh: 10000, InitialSoC: 0.5, + SoCMin: 0.1, SoCMax: 1.0, SoCLevels: 101, ActionLevels: 41, + MaxChargeW: 2000, MaxDischargeW: 2000, + ChargeEfficiency: 1, DischargeEfficiency: 1, + } + build := func(first, second float64) Plan { + return Plan{Actions: []Action{ + {SlotStartMs: slots[0].StartMs, SlotLenMin: 60, BatteryW: first}, + {SlotStartMs: slots[1].StartMs, SlotLenMin: 60, BatteryW: second}, + }} + } + + // Charge 2 kW at 100 öre, discharge 2 kW at 400 öre. + // slot 0: grid = 1000 + 2000 = 3 kWh × 100 = 300 öre + // slot 1: grid = 1000 − 2000 = −1 kWh × 400 = −400 öre + smart, ok := evaluatePlanOre(build(2000, -2000), slots, p) + if !ok { + t.Fatal("the sensible plan was refused") + } + if math.Abs(smart-(-100)) > 1e-9 { + t.Fatalf("buy-low-sell-high costs %.6f öre, want −100", smart) + } + + // The same energy moved the wrong way round. + // slot 0: grid = 1000 − 2000 = −1 kWh × 100 = −100 öre + // slot 1: grid = 1000 + 2000 = 3 kWh × 400 = 1200 öre + silly, ok := evaluatePlanOre(build(-2000, 2000), slots, p) + if !ok { + t.Fatal("the wasteful plan was refused") + } + if math.Abs(silly-1100) > 1e-9 { + t.Fatalf("sell-low-buy-high costs %.6f öre, want 1100", silly) + } + if silly <= smart { + t.Fatalf("the wasteful plan (%v) did not cost more than the sensible one (%v)", silly, smart) + } + + // Doing nothing sits between them: 2 kWh of house load, 100 + 400. + idle, ok := evaluatePlanOre(build(0, 0), slots, p) + if !ok { + t.Fatal("the idle plan was refused") + } + if math.Abs(idle-500) > 1e-9 { + t.Fatalf("idling costs %.6f öre, want 500", idle) + } + if !(smart < idle && idle < silly) { + t.Fatalf("ranking broken: smart=%v idle=%v silly=%v", smart, idle, silly) + } +} + +// TestEvaluatePlanIgnoresTheCostAPlanClaims — the whole point. A challenger +// that reports a flattering total gets no credit for it. +func TestEvaluatePlanIgnoresTheCostAPlanClaims(t *testing.T) { + p := evaluationParams(ModeArbitrage) + slots := evaluationSlots(16) + plan := Optimize(slots, p) + honest, ok := evaluatePlanOre(plan, slots, p) + if !ok { + t.Fatal("refused") + } + plan.TotalCostOre = -999999 + for i := range plan.Actions { + plan.Actions[i].CostOre = -1 + } + plan.Actions[len(plan.Actions)-1].SoC = 1.0 + flattered, ok := evaluatePlanOre(plan, slots, p) + if !ok { + t.Fatal("refused after the totals were rewritten") + } + if flattered != honest { + t.Fatalf("rewriting the reported cost moved the evaluation: %v then %v", honest, flattered) + } +} diff --git a/go/internal/mpc/python_shadow.go b/go/internal/mpc/python_shadow.go index c5a7a4529..75e3b4919 100644 --- a/go/internal/mpc/python_shadow.go +++ b/go/internal/mpc/python_shadow.go @@ -4,6 +4,7 @@ import ( "context" "errors" "log/slog" + "math" "time" ) @@ -34,8 +35,24 @@ const ( // shadowErrWindowLimit bounds the suppression map. Errors carrying a // request id or timestamp would otherwise make every message distinct. shadowErrWindowLimit = 32 + // championEvalDriftOre is how far Core's valuation of its own plan may + // sit from the cost that plan reported before it counts as a bug rather + // than float residue. A 192-slot horizon accumulates far less than this. + championEvalDriftOre = 0.5 ) +// shadowRefusalReason names which side Core declined to cost, and why. +func shadowRefusalReason(championErr, shadowErr error) string { + switch { + case championErr != nil && shadowErr != nil: + return "core plan: " + championErr.Error() + "; python plan: " + shadowErr.Error() + case championErr != nil: + return "core plan: " + championErr.Error() + default: + return "python plan: " + shadowErr.Error() + } +} + type shadowErrWindow struct { openedAt time.Time suppressed int @@ -98,34 +115,75 @@ func (s *Service) runPythonShadow(ctx context.Context, cancel context.CancelFunc block := compareDPShadow(champion, shadow) block.ForecastBasis = "same downside input, python challenger" block.Solver = shadow.Solver - block.TotalCostOre = shadow.TotalCostOre - block.ActiveMinusShadowOre = champion.TotalCostOre - shadow.TotalCostOre + block.SelfReportedOre = shadow.TotalCostOre + if shadow.Solver != nil { + block.SelfReportedObjectiveOre = shadow.Solver.ObjectiveOre + } if block.FirstAction != nil { mode, _, _ := actionToSlot(*block.FirstAction, p.Mode) block.FirstAction.EMSMode = mode } - // Raw totals do not compare: a plan that ends the horizon fuller looks - // expensive while it is merely storing value. Correct both sides before - // the difference is written anywhere a human will read it. - championOre := terminalCorrectedOre(champion.TotalCostOre, planEndSoC(&champion), p) - shadowOre := terminalCorrectedOre(shadow.TotalCostOre, planEndSoC(&shadow), p) - block.ActiveTerminalCorrectedOre = championOre - block.TerminalCorrectedOre = shadowOre - block.ActiveMinusShadowTerminalCorrectedOre = championOre - shadowOre if block.Solver != nil && block.Solver.SolveMs == 0 { block.Solver.SolveMs = solveMs } - slog.Info("mpc: core champion vs python shadow", + // Both plans are costed by Core, not by whoever produced them. Doing it + // for the champion too is not ceremony: it keeps the subtraction + // symmetric, so the verdict cannot drift if the DP's own bookkeeping + // ever changes, and it is the only way the cross-check below exists. + championEval, championErr := evaluatePlan(champion, slots, p) + shadowEval, shadowErr := evaluatePlan(shadow, slots, p) + + base := []any{ "decision_id", champion.DecisionID, "reason", reason, - "core_cost_ore", champion.TotalCostOre, - "python_cost_ore", shadow.TotalCostOre, - "python_minus_core_ore_terminal_corrected", shadowOre-championOre, + "python_self_reported_ore", shadow.TotalCostOre, + "python_objective_ore", block.SelfReportedObjectiveOre, "python_solve_ms", solveMs, "mean_abs_battery_delta_w", block.MeanAbsBatteryDeltaW, "direction_disagreements", block.DirectionDisagreements, - "compared_slots", block.ComparedSlots) + "compared_slots", block.ComparedSlots, + } + if championErr != nil || shadowErr != nil { + block.EvaluationRefusedReason = shadowRefusalReason(championErr, shadowErr) + slog.Warn("mpc: core champion vs python shadow not scored", + append(base, "evaluation_refused", block.EvaluationRefusedReason)...) + s.recordPythonShadow(champion, slots, p, reason, replanAtMs, block) + return + } + + block.ActiveEvaluationDriftOre = championEval.CostOre - champion.TotalCostOre + if math.Abs(block.ActiveEvaluationDriftOre) > championEvalDriftOre { + slog.Warn("mpc: core's own plan does not cost what core reported", + "decision_id", champion.DecisionID, + "reported_ore", champion.TotalCostOre, + "evaluated_ore", championEval.CostOre, + "drift_ore", block.ActiveEvaluationDriftOre) + } + block.TotalCostOre = shadowEval.CostOre + block.ActiveMinusShadowOre = championEval.CostOre - shadowEval.CostOre + block.ActivePVCurtailmentSlots = championEval.CurtailedSlots + block.PVCurtailmentSlots = shadowEval.CurtailedSlots + + // Raw totals do not compare: a plan that ends the horizon fuller looks + // expensive while it is merely storing value. Correct both sides before + // the difference is written anywhere a human will read it. + championOre := terminalCorrectedOre(championEval.CostOre, championEval.EndSoC, p) + shadowOre := terminalCorrectedOre(shadowEval.CostOre, shadowEval.EndSoC, p) + block.ActiveTerminalCorrectedOre = championOre + block.TerminalCorrectedOre = shadowOre + block.ActiveMinusShadowTerminalCorrectedOre = championOre - shadowOre + + slog.Info("mpc: core champion vs python shadow", + append(base, + "core_cost_ore", championEval.CostOre, + "python_cost_ore", shadowEval.CostOre, + "core_cost_ore_terminal_corrected", championOre, + "python_cost_ore_terminal_corrected", shadowOre, + "python_minus_core_ore_terminal_corrected", shadowOre-championOre, + "core_evaluation_drift_ore", block.ActiveEvaluationDriftOre, + "pv_curtailment_slots_core", championEval.CurtailedSlots, + "pv_curtailment_slots_python", shadowEval.CurtailedSlots)...) s.recordPythonShadow(champion, slots, p, reason, replanAtMs, block) } diff --git a/go/internal/mpc/python_shadow_test.go b/go/internal/mpc/python_shadow_test.go index 660ce28fa..70528c05c 100644 --- a/go/internal/mpc/python_shadow_test.go +++ b/go/internal/mpc/python_shadow_test.go @@ -6,6 +6,7 @@ import ( "errors" "math" "path/filepath" + "strings" "sync/atomic" "testing" "time" @@ -13,29 +14,59 @@ import ( "github.com/srcfl/ftw/go/internal/state" ) -// costShadowOptimizer answers with the DP's own plan re-labelled as the -// external solver, then overrides the two numbers the comparison is made of. -// That keeps the plan structurally valid while the cost difference stays a -// hand-computable constant. +// costShadowOptimizer answers with the DP's own plan, optionally scaled into +// a genuinely different action sequence, and then claims whatever cost and +// end SoC the test asks it to. The claims are the point: a challenger's own +// numbers must not reach the verdict, because they are the value of ITS +// objective, not Core's price for its plan. type costShadowOptimizer struct { totalCostOre float64 + objectiveOre float64 endSoC float64 - calls atomic.Int32 + // chargeLastSlotW rewrites the final slot's battery power, making the + // challenger a genuinely different — and here deliberately worse — plan + // rather than the champion's own actions handed back. + chargeLastSlotW float64 + calls atomic.Int32 } func (o *costShadowOptimizer) Optimize(_ context.Context, slots []Slot, p Params) (Plan, error) { o.calls.Add(1) plan := Optimize(slots, p) + if n := len(plan.Actions); n > 0 && o.chargeLastSlotW != 0 { + plan.Actions[n-1].BatteryW = o.chargeLastSlotW + } plan.TotalCostOre = o.totalCostOre if n := len(plan.Actions); n > 0 { plan.Actions[n-1].SoC = o.endSoC } - plan.Solver = &SolverInfo{Engine: "cvxpy", Backend: "highs", Status: "optimal", SolveMs: 42} + plan.Solver = &SolverInfo{ + Engine: "cvxpy", Backend: "highs", Status: "optimal", SolveMs: 42, + ObjectiveOre: o.objectiveOre, + } return plan, nil } func (o *costShadowOptimizer) Close() error { return nil } +// refusedShadowOptimizer answers with a plan Core cannot cost: the battery +// drives straight through the floor. A real challenger doing this is a +// finding, not a number. +type refusedShadowOptimizer struct{ calls atomic.Int32 } + +func (o *refusedShadowOptimizer) Optimize(_ context.Context, slots []Slot, p Params) (Plan, error) { + o.calls.Add(1) + plan := Optimize(slots, p) + for i := range plan.Actions { + plan.Actions[i].BatteryW = -p.MaxDischargeW + } + plan.TotalCostOre = -100000 // and a flattering total to go with it + plan.Solver = &SolverInfo{Engine: "cvxpy", Backend: "highs", Status: "optimal"} + return plan, nil +} + +func (o *refusedShadowOptimizer) Close() error { return nil } + type failingShadowOptimizer struct{ calls atomic.Int32 } func (o *failingShadowOptimizer) Optimize(context.Context, []Slot, Params) (Plan, error) { @@ -175,10 +206,15 @@ func TestPrimaryFailureStillMarksTheDPPlanAsFallback(t *testing.T) { // TestPythonShadowRecordsTerminalCorrectedComparison is the soak instrument: // same inputs, one number per replan, on the Diagnostic that -// /api/mpc/diagnose/at hands out. +// /api/mpc/diagnose/at hands out. Both plans are costed by Core, so a +// challenger cannot move the verdict by reporting a flattering total of its +// own — which is exactly how a −97 öre field verdict once stood beside a +// +32 öre bench result on the same day's data. func TestPythonShadowRecordsTerminalCorrectedComparison(t *testing.T) { svc := shadowTestService(t) - shadow := &costShadowOptimizer{totalCostOre: 1234, endSoC: 0.75} + shadow := &costShadowOptimizer{ + totalCostOre: 1234, objectiveOre: 4321, endSoC: 0.75, chargeLastSlotW: 1000, + } svc.ShadowOptimizer = shadow var saved atomic.Int32 var withShadow atomic.Int32 @@ -210,30 +246,47 @@ func TestPythonShadowRecordsTerminalCorrectedComparison(t *testing.T) { if block.ComparedSlots != len(plan.Actions) || block.FirstAction == nil { t.Fatalf("comparison incomplete: %+v", block) } - if block.TotalCostOre != 1234 { - t.Fatalf("shadow raw cost = %v, want 1234", block.TotalCostOre) + if block.EvaluationRefusedReason != "" { + t.Fatalf("a feasible challenger was refused: %q", block.EvaluationRefusedReason) } - if got, want := block.ActiveMinusShadowOre, plan.TotalCostOre-1234; got != want { - t.Fatalf("raw core − python = %v, want %v", got, want) + + // What the challenger claimed is kept, and kept apart. + if block.SelfReportedOre != 1234 || block.SelfReportedObjectiveOre != 4321 { + t.Fatalf("self-reported figures = %v / %v, want 1234 / 4321", + block.SelfReportedOre, block.SelfReportedObjectiveOre) + } + // ...but none of it reaches the comparison. + if block.TotalCostOre == 1234 { + t.Fatal("the challenger's own total became the comparison") + } + svcParams := Params{TerminalSoCPrice: 200, CapacityWh: 10000} + if terminalCorrectedOre(1234, 0.75, svcParams) == block.TerminalCorrectedOre { + t.Fatal("the challenger's own end SoC and total set the corrected figure") } - // Hand value: corrected = raw − price·(SoC·capacity)/1000 - // = 1234 − 200·(0.75·10000)/1000 = 1234 − 1500 = −266. - if d.Params.TerminalSoCPrice != 200 || d.Params.CapacityWh != 10000 { - t.Fatalf("terminal economics moved: price=%v capacity=%v", - d.Params.TerminalSoCPrice, d.Params.CapacityWh) + // Core's own plan must cost what Core said it costs. + if math.Abs(block.ActiveEvaluationDriftOre) > 1e-9 { + t.Fatalf("core's plan drifted from its own bookkeeping by %v öre", block.ActiveEvaluationDriftOre) + } + if got, want := block.ActiveTerminalCorrectedOre, + terminalCorrectedOre(plan.TotalCostOre, planEndSoC(plan), svcParams); math.Abs(got-want) > 1e-9 { + t.Fatalf("champion corrected = %v, want %v", got, want) } - wantShadow := -266.0 - wantChampion := terminalCorrectedOre(plan.TotalCostOre, - plan.Actions[len(plan.Actions)-1].SoC, Params{TerminalSoCPrice: 200, CapacityWh: 10000}) - if block.TerminalCorrectedOre != wantShadow { - t.Fatalf("shadow corrected = %v, want %v", block.TerminalCorrectedOre, wantShadow) + + // The challenger really is a different plan, so the verdict is not zero. + if block.MeanAbsBatteryDeltaW == 0 { + t.Fatal("the challenger returned the champion's own actions") } - if block.ActiveTerminalCorrectedOre != wantChampion { - t.Fatalf("champion corrected = %v, want %v", block.ActiveTerminalCorrectedOre, wantChampion) + wantDiff := block.ActiveTerminalCorrectedOre - block.TerminalCorrectedOre + if got := block.ActiveMinusShadowTerminalCorrectedOre; math.Abs(got-wantDiff) > 1e-9 { + t.Fatalf("corrected difference = %v, want %v", got, wantDiff) } - if got := block.ActiveMinusShadowTerminalCorrectedOre; got != wantChampion-wantShadow { - t.Fatalf("corrected difference = %v, want %v", got, wantChampion-wantShadow) + if got, want := block.ActiveMinusShadowOre, plan.TotalCostOre-block.TotalCostOre; math.Abs(got-want) > 1e-9 { + t.Fatalf("raw core − python = %v, want %v", got, want) + } + // The DP is optimal on these inputs, so a halved challenger costs more. + if block.ActiveMinusShadowTerminalCorrectedOre >= 0 { + t.Fatalf("a worse challenger scored as cheap: %v", block.ActiveMinusShadowTerminalCorrectedOre) } if saved.Load() != 2 || withShadow.Load() != 1 { t.Fatalf("diagnostic writes = %d (%d carrying the shadow), want the replan's write plus one rewrite", @@ -241,6 +294,47 @@ func TestPythonShadowRecordsTerminalCorrectedComparison(t *testing.T) { } } +// TestPythonShadowRefusesToScoreAnInfeasiblePlan — a challenger Core cannot +// cost produces a recorded reason and no difference at all. Half a +// measurement is worse than none: it looks exactly like a whole one. +func TestPythonShadowRefusesToScoreAnInfeasiblePlan(t *testing.T) { + svc := shadowTestService(t) + svc.ShadowOptimizer = &refusedShadowOptimizer{} + svc.SaveDiag = func(*Diagnostic, string) error { return nil } + + plan := svc.Replan(context.Background()) + if plan == nil { + t.Fatal("no champion plan") + } + waitFor(t, "the refused shadow to land", func() bool { + d := svc.Diagnose() + return d != nil && d.PythonShadow != nil + }) + + block := svc.Diagnose().PythonShadow + if block.EvaluationRefusedReason == "" { + t.Fatalf("an unscoreable plan was scored anyway: %+v", block) + } + if !strings.Contains(block.EvaluationRefusedReason, "python plan") { + t.Fatalf("refusal does not name the side that failed: %q", block.EvaluationRefusedReason) + } + for name, got := range map[string]float64{ + "total_cost_ore": block.TotalCostOre, + "active_minus_shadow_ore": block.ActiveMinusShadowOre, + "terminal_corrected_ore": block.TerminalCorrectedOre, + "active_terminal_corrected_ore": block.ActiveTerminalCorrectedOre, + "active_minus_shadow_terminal_corrected_ore": block.ActiveMinusShadowTerminalCorrectedOre, + } { + if got != 0 { + t.Fatalf("%s = %v on a refused comparison, want no number at all", name, got) + } + } + // The challenger's own claim is still recorded — it is the evidence. + if block.SelfReportedOre != -100000 { + t.Fatalf("self-reported cost = %v, want the claim that was refused", block.SelfReportedOre) + } +} + // TestPythonShadowFailureLeavesTheChampionAlone — a broken challenger costs // the site nothing but a log line. func TestPythonShadowFailureLeavesTheChampionAlone(t *testing.T) { diff --git a/go/internal/mpc/replay_bench_test.go b/go/internal/mpc/replay_bench_test.go index c53e3e5fe..46e4b98af 100644 --- a/go/internal/mpc/replay_bench_test.go +++ b/go/internal/mpc/replay_bench_test.go @@ -61,15 +61,42 @@ func loadDiagnosticBlob(data []byte) (*Diagnostic, error) { // field shadow reports the same correction every replan, so the bench // and the running planner must not drift apart on the formula. +// benchCorrectedOre costs one plan under Core's model and terminal-corrects +// it. A plan Core cannot cost fails the bench outright rather than being +// silently replaced by the number it came with — the same rule the field +// shadow follows. +func benchCorrectedOre(t *testing.T, leg, snapshot string, plan *Plan, slots []Slot, p Params) float64 { + t.Helper() + eval, err := evaluatePlan(*plan, slots, p) + if err != nil { + t.Fatalf("%s: core cannot cost the %s plan: %v", snapshot, leg, err) + } + return terminalCorrectedOre(eval.CostOre, eval.EndSoC, p) +} + // TestReplayBenchSnapshots is the A/B instrument. Skipped without // FTW_MPC_SNAPSHOT_DIR. Optional knobs: // -// FTW_TEST_OPTIMIZER_PYTHON — adds the Python champion leg +// FTW_TEST_OPTIMIZER_PYTHON — adds the Python challenger leg // FTW_MPC_BENCH_SPREAD_ORE — MinArbitrageSpreadOreKwh fallback for // blobs written before the diagnostic // persisted it. A spread carried by the // snapshot always wins: it is what the // replan actually solved under. +// FTW_MPC_BENCH_CVAR_WEIGHT — the site's optimizer.cvar_weight +// FTW_MPC_BENCH_CVAR_ALPHA — the site's optimizer.cvar_alpha +// FTW_MPC_BENCH_MIP_GAP — the site's optimizer.mip_rel_gap +// +// The three optimizer knobs exist because the diagnostic records the +// PLANNING inputs, not the challenger's solver configuration: replaying a +// box's snapshot with the bench's defaults asks Python a different question +// than the box asked it, and the answer differs by more than the gap being +// measured. Set them to the site's values before reading a py column as "what +// that box's shadow would have said". +// +// py_self is what Python reported for its own plan; py_corr is what that same +// plan costs under Core's model, terminal-corrected. Only py_corr may be +// subtracted from dp_corr. // // Run with -v; the verdict is the table, not a pass/fail. func TestReplayBenchSnapshots(t *testing.T) { @@ -90,6 +117,15 @@ func TestReplayBenchSnapshots(t *testing.T) { } } + benchFloat := func(key string, fallback float64) float64 { + if v := os.Getenv(key); v != "" { + if f, err := strconv.ParseFloat(v, 64); err == nil { + return f + } + } + return fallback + } + var ext *ExternalOptimizer if python := os.Getenv("FTW_TEST_OPTIMIZER_PYTHON"); python != "" { _, file, _, ok := runtime.Caller(0) @@ -97,13 +133,19 @@ func TestReplayBenchSnapshots(t *testing.T) { t.Fatal("runtime.Caller failed") } moduleDir := filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", "..", "optimizer")) + cvarWeight := benchFloat("FTW_MPC_BENCH_CVAR_WEIGHT", 0) + cvarAlpha := benchFloat("FTW_MPC_BENCH_CVAR_ALPHA", 0.9) + mipGap := benchFloat("FTW_MPC_BENCH_MIP_GAP", 0.001) + t.Logf("python leg: cvar_weight=%g cvar_alpha=%g mip_rel_gap=%g", cvarWeight, cvarAlpha, mipGap) ext, err = NewExternalOptimizer(ExternalOptimizerConfig{ Command: []string{python, "-m", "ftw_optimizer.worker"}, ModuleDir: moduleDir, Timeout: 60 * time.Second, Solver: "HIGHS", Formulation: "auto", - MIPRelGap: 0.001, + MIPRelGap: mipGap, + CVaRWeight: cvarWeight, + CVaRAlpha: cvarAlpha, IdleTimeout: 5 * time.Second, }) if err != nil { @@ -112,8 +154,8 @@ func TestReplayBenchSnapshots(t *testing.T) { defer ext.Close() } - t.Logf("%-15s %-18s %10s %10s %10s %10s %10s %10s", - "snapshot", "mode", "rec_corr", "dp_corr", "dp-rec", "py_corr", "py-dp", "dp_ms") + t.Logf("%-15s %-18s %10s %10s %10s %10s %10s %10s %10s", + "snapshot", "mode", "rec_corr", "dp_corr", "dp-rec", "py_self", "py_corr", "py-dp", "dp_ms") var sumDPvsRec, sumPYvsDP float64 var nDP, nPY int for _, path := range paths { @@ -149,38 +191,51 @@ func TestReplayBenchSnapshots(t *testing.T) { } } - recCorr := terminalCorrectedOre(recorded.TotalCostOre, planEndSoC(recorded), params) + // Every leg is costed by Core, on Core's terms. A solver's own + // total is its objective's opinion of its own plan; subtracting + // one solver's objective from another's measures the objectives, + // not the plans (#1020 follow-up). + recCorr := benchCorrectedOre(t, "recorded", filepath.Base(path), recorded, slots, params) dpStart := time.Now() dpPlan := Optimize(slots, params) dpMs := time.Since(dpStart).Milliseconds() - dpCorr := terminalCorrectedOre(dpPlan.TotalCostOre, planEndSoC(&dpPlan), params) + dpCorr := benchCorrectedOre(t, "dp", filepath.Base(path), &dpPlan, slots, params) sumDPvsRec += dpCorr - recCorr nDP++ - pyCol, pyDelta := "-", "-" + pySelf, pyCol, pyDelta := "-", "-", "-" if ext != nil { pyPlan, err := ext.Optimize(t.Context(), slots, params) - if err != nil { + switch { + case err != nil: pyCol = "ERR" t.Logf("%-15s python: %v", filepath.Base(path), err) - } else { - pyCorr := terminalCorrectedOre(pyPlan.TotalCostOre, planEndSoC(&pyPlan), params) + default: + pySelf = fmt.Sprintf("%10.1f", + terminalCorrectedOre(pyPlan.TotalCostOre, planEndSoC(&pyPlan), params)) + pyEval, evalErr := evaluatePlan(pyPlan, slots, params) + if evalErr != nil { + pyCol = "REFUSED" + t.Logf("%-15s python plan not costable by core: %v", filepath.Base(path), evalErr) + break + } + pyCorr := terminalCorrectedOre(pyEval.CostOre, pyEval.EndSoC, params) pyCol = fmt.Sprintf("%10.1f", pyCorr) pyDelta = fmt.Sprintf("%10.1f", pyCorr-dpCorr) sumPYvsDP += pyCorr - dpCorr nPY++ } } - t.Logf("%-15s %-18s %10.1f %10.1f %10.1f %10s %10s %8dms", + t.Logf("%-15s %-18s %10.1f %10.1f %10.1f %10s %10s %10s %8dms", filepath.Base(path), string(params.Mode), - recCorr, dpCorr, dpCorr-recCorr, pyCol, pyDelta, dpMs) + recCorr, dpCorr, dpCorr-recCorr, pySelf, pyCol, pyDelta, dpMs) } if nDP > 0 { t.Logf("SUMMARY dp_vs_recorded: n=%d mean=%.1f öre/plan", nDP, sumDPvsRec/float64(nDP)) } if nPY > 0 { - t.Logf("SUMMARY python_vs_dp (terminal-corrected, positive = python costs more): n=%d mean=%.1f öre/plan", nPY, sumPYvsDP/float64(nPY)) + t.Logf("SUMMARY python_vs_dp (both costed by core, terminal-corrected, positive = python costs more): n=%d mean=%.1f öre/plan", nPY, sumPYvsDP/float64(nPY)) } } From f5e9264a31c9893dbcfa4bcc9ffefa59d6c4fcb8 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 31 Aug 2026 17:03:04 +0200 Subject: [PATCH 2/3] test(mpc): sweep the terminal price, and report end SoC per plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shadow gap had to be attributable before anyone acted on it. The bench now scales each snapshot's TerminalSoCPrice through FTW_MPC_BENCH_TERMINAL_SCALE, applied before anything solves so the DP, Python and the correction all price stored energy the same way, and prints raw totals and end SoC for both plans beside the corrected ones. That is what the sweep needed, and the sweep settles the question: at a common 201x401 grid with no terminal credit, python - dp is -7.0 öre/plan across 13 snapshots, and on the only snapshot recorded under today's Core it is +39.4 in Core's favour. The -51.5 öre mean measured at each blob's own recorded resolution was the old 41x81 grid the Python era wrote, replayed faithfully. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C1d4uknzs7QUv7nE9rW4fi --- go/internal/mpc/replay_bench_test.go | 73 ++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 19 deletions(-) diff --git a/go/internal/mpc/replay_bench_test.go b/go/internal/mpc/replay_bench_test.go index 46e4b98af..70345a0ac 100644 --- a/go/internal/mpc/replay_bench_test.go +++ b/go/internal/mpc/replay_bench_test.go @@ -61,17 +61,19 @@ func loadDiagnosticBlob(data []byte) (*Diagnostic, error) { // field shadow reports the same correction every replan, so the bench // and the running planner must not drift apart on the formula. -// benchCorrectedOre costs one plan under Core's model and terminal-corrects -// it. A plan Core cannot cost fails the bench outright rather than being -// silently replaced by the number it came with — the same rule the field -// shadow follows. -func benchCorrectedOre(t *testing.T, leg, snapshot string, plan *Plan, slots []Slot, p Params) float64 { +// benchEvaluate costs one plan under Core's model. A plan Core cannot cost +// fails the bench outright rather than being silently replaced by the number +// it came with — the same rule the field shadow follows. The caller +// terminal-corrects; the raw cost and the reached end SoC are reported +// alongside so a reader can see whether an advantage is cheaper decisions or +// merely more energy left in the pack. +func benchEvaluate(t *testing.T, leg, snapshot string, plan *Plan, slots []Slot, p Params) planEvaluation { t.Helper() eval, err := evaluatePlan(*plan, slots, p) if err != nil { t.Fatalf("%s: core cannot cost the %s plan: %v", snapshot, leg, err) } - return terminalCorrectedOre(eval.CostOre, eval.EndSoC, p) + return eval } // TestReplayBenchSnapshots is the A/B instrument. Skipped without @@ -86,6 +88,9 @@ func benchCorrectedOre(t *testing.T, leg, snapshot string, plan *Plan, slots []S // FTW_MPC_BENCH_CVAR_WEIGHT — the site's optimizer.cvar_weight // FTW_MPC_BENCH_CVAR_ALPHA — the site's optimizer.cvar_alpha // FTW_MPC_BENCH_MIP_GAP — the site's optimizer.mip_rel_gap +// FTW_MPC_BENCH_TERMINAL_SCALE — multiplier on the snapshot's +// TerminalSoCPrice: 1.0 as recorded, 0 for no +// terminal credit at all. // // The three optimizer knobs exist because the diagnostic records the // PLANNING inputs, not the challenger's solver configuration: replaying a @@ -94,9 +99,19 @@ func benchCorrectedOre(t *testing.T, leg, snapshot string, plan *Plan, slots []S // measured. Set them to the site's values before reading a py column as "what // that box's shadow would have said". // +// The terminal knob answers a different question: whether a measured gap is a +// gap between plans or an artifact of how stored energy is priced. The +// terminal price is both an input the solvers optimize against and the rate +// the comparison credits leftover charge at, so the scale moves both together +// — every leg still solves and is scored under one consistent price of stored +// energy. If a challenger's lead melts as the scale goes to 0, the lead was +// bought with end-of-horizon SoC, not with cheaper decisions. +// // py_self is what Python reported for its own plan; py_corr is what that same // plan costs under Core's model, terminal-corrected. Only py_corr may be -// subtracted from dp_corr. +// subtracted from dp_corr. The *_raw columns are the same plans before the +// correction and *_soc the SoC each parks the horizon at, which is what makes +// the difference between the two readable. // // Run with -v; the verdict is the table, not a pass/fail. func TestReplayBenchSnapshots(t *testing.T) { @@ -154,9 +169,16 @@ func TestReplayBenchSnapshots(t *testing.T) { defer ext.Close() } - t.Logf("%-15s %-18s %10s %10s %10s %10s %10s %10s %10s", - "snapshot", "mode", "rec_corr", "dp_corr", "dp-rec", "py_self", "py_corr", "py-dp", "dp_ms") - var sumDPvsRec, sumPYvsDP float64 + terminalScale := benchFloat("FTW_MPC_BENCH_TERMINAL_SCALE", 1) + if terminalScale != 1 { + t.Logf("terminal price scaled by %g (solve and scoring alike)", terminalScale) + } + + t.Logf("%-15s %-18s %9s %9s %6s %9s %9s %9s %9s %6s %9s %9s %9s %8s", + "snapshot", "mode", "rec_corr", + "dp_raw", "dp_soc", "dp_corr", "dp-rec", + "py_self", "py_raw", "py_soc", "py_corr", "py-dp", "py-dp_raw", "dp_ms") + var sumDPvsRec, sumPYvsDP, sumPYvsDPRaw, sumSoCDiff float64 var nDP, nPY int for _, path := range paths { data, err := os.ReadFile(path) @@ -176,6 +198,9 @@ func TestReplayBenchSnapshots(t *testing.T) { if params.MinArbitrageSpreadOreKwh == 0 { params.MinArbitrageSpreadOreKwh = fallbackSpreadOre } + // Applied before anything solves, so the DP, Python and the + // correction all price stored energy the same way. + params.TerminalSoCPrice *= terminalScale // A blob carries the RECORDED replan's grid resolution, so a // default bump would be invisible here without an override — // judging a resolution change is exactly what the knobs exist @@ -195,16 +220,18 @@ func TestReplayBenchSnapshots(t *testing.T) { // total is its objective's opinion of its own plan; subtracting // one solver's objective from another's measures the objectives, // not the plans (#1020 follow-up). - recCorr := benchCorrectedOre(t, "recorded", filepath.Base(path), recorded, slots, params) + recEval := benchEvaluate(t, "recorded", filepath.Base(path), recorded, slots, params) + recCorr := terminalCorrectedOre(recEval.CostOre, recEval.EndSoC, params) dpStart := time.Now() dpPlan := Optimize(slots, params) dpMs := time.Since(dpStart).Milliseconds() - dpCorr := benchCorrectedOre(t, "dp", filepath.Base(path), &dpPlan, slots, params) + dpEval := benchEvaluate(t, "dp", filepath.Base(path), &dpPlan, slots, params) + dpCorr := terminalCorrectedOre(dpEval.CostOre, dpEval.EndSoC, params) sumDPvsRec += dpCorr - recCorr nDP++ - pySelf, pyCol, pyDelta := "-", "-", "-" + pySelf, pyRaw, pySoC, pyCol, pyDelta, pyDeltaRaw := "-", "-", "-", "-", "-", "-" if ext != nil { pyPlan, err := ext.Optimize(t.Context(), slots, params) switch { @@ -212,7 +239,7 @@ func TestReplayBenchSnapshots(t *testing.T) { pyCol = "ERR" t.Logf("%-15s python: %v", filepath.Base(path), err) default: - pySelf = fmt.Sprintf("%10.1f", + pySelf = fmt.Sprintf("%9.1f", terminalCorrectedOre(pyPlan.TotalCostOre, planEndSoC(&pyPlan), params)) pyEval, evalErr := evaluatePlan(pyPlan, slots, params) if evalErr != nil { @@ -221,21 +248,29 @@ func TestReplayBenchSnapshots(t *testing.T) { break } pyCorr := terminalCorrectedOre(pyEval.CostOre, pyEval.EndSoC, params) - pyCol = fmt.Sprintf("%10.1f", pyCorr) - pyDelta = fmt.Sprintf("%10.1f", pyCorr-dpCorr) + pyRaw = fmt.Sprintf("%9.1f", pyEval.CostOre) + pySoC = fmt.Sprintf("%6.3f", pyEval.EndSoC) + pyCol = fmt.Sprintf("%9.1f", pyCorr) + pyDelta = fmt.Sprintf("%9.1f", pyCorr-dpCorr) + pyDeltaRaw = fmt.Sprintf("%9.1f", pyEval.CostOre-dpEval.CostOre) sumPYvsDP += pyCorr - dpCorr + sumPYvsDPRaw += pyEval.CostOre - dpEval.CostOre + sumSoCDiff += pyEval.EndSoC - dpEval.EndSoC nPY++ } } - t.Logf("%-15s %-18s %10.1f %10.1f %10.1f %10s %10s %10s %8dms", - filepath.Base(path), string(params.Mode), - recCorr, dpCorr, dpCorr-recCorr, pySelf, pyCol, pyDelta, dpMs) + t.Logf("%-15s %-18s %9.1f %9.1f %6.3f %9.1f %9.1f %9s %9s %6s %9s %9s %9s %6dms", + filepath.Base(path), string(params.Mode), recCorr, + dpEval.CostOre, dpEval.EndSoC, dpCorr, dpCorr-recCorr, + pySelf, pyRaw, pySoC, pyCol, pyDelta, pyDeltaRaw, dpMs) } if nDP > 0 { t.Logf("SUMMARY dp_vs_recorded: n=%d mean=%.1f öre/plan", nDP, sumDPvsRec/float64(nDP)) } if nPY > 0 { t.Logf("SUMMARY python_vs_dp (both costed by core, terminal-corrected, positive = python costs more): n=%d mean=%.1f öre/plan", nPY, sumPYvsDP/float64(nPY)) + t.Logf("SUMMARY python_vs_dp_raw (no terminal correction): n=%d mean=%.1f öre/plan; mean end_soc diff (py-dp) = %+.4f (%+.1f Wh)", + nPY, sumPYvsDPRaw/float64(nPY), sumSoCDiff/float64(nPY), sumSoCDiff/float64(nPY)*9600) } } From 2a8686f88bb00ac2fc18e66bad752b53f8f0e2f4 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 31 Aug 2026 17:34:42 +0200 Subject: [PATCH 3/3] test(mpc): measure what the price twin changes about the next 15 minutes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spot prices publish around 13:00. Before that the box holds roughly 12 hours of real day-ahead price; the rest of the 48 h horizon is an ML twin, blended toward the horizon mean by confidence. The owner asked whether guessing tomorrow earns anything. MPC dispatches only the first slot, so the question is not whether the guess moves a plan 30 hours out — it is whether it moves the watt that reaches hardware now. Two bench tests re-solve recorded /api/mpc/diagnose snapshots three ways on one grid: as recorded, with every forecast slot flattened to the horizon mean, and with the forecast slots deleted. They report the first slot's battery power, how many hours the plans stay together, and cost over the KNOWN window only, so all three are scored on the same real prices instead of one being judged on 48 h of guesses. The flattened variant uses confidence 1e-9, not 0. Core reads confidence <= 0 as "caller did not fill this in" and coerces it to 1.0, twice, so a variant written with a zero would silently measure total trust — the opposite of what it claims. A fixture test pins that coercion so the epsilon is not mistaken for fussiness. A second bench shortens the known window by hand — 2, 4, 6, 8, 12 h — leaving prices untouched, so it isolates the confidence rule from the twin's accuracy and finds where the guess starts to matter. Bench only; no planner code moves. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C1d4uknzs7QUv7nE9rW4fi --- .changeset/shadow-costed-by-core.md | 7 + go/internal/mpc/forecast_value_bench_test.go | 798 +++++++++++++++++++ 2 files changed, 805 insertions(+) create mode 100644 go/internal/mpc/forecast_value_bench_test.go diff --git a/.changeset/shadow-costed-by-core.md b/.changeset/shadow-costed-by-core.md index 652199890..4188984e7 100644 --- a/.changeset/shadow-costed-by-core.md +++ b/.changeset/shadow-costed-by-core.md @@ -28,3 +28,10 @@ a snapshot records the planning inputs but not the site's solver settings, so replaying one with the bench's defaults asks the challenger a different question than the box asked — worth more than 130 öre per plan on a summer day with PV uncertainty, which is larger than the gap being measured. + +A second bench asks what the price twin is worth. It re-solves each snapshot +three ways — the confidences the box used, those same forecast slots +flattened to the horizon mean, and the forecast slots deleted — and reports +the only number that reaches hardware: the first slot's battery power. On +every snapshot recorded so far the three agree exactly, and they keep +agreeing until the known window falls under four hours. diff --git a/go/internal/mpc/forecast_value_bench_test.go b/go/internal/mpc/forecast_value_bench_test.go new file mode 100644 index 000000000..03de970dd --- /dev/null +++ b/go/internal/mpc/forecast_value_bench_test.go @@ -0,0 +1,798 @@ +package mpc + +// Forecast-value bench: does guessing tomorrow's price change what the box +// does RIGHT NOW? +// +// Spot prices publish around 13:00 local. Before that the box holds perhaps +// 12 hours of real day-ahead prices; after, up to 36. The rest of the 48 h +// horizon is an ML price twin, marked Confidence < 1.0, and the DP blends it +// toward the horizon mean: +// +// effPrice(slot) = confidence × rawPrice + (1 − confidence) × horizonMean +// +// MPC dispatches only the FIRST slot, so the decisive question is not whether +// the guess moves a plan 30 hours out. It is whether it moves the watt that +// reaches hardware in the next 15 minutes. This bench re-solves each recorded +// snapshot three ways on one grid and reports that watt: +// +// A as recorded — the confidences the box actually used +// B flat guess — every forecast slot pinned to the horizon mean +// C truncated — the guessed slots deleted, solve the known window only +// +// Two honest limits on what B and C mean, both worth stating before reading +// any number below: +// +// - B does not erase the guess, it flattens it. horizonMeans() averages +// over every slot, forecast rows included, so B still inherits the LEVEL +// the twin predicted — it discards only the SHAPE. That is exactly what +// Confidence → 0 does inside the DP today, which is the thing being +// measured. +// - C deletes the guessed slots from the solve but still runs at the +// terminal price the box derived from the FULL horizon. A box that truly +// knew nothing past day-ahead would price stored energy off the known +// window alone, so the sweep below also reports C at a terminal price +// rederived from the known slots, using Core's own mode-dependent +// formula. +// +// And the limit on the whole exercise: these snapshots carry no ground truth +// about what prices turned out to be. This bench can say whether the guess +// changes the action. It cannot say which action earned more money. That +// needs a closed-loop backtest against realised prices. +// +// Skipped without FTW_MPC_SNAPSHOT_DIR; the snapshot directory stays outside +// the repository because real blobs carry a household's load traces. + +import ( + "fmt" + "math" + "os" + "path/filepath" + "slices" + "sort" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/state" +) + +// benchNoTrustConfidence is the "trust nothing" confidence for variant B. +// +// It is not 0. Confidence ≤ 0 means "caller did not fill this in" to Core and +// is coerced to 1.0 — twice, once in sanitizeOptimizeSlots and again in +// Optimize's own defaulting loop — so asking for zero trust with a zero would +// silently ask for TOTAL trust, the exact opposite of the variant. The +// smallest value that survives both coercions is any positive float; 1e-9 +// puts effPrice within ~1e-7 öre of the horizon mean, which is far below the +// öre the DP can act on. +const benchNoTrustConfidence = 1e-9 + +// benchActionDiffW is how far two plans' battery power must part before the +// slot counts as a disagreement. 50 W is well under the resolution any +// inverter tracks and well over DP grid residue. +const benchActionDiffW = 50.0 + +// benchLastKnownSlot reports the index of the last confidence-1.0 slot and +// whether the known slots form a contiguous prefix. A non-contiguous known +// window would make "truncate to what we know" ill-defined, so the caller +// says so rather than quietly truncating at a hole. +func benchLastKnownSlot(slots []Slot) (last int, contiguous bool, count int) { + last = -1 + for i, s := range slots { + if s.Confidence >= 1.0 { + last = i + count++ + } + } + if last < 0 { + return -1, false, 0 + } + return last, count == last+1, count +} + +// benchFlattenForecast returns a copy of slots with every slot the box was +// unsure about pinned to no-trust confidence. Known slots are untouched. +func benchFlattenForecast(slots []Slot) []Slot { + out := make([]Slot, len(slots)) + copy(out, slots) + for i := range out { + if out[i].Confidence < 1.0 { + out[i].Confidence = benchNoTrustConfidence + } + } + return out +} + +// benchHoursTo is the wall-clock offset of slot index i, so a divergence +// index can be read as "the guess stops mattering after N hours". +func benchHoursTo(slots []Slot, i int) float64 { + var min float64 + for k := 0; k < i && k < len(slots); k++ { + min += float64(slots[k].LenMin) + } + return min / 60.0 +} + +// benchFirstDivergence finds the first slot where two plans' battery power +// parts by more than benchActionDiffW, comparing only as far as the shorter +// plan reaches. ok is false when they agree the whole way. +func benchFirstDivergence(a, b *Plan) (idx int, ok bool) { + n := len(a.Actions) + if len(b.Actions) < n { + n = len(b.Actions) + } + for i := 0; i < n; i++ { + if math.Abs(a.Actions[i].BatteryW-b.Actions[i].BatteryW) > benchActionDiffW { + return i, true + } + } + return n, false +} + +// benchFirstSlotW is the only number in this file that reaches hardware. +func benchFirstSlotW(p *Plan) float64 { + if p == nil || len(p.Actions) == 0 { + return math.NaN() + } + return p.Actions[0].BatteryW +} + +// benchKnownWindowCost costs a plan over the KNOWN slots only, under Core's +// model, terminal-corrected at the scoring price. +// +// All three variants must be scored on the same real-price slots. Costing A +// over 48 h of guessed prices and C over 13 h of real ones compares two +// different questions and would make the shorter horizon look cheap for no +// reason but being shorter. +func benchKnownWindowCost(plan *Plan, slots []Slot, n int, solveP, scoreP Params) (corrected, raw, endSoC float64, err error) { + if plan == nil || len(plan.Actions) < n { + return 0, 0, 0, fmt.Errorf("plan has %d actions, need %d", len(plan.Actions), n) + } + trunc := Plan{Actions: plan.Actions[:n]} + eval, err := evaluatePlan(trunc, slots[:n], solveP) + if err != nil { + return 0, 0, 0, err + } + return terminalCorrectedOre(eval.CostOre, eval.EndSoC, scoreP), eval.CostOre, eval.EndSoC, nil +} + +// benchTerminalPrice rederives the terminal credit from a given set of slots +// using Core's own mode-dependent formula (service.go), so "C, knowing +// nothing beyond day-ahead" can price stored energy off the known window +// instead of inheriting a number computed from the guess. +func benchTerminalPrice(mode Mode, slots []Slot) float64 { + prices := make([]state.PricePoint, 0, len(slots)) + for _, s := range slots { + prices = append(prices, state.PricePoint{ + SlotTsMs: s.StartMs, + SlotLenMin: s.LenMin, + SpotOreKwh: s.SpotOre, + TotalOreKwh: s.PriceOre, + }) + } + switch mode { + case ModeSelfConsumption, ModeCheapCharge, ModePassiveArbitrage: + return selfConsumptionTerminalPrice(prices, 0, 0) + default: + return upperHalfMeanPrice(prices) + } +} + +type forecastVariant struct { + name string + plan *Plan + firstW float64 + corrected float64 + raw float64 + endSoC float64 + costErr error + solveMs int64 +} + +// TestForecastValueBench answers the owner's question with numbers: does the +// price twin change the watt dispatched now, and for how many hours does the +// guess keep mattering? Run with -v; the verdict is the table. +// +// Knobs: +// +// FTW_MPC_SNAPSHOT_DIR — directory of /api/mpc/diagnose blobs (required) +// FTW_MPC_FORECAST_SOC — SoC grid (default 201) +// FTW_MPC_FORECAST_ACTIONS — action grid (default 401) +// +// Both grids are forced so every variant answers on one resolution: a blob +// carries the recorded replan's grid, and the older blobs were recorded at a +// coarser one under a different planner. +func TestForecastValueBench(t *testing.T) { + dir := os.Getenv("FTW_MPC_SNAPSHOT_DIR") + if dir == "" { + t.Skip("FTW_MPC_SNAPSHOT_DIR not set") + } + paths, err := filepath.Glob(filepath.Join(dir, "*.json")) + if err != nil || len(paths) == 0 { + t.Fatalf("no snapshots in %q (err=%v)", dir, err) + } + sort.Strings(paths) + + socLevels, actionLevels := 201, 401 + if v := os.Getenv("FTW_MPC_FORECAST_SOC"); v != "" { + fmt.Sscanf(v, "%d", &socLevels) + } + if v := os.Getenv("FTW_MPC_FORECAST_ACTIONS"); v != "" { + fmt.Sscanf(v, "%d", &actionLevels) + } + t.Logf("grid forced to SoCLevels=%d ActionLevels=%d for every variant", socLevels, actionLevels) + t.Logf("A = as recorded; B = forecast slots flattened to the horizon mean (confidence %g);"+ + " C = forecast slots deleted, known window only", benchNoTrustConfidence) + + terminalScales := []float64{0.5, 1.0, 1.5} + + t.Logf("") + t.Logf("=== decision table: the first slot is the only number that reaches hardware ===") + t.Logf("%-22s %-18s %7s %9s %9s %9s %9s %9s %8s %8s", + "snapshot", "mode", "known_h", "rec_w", "A_w", "B_w", "C_w", "B-A_w", "C-A_w", "A_soc0") + + type row struct { + name string + live bool + mode Mode + knownH float64 + knownN int + recW, aW, bW, cW float64 + divABIdx, divACIdx int + divABOK, divACOK bool + divABH, divACH float64 + aCorr, bCorr, cCorr float64 + aRaw, bRaw, cRaw float64 + aSoC, bSoC, cSoC float64 + terminalOre, knownTermOre float64 + knownMeanOre float64 + sweepW map[float64]float64 + sweepCorr map[float64]float64 + sweepSoC map[float64]float64 + sweepDivIdx int + sweepDivOK bool + knownTermW, knownTermCorr float64 + knownTermSoC float64 + diffSlotsAB, diffSlotsAC int + costable bool + } + var rows []row + + for _, path := range paths { + name := filepath.Base(path) + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + d, err := loadDiagnosticBlob(data) + if err != nil { + t.Logf("%-22s SKIP: %v", name, err) + continue + } + recorded, slots, params, _, ok := planFromDiagnostic(d) + if !ok { + t.Logf("%-22s SKIP: not rehydratable", name) + continue + } + params.SoCLevels = socLevels + params.ActionLevels = actionLevels + + lastKnown, contiguous, knownCount := benchLastKnownSlot(slots) + if lastKnown < 0 { + t.Logf("%-22s SKIP: no confidence-1.0 slot, nothing is known", name) + continue + } + if !contiguous { + t.Logf("%-22s WARN: known slots are not a contiguous prefix (%d known, last at %d);"+ + " C truncates at the last one anyway", name, knownCount, lastKnown) + } + n := lastKnown + 1 + + // Scoring price never moves: every variant, every terminal scale, is + // costed with the terminal price the box actually derived. + scoreParams := params + + r := row{ + name: name, + live: name == "live-2026-08-31.json", + mode: params.Mode, + knownN: n, + knownH: benchHoursTo(slots, n), + recW: benchFirstSlotW(recorded), + terminalOre: params.TerminalSoCPrice, + sweepW: map[float64]float64{}, + sweepCorr: map[float64]float64{}, + sweepSoC: map[float64]float64{}, + costable: true, + } + { + var sum, w float64 + for _, s := range slots[:n] { + sum += s.PriceOre * float64(s.LenMin) + w += float64(s.LenMin) + } + if w > 0 { + r.knownMeanOre = sum / w + } + } + + solve := func(label string, in []Slot, p Params) forecastVariant { + start := time.Now() + plan := Optimize(in, p) + ms := time.Since(start).Milliseconds() + v := forecastVariant{name: label, plan: &plan, firstW: benchFirstSlotW(&plan), solveMs: ms} + v.corrected, v.raw, v.endSoC, v.costErr = benchKnownWindowCost(&plan, slots, n, p, scoreParams) + if v.costErr != nil { + t.Logf("%-22s %s: known-window cost refused: %v", name, label, v.costErr) + } + return v + } + + a := solve("A", slots, params) + b := solve("B", benchFlattenForecast(slots), params) + c := solve("C", slots[:n], params) + + r.aW, r.bW, r.cW = a.firstW, b.firstW, c.firstW + r.aCorr, r.bCorr, r.cCorr = a.corrected, b.corrected, c.corrected + r.aRaw, r.bRaw, r.cRaw = a.raw, b.raw, c.raw + r.aSoC, r.bSoC, r.cSoC = a.endSoC, b.endSoC, c.endSoC + if a.costErr != nil || b.costErr != nil || c.costErr != nil { + r.costable = false + } + r.divABIdx, r.divABOK = benchFirstDivergence(a.plan, b.plan) + r.divACIdx, r.divACOK = benchFirstDivergence(a.plan, c.plan) + r.divABH = benchHoursTo(slots, r.divABIdx) + r.divACH = benchHoursTo(slots, r.divACIdx) + // How much of the KNOWN window the variants disagree about at all. + // "The plans differ" and "the dispatched watt differs" are separate + // facts, and only the second one reaches hardware. + for i := 0; i < n; i++ { + if math.Abs(a.plan.Actions[i].BatteryW-b.plan.Actions[i].BatteryW) > benchActionDiffW { + r.diffSlotsAB++ + } + if math.Abs(a.plan.Actions[i].BatteryW-c.plan.Actions[i].BatteryW) > benchActionDiffW { + r.diffSlotsAC++ + } + } + + // Terminal sweep on C. Short horizon plus a terminal price above the + // window's own mean is the combination that would make C bank energy + // it has no measured reason to bank, so the end-of-window SoC is + // reported next to the first-slot watt: the terminal price can move + // the plan without moving the decision. + sweepPlans := map[float64]*Plan{} + for _, scale := range terminalScales { + ps := params + ps.TerminalSoCPrice = params.TerminalSoCPrice * scale + plan := Optimize(slots[:n], ps) + sweepPlans[scale] = &plan + r.sweepW[scale] = benchFirstSlotW(&plan) + corr, _, endSoC, err := benchKnownWindowCost(&plan, slots, n, ps, scoreParams) + if err == nil { + r.sweepCorr[scale] = corr + r.sweepSoC[scale] = endSoC + } else { + r.sweepCorr[scale] = math.NaN() + r.sweepSoC[scale] = math.NaN() + } + } + if lo, hi := sweepPlans[0.5], sweepPlans[1.5]; lo != nil && hi != nil { + r.sweepDivIdx, r.sweepDivOK = benchFirstDivergence(lo, hi) + } + // And C as a box that knows nothing past day-ahead would really run + // it: terminal price rederived from the known window alone. + r.knownTermOre = benchTerminalPrice(params.Mode, slots[:n]) + pk := params + pk.TerminalSoCPrice = r.knownTermOre + planK := Optimize(slots[:n], pk) + r.knownTermW = benchFirstSlotW(&planK) + if corr, _, endSoC, err := benchKnownWindowCost(&planK, slots, n, pk, scoreParams); err == nil { + r.knownTermCorr = corr + r.knownTermSoC = endSoC + } else { + r.knownTermCorr = math.NaN() + r.knownTermSoC = math.NaN() + } + + rows = append(rows, r) + t.Logf("%-22s %-18s %7.2f %9.1f %9.1f %9.1f %9.1f %9.1f %8.1f %8.4f", + r.name, string(r.mode), r.knownH, r.recW, r.aW, r.bW, r.cW, + r.bW-r.aW, r.cW-r.aW, params.InitialSoC) + } + if len(rows) == 0 { + t.Fatal("no snapshot produced a comparison") + } + + t.Logf("") + t.Logf("=== how long the guess keeps mattering (first slot where |ΔbatteryW| > %.0f W) ===", benchActionDiffW) + t.Logf("%-22s %10s %10s %10s %10s %12s %12s", "snapshot", + "A_vs_B_idx", "A_vs_B_h", "A_vs_C_idx", "A_vs_C_h", "AB_diff/known", "AC_diff/known") + for _, r := range rows { + ab, ac := fmt.Sprintf("%d", r.divABIdx), fmt.Sprintf("%d", r.divACIdx) + abh, ach := fmt.Sprintf("%.2f", r.divABH), fmt.Sprintf("%.2f", r.divACH) + if !r.divABOK { + ab, abh = "none", ">"+fmt.Sprintf("%.2f", r.divABH) + } + if !r.divACOK { + ac, ach = "none", ">"+fmt.Sprintf("%.2f", r.divACH) + } + t.Logf("%-22s %10s %10s %10s %10s %12s %12s", r.name, ab, abh, ac, ach, + fmt.Sprintf("%d/%d", r.diffSlotsAB, r.knownN), + fmt.Sprintf("%d/%d", r.diffSlotsAC, r.knownN)) + } + + t.Logf("") + t.Logf("=== cost over the KNOWN window only (%s), öre, terminal-corrected at the recorded price ===", + "real day-ahead slots, identical for all three") + t.Logf("%-22s %6s %10s %10s %10s %9s %9s %8s %8s %8s", + "snapshot", "known_n", "A_corr", "B_corr", "C_corr", "B-A", "C-A", "A_soc", "B_soc", "C_soc") + for _, r := range rows { + if !r.costable { + t.Logf("%-22s %6d REFUSED", r.name, r.knownN) + continue + } + t.Logf("%-22s %6d %10.1f %10.1f %10.1f %9.1f %9.1f %8.4f %8.4f %8.4f", + r.name, r.knownN, r.aCorr, r.bCorr, r.cCorr, + r.bCorr-r.aCorr, r.cCorr-r.aCorr, r.aSoC, r.bSoC, r.cSoC) + } + + t.Logf("") + t.Logf("=== C's dependence on the terminal price ===") + t.Logf("term_ore is the price the box derived from the FULL horizon; known_mean is the known window's own mean.") + t.Logf("knownT_* rederives the terminal price from the known slots alone, with Core's mode-dependent formula.") + t.Logf("%-22s %9s %9s %9s %9s %9s %9s %9s %9s %9s %8s", + "snapshot", "term_ore", "known_mean", "C@0.5_w", "C@1.0_w", "C@1.5_w", + "knownT_ore", "knownT_w", "A_w", "sweep_div", "C_w_sprd") + for _, r := range rows { + div := fmt.Sprintf("%d", r.sweepDivIdx) + if !r.sweepDivOK { + div = "none" + } + t.Logf("%-22s %9.1f %9.1f %9.1f %9.1f %9.1f %9.1f %9.1f %9.1f %9s %8.1f", + r.name, r.terminalOre, r.knownMeanOre, + r.sweepW[0.5], r.sweepW[1.0], r.sweepW[1.5], r.knownTermOre, r.knownTermW, r.aW, + div, math.Abs(r.sweepW[1.5]-r.sweepW[0.5])) + } + t.Logf("--- and what the terminal price DOES move: SoC parked at the end of the known window ---") + t.Logf("%-22s %9s %9s %9s %9s %11s %11s %11s", + "snapshot", "C@0.5_soc", "C@1.0_soc", "C@1.5_soc", "knownT_soc", "C@0.5_ore", "C@1.0_ore", "C@1.5_ore") + for _, r := range rows { + t.Logf("%-22s %9.4f %9.4f %9.4f %9.4f %11.1f %11.1f %11.1f", + r.name, r.sweepSoC[0.5], r.sweepSoC[1.0], r.sweepSoC[1.5], r.knownTermSoC, + r.sweepCorr[0.5], r.sweepCorr[1.0], r.sweepCorr[1.5]) + } + + summarize := func(label string, sel func(row) bool) { + var n, sameAB, sameAC, exactAB, exactAC int + var sumAbsAB, sumAbsAC, maxAbsAB, maxAbsAC float64 + var sumCostAB, sumCostAC float64 + var sumDivAB, sumDivAC float64 + var sumDiffAB, sumDiffAC, sumKnownN float64 + var sumSoCAC, sumSweepSoC float64 + var sweepSpread float64 + for _, r := range rows { + if !sel(r) { + continue + } + n++ + dAB, dAC := math.Abs(r.bW-r.aW), math.Abs(r.cW-r.aW) + sumAbsAB += dAB + sumAbsAC += dAC + maxAbsAB = math.Max(maxAbsAB, dAB) + maxAbsAC = math.Max(maxAbsAC, dAC) + if dAB <= benchActionDiffW { + sameAB++ + } + if dAC <= benchActionDiffW { + sameAC++ + } + if dAB < 1 { + exactAB++ + } + if dAC < 1 { + exactAC++ + } + sumCostAB += r.bCorr - r.aCorr + sumCostAC += r.cCorr - r.aCorr + sumDivAB += r.divABH + sumDivAC += r.divACH + sumDiffAB += float64(r.diffSlotsAB) + sumDiffAC += float64(r.diffSlotsAC) + sumKnownN += float64(r.knownN) + sumSoCAC += r.cSoC - r.aSoC + sumSweepSoC += math.Abs(r.sweepSoC[1.5] - r.sweepSoC[0.5]) + lo, hi := r.sweepW[0.5], r.sweepW[1.5] + sweepSpread = math.Max(sweepSpread, math.Abs(hi-lo)) + } + if n == 0 { + return + } + t.Logf("") + t.Logf("SUMMARY %s (n=%d)", label, n) + t.Logf(" first-slot W, B vs A: identical(<1W) %d/%d, within %.0f W %d/%d, mean |Δ| %.1f W, max |Δ| %.1f W", + exactAB, n, benchActionDiffW, sameAB, n, sumAbsAB/float64(n), maxAbsAB) + t.Logf(" first-slot W, C vs A: identical(<1W) %d/%d, within %.0f W %d/%d, mean |Δ| %.1f W, max |Δ| %.1f W", + exactAC, n, benchActionDiffW, sameAC, n, sumAbsAC/float64(n), maxAbsAC) + t.Logf(" mean hours before the plans part: A/B %.2f h, A/C %.2f h", sumDivAB/float64(n), sumDivAC/float64(n)) + t.Logf(" slots of the known window the plans disagree about: A/B %.1f of %.1f, A/C %.1f of %.1f", + sumDiffAB/float64(n), sumKnownN/float64(n), sumDiffAC/float64(n), sumKnownN/float64(n)) + t.Logf(" known-window cost, mean B−A %+.1f öre, mean C−A %+.1f öre (positive = the variant costs more)", + sumCostAB/float64(n), sumCostAC/float64(n)) + t.Logf(" SoC parked at the end of the known window, mean C−A %+.4f (%+.0f Wh on a 9.6 kWh pack)", + sumSoCAC/float64(n), sumSoCAC/float64(n)*9600) + t.Logf(" C first-slot W across terminal scales 0.5…1.5: worst spread %.1f W;"+ + " mean end-of-window SoC spread %.4f", sweepSpread, sumSweepSoC/float64(n)) + } + summarize("all snapshots", func(r row) bool { return true }) + summarize("live snapshot only (today's Core)", func(r row) bool { return r.live }) + summarize("older Python-era snapshots only", func(r row) bool { return !r.live }) + t.Logf("") + t.Logf("NOTE: no snapshot carries realised prices, so nothing above says which variant EARNS more." + + " It says only whether the guess changes the action.") +} + +// benchRelabelKnown returns a copy of slots where the first k count as real +// day-ahead and everything after is marked forecast at forecastConf. The +// PRICES do not move — only how far the box is told to trust them. +// +// That is the point: it isolates the confidence mechanism from forecast +// error. Whatever the twin would have got wrong is held at zero here, so any +// difference the sweep finds is caused by the blending rule alone, and a real +// twin with real error can only differ by more. +func benchRelabelKnown(slots []Slot, k int, forecastConf float64) []Slot { + out := make([]Slot, len(slots)) + copy(out, slots) + for i := range out { + if i < k { + out[i].Confidence = 1.0 + } else { + out[i].Confidence = forecastConf + } + } + return out +} + +// benchForecastConfidence is the confidence the box actually stamped on its +// guessed slots — 0.6 on these sites. Falls back to 0.6 when a snapshot has +// no forecast row to read it from. +func benchForecastConfidence(slots []Slot) float64 { + for _, s := range slots { + if s.Confidence > 0 && s.Confidence < 1.0 { + return s.Confidence + } + } + return 0.6 +} + +// TestForecastValueKnownWindowSweep asks the same question in the regime +// where the guess has the best chance of mattering: a SHORT known window. +// +// Every snapshot here happens to carry 11.75–13.5 h of real day-ahead price, +// which is the comfortable end of the daily cycle. Just before publication a +// box can be down to a few hours. The sweep shortens the known window by +// hand — 2, 4, 6, 8, 12 h and as recorded — and re-asks whether A (trust the +// guess as the box does), B (flatten it) and C (delete it) dispatch the same +// watt. +// +// Prices are left at their recorded values throughout, so this measures the +// blending rule, not the twin's accuracy. Skipped without +// FTW_MPC_SNAPSHOT_DIR. +func TestForecastValueKnownWindowSweep(t *testing.T) { + dir := os.Getenv("FTW_MPC_SNAPSHOT_DIR") + if dir == "" { + t.Skip("FTW_MPC_SNAPSHOT_DIR not set") + } + paths, err := filepath.Glob(filepath.Join(dir, "*.json")) + if err != nil || len(paths) == 0 { + t.Fatalf("no snapshots in %q (err=%v)", dir, err) + } + sort.Strings(paths) + + socLevels, actionLevels := 201, 401 + if v := os.Getenv("FTW_MPC_FORECAST_SOC"); v != "" { + fmt.Sscanf(v, "%d", &socLevels) + } + if v := os.Getenv("FTW_MPC_FORECAST_ACTIONS"); v != "" { + fmt.Sscanf(v, "%d", &actionLevels) + } + + // 15-minute slots on every snapshot here, so k is hours × 4. + windows := []int{8, 16, 24, 32, 48} + t.Logf("grid forced to SoCLevels=%d ActionLevels=%d", socLevels, actionLevels) + t.Logf("prices are unchanged at every k — only the trust label moves, so this isolates") + t.Logf("the confidence rule from forecast error") + t.Logf("%-22s %6s %7s %9s %9s %9s %9s %8s", "snapshot", "known_k", "known_h", "A_w", "B_w", "C_w", "B-A_w", "C-A_w") + + agree := map[int][2]int{} + total := map[int]int{} + for _, path := range paths { + name := filepath.Base(path) + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + d, err := loadDiagnosticBlob(data) + if err != nil { + continue + } + _, slots, params, _, ok := planFromDiagnostic(d) + if !ok { + continue + } + params.SoCLevels = socLevels + params.ActionLevels = actionLevels + fConf := benchForecastConfidence(slots) + recordedK, _, _ := benchLastKnownSlot(slots) + ks := append([]int{}, windows...) + if !slices.Contains(ks, recordedK+1) { + ks = append(ks, recordedK+1) + } + for _, k := range ks { + if k < 2 || k > len(slots) { + continue + } + aPlan := Optimize(benchRelabelKnown(slots, k, fConf), params) + bPlan := Optimize(benchRelabelKnown(slots, k, benchNoTrustConfidence), params) + cPlan := Optimize(slots[:k], params) + aW, bW, cW := benchFirstSlotW(&aPlan), benchFirstSlotW(&bPlan), benchFirstSlotW(&cPlan) + label := name + if k == recordedK+1 { + label = name + " (as recorded)" + } + t.Logf("%-22s %6d %7.2f %9.1f %9.1f %9.1f %9.1f %8.1f", + label, k, benchHoursTo(slots, k), aW, bW, cW, bW-aW, cW-aW) + counts := agree[k] + if math.Abs(bW-aW) <= benchActionDiffW { + counts[0]++ + } + if math.Abs(cW-aW) <= benchActionDiffW { + counts[1]++ + } + agree[k] = counts + total[k]++ + } + } + ks := make([]int, 0, len(total)) + for k := range total { + ks = append(ks, k) + } + sort.Ints(ks) + t.Logf("") + t.Logf("SUMMARY first-slot agreement within %.0f W, by assumed known window", benchActionDiffW) + t.Logf("%6s %7s %14s %14s", "known_k", "known_h", "B_matches_A", "C_matches_A") + for _, k := range ks { + t.Logf("%6d %7.2f %14s %14s", k, float64(k)*15/60, + fmt.Sprintf("%d/%d", agree[k][0], total[k]), + fmt.Sprintf("%d/%d", agree[k][1], total[k])) + } +} + +// ---- CI-runnable fixture tests (no external data) ---- + +// TestForecastBenchZeroConfidenceMeansTotalTrust pins the trap the bench's +// epsilon exists for: asking for zero confidence gets full confidence, so a +// "trust nothing" variant written with Confidence = 0 would silently measure +// the opposite of what it claims. +func TestForecastBenchZeroConfidenceMeansTotalTrust(t *testing.T) { + got := sanitizeOptimizeSlots([]Slot{{LenMin: 15, PriceOre: 100, Confidence: 0}}) + if len(got) != 1 || got[0].Confidence != 1.0 { + t.Fatalf("Confidence 0 sanitized to %v, want 1.0 (the coercion the bench works around)", got) + } + eps := sanitizeOptimizeSlots([]Slot{{LenMin: 15, PriceOre: 100, Confidence: benchNoTrustConfidence}}) + if len(eps) != 1 || eps[0].Confidence != benchNoTrustConfidence { + t.Fatalf("epsilon confidence did not survive sanitisation: %v", eps) + } +} + +// TestForecastBenchFlattenLeavesKnownSlotsAlone pins that variant B touches +// only the slots the box was unsure about. +func TestForecastBenchFlattenLeavesKnownSlotsAlone(t *testing.T) { + in := []Slot{ + {LenMin: 15, PriceOre: 100, Confidence: 1.0}, + {LenMin: 15, PriceOre: 300, Confidence: 0.6}, + {LenMin: 15, PriceOre: 200, Confidence: 1.0}, + } + out := benchFlattenForecast(in) + if out[0].Confidence != 1.0 || out[2].Confidence != 1.0 { + t.Fatalf("known slots were flattened: %v", out) + } + if out[1].Confidence != benchNoTrustConfidence { + t.Fatalf("forecast slot not flattened: %v", out[1]) + } + if in[1].Confidence != 0.6 { + t.Fatal("benchFlattenForecast mutated the caller's slice") + } +} + +// TestForecastBenchRelabelKnownMovesTrustNotPrice pins that the known-window +// sweep shortens what the box is told it knows without touching a single +// price — the property that lets the sweep isolate the confidence rule from +// the twin's accuracy. +func TestForecastBenchRelabelKnownMovesTrustNotPrice(t *testing.T) { + in := []Slot{ + {LenMin: 15, PriceOre: 100, Confidence: 1.0}, + {LenMin: 15, PriceOre: 200, Confidence: 1.0}, + {LenMin: 15, PriceOre: 300, Confidence: 1.0}, + } + out := benchRelabelKnown(in, 1, 0.6) + for i := range out { + if out[i].PriceOre != in[i].PriceOre { + t.Fatalf("slot %d price moved: %v -> %v", i, in[i].PriceOre, out[i].PriceOre) + } + } + if out[0].Confidence != 1.0 || out[1].Confidence != 0.6 || out[2].Confidence != 0.6 { + t.Fatalf("trust labels wrong: %v", []float64{out[0].Confidence, out[1].Confidence, out[2].Confidence}) + } + if in[1].Confidence != 1.0 { + t.Fatal("benchRelabelKnown mutated the caller's slice") + } + if got := benchForecastConfidence(out); got != 0.6 { + t.Fatalf("forecast confidence read back as %v, want 0.6", got) + } +} + +// TestForecastBenchKnownWindow pins the prefix detection and the hours +// conversion that turn a slot index into "the guess stops mattering after N +// hours". +func TestForecastBenchKnownWindow(t *testing.T) { + slots := []Slot{ + {LenMin: 15, Confidence: 1.0}, + {LenMin: 15, Confidence: 1.0}, + {LenMin: 15, Confidence: 0.6}, + {LenMin: 15, Confidence: 0.6}, + } + last, contiguous, count := benchLastKnownSlot(slots) + if last != 1 || !contiguous || count != 2 { + t.Fatalf("last=%d contiguous=%v count=%d, want 1/true/2", last, contiguous, count) + } + if h := benchHoursTo(slots, 2); h != 0.5 { + t.Fatalf("hours to slot 2 = %v, want 0.5", h) + } + slots[3].Confidence = 1.0 + if _, contiguous, _ := benchLastKnownSlot(slots); contiguous { + t.Fatal("a hole in the known window must not report contiguous") + } +} + +// TestForecastBenchFlatForecastRemovesArbitrageBeyondKnown is the property +// the whole bench rests on: flattening a forecast slot's confidence removes +// the price SHAPE the DP could arbitrage against, so a plan that only charges +// because of a guessed evening peak stops charging. +func TestForecastBenchFlatForecastRemovesArbitrageBeyondKnown(t *testing.T) { + // Known window: four flat, unremarkable slots. Guessed window: a deep + // cheap trough then an expensive peak — pure invented arbitrage. + slots := make([]Slot, 12) + for i := range slots { + price := 200.0 + conf := 1.0 + switch { + case i >= 4 && i < 8: + price, conf = 20, 0.6 + case i >= 8: + price, conf = 600, 0.6 + } + slots[i] = Slot{ + StartMs: 1_700_000_000_000 + int64(i)*3_600_000, LenMin: 60, + PriceOre: price, SpotOre: price / 2, Confidence: conf, LoadW: 1000, + } + } + p := Params{ + Mode: ModeArbitrage, InitialSoC: 0.5, SoCMin: 0.1, SoCMax: 1.0, + SoCLevels: 41, ActionLevels: 41, MaxChargeW: 3000, MaxDischargeW: 3000, + ChargeEfficiency: 0.95, DischargeEfficiency: 0.95, CapacityWh: 9600, + TerminalSoCPrice: 200, + } + trusted := Optimize(slots, p) + flat := Optimize(benchFlattenForecast(slots), p) + // Under the guess, the cheap trough is worth charging into; flattened, + // those four slots all price at the horizon mean and there is nothing to + // arbitrage, so the trough charging must weaken. + var trustedTrough, flatTrough float64 + for i := 4; i < 8; i++ { + trustedTrough += trusted.Actions[i].BatteryW + flatTrough += flat.Actions[i].BatteryW + } + if trustedTrough <= flatTrough { + t.Fatalf("flattening did not remove the invented arbitrage: trusted charges %.0f W, flat %.0f W", + trustedTrough, flatTrough) + } +}