diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2124afe..44cfebb 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,3 +11,9 @@ updates: directory: "/" schedule: interval: "weekly" + ignore: + # The v5→v7 bump silently broke coverage uploads ("Missing Head Commit" + # on PRs). Keep codecov-action pinned until a deliberate, verified + # migration — see the comment in .github/workflows/CI.yml. + - dependency-name: "codecov/codecov-action" + update-types: ["version-update:semver-major"] diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 852dc10..c3e707b 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -19,12 +19,13 @@ jobs: permissions: # needed to allow julia-actions/cache to proactively delete old caches that it has created actions: write contents: read + id-token: write # OIDC token for tokenless Codecov uploads (see codecov step) strategy: fail-fast: false matrix: version: - '1.10' - # - 'nightly' + - '1.12' os: - ubuntu-latest arch: @@ -39,8 +40,22 @@ jobs: - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v7 + # Pinned to v5: the dependabot bump to v7 (2026-06-25) silently broke + # uploads — Codecov has no commit newer than 2026-06-15, which is what + # produces "Missing Head Commit" on PRs. v5 is the last version verified + # to upload from this workflow. Before re-bumping, migrate deliberately + # (e.g. OIDC: `use_oidc: true` + `id-token: write` permission) and + # confirm a commit appears on Codecov. + # Authentication uses OIDC (`use_oidc` + the job's `id-token: write` + # permission) because the CODECOV_TOKEN secret is not set in this repo + # ("Token length: 0" in CI) and tokenless uploads are rejected on + # protected branches. OIDC requires the Codecov GitHub App to be + # installed for the organization; if uploads fail with an OIDC error, + # either install the app or set the CODECOV_TOKEN secret and replace + # `use_oidc` with `token: ${{ secrets.CODECOV_TOKEN }}`. + - uses: codecov/codecov-action@v5 with: files: lcov.info - token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: false + use_oidc: true + # Fail loudly: a silent upload failure hid this breakage for weeks. + fail_ci_if_error: true diff --git a/.gitignore b/.gitignore index c019973..cfb7f58 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,7 @@ plan/ *_cuts.json settings.json *.sh +*-backup +*.cuts.json.stale_stronggrid +*strong.json +.claude/settings.local.json diff --git a/README.md b/README.md index 710b710..86d8ebe 100644 --- a/README.md +++ b/README.md @@ -54,22 +54,25 @@ using DecisionRules, JuMP, DiffOpt, Flux using SCS # 1) Build per-stage subproblems (DiffOpt-enabled) and collect: -# subproblems, state_params_in, state_params_out, uncertainty_sampler, uncertainties_structure +# subproblems, state_params_in, state_params_out, uncertainty_samples # 2) Build the deterministic equivalent over the full horizon det = DiffOpt.diff_model(() -> DiffOpt.diff_optimizer(SCS.Optimizer)) -det, uncertainties_structure_det = DecisionRules.deterministic_equivalent!( +det, uncertainty_samples_det = DecisionRules.deterministic_equivalent!( det, subproblems, state_params_in, state_params_out, Float64.(initial_state), - uncertainties_structure, + uncertainty_samples, ) +# deterministic_equivalent! remaps state_params_in/state_params_out in place. +# Copy those arrays first if you also need the original stage-wise refs later. + # 3) Train a TS-DDR policy end-to-end -num_uncertainties = length(uncertainty_sampler()[1]) # number of uncertainty components per stage +num_uncertainties = length(uncertainty_samples[1]) # number of uncertainty components per stage policy = Chain( Dense(DecisionRules.policy_input_dim(num_uncertainties, length(initial_state)), 64, relu), Dense(64, length(initial_state)), @@ -79,9 +82,9 @@ DecisionRules.train_multistage( policy, initial_state, det, - state_in_det, - state_out_det, - uncertainty_sampler; + state_params_in, + state_params_out, + uncertainty_samples_det; num_batches=100, num_train_per_batch=32, optimizer=Flux.Adam(1e-3), @@ -97,7 +100,7 @@ Single shooting solves one optimization per stage and rolls forward using the re ```julia using DecisionRules, Flux -num_uncertainties = length(uncertainty_sampler()[1]) +num_uncertainties = length(uncertainty_samples[1]) policy = Chain( Dense(DecisionRules.policy_input_dim(num_uncertainties, length(initial_state)), 64, relu), Dense(64, length(initial_state)), @@ -109,7 +112,7 @@ DecisionRules.train_multistage( subproblems, state_params_in, state_params_out, - uncertainty_sampler; + uncertainty_samples; num_batches=100, num_train_per_batch=32, optimizer=Flux.Adam(1e-3), @@ -126,7 +129,7 @@ Multiple shooting partitions the horizon into windows of length `window_size`. E using DecisionRules, Flux, DiffOpt using SCS -num_uncertainties = length(uncertainty_sampler()[1]) +num_uncertainties = length(uncertainty_samples[1]) policy = Chain( Dense(DecisionRules.policy_input_dim(num_uncertainties, length(initial_state)), 64, relu), Dense(64, length(initial_state)), @@ -149,10 +152,7 @@ DecisionRules.train_multiple_shooting( policy, initial_state, windows, - state_params_in, - state_params_out, - uncertainty_sampler; - window_size=24, # e.g., 6, 24, ... + uncertainty_samples; num_batches=100, num_train_per_batch=32, optimizer=Flux.Adam(1e-3), @@ -168,7 +168,8 @@ The training loops record metrics through a per-sample `SampleLog` cache and a p ```julia using DecisionRules, Random -# Materialize a FIXED held-out evaluation set once, before training +# Materialize a FIXED held-out evaluation set once, before training. +# Use stage-wise subproblems and parameter refs, not DE-remapped refs. Random.seed!(1234) eval_scenarios = [DecisionRules.sample(uncertainty_samples) for _ in 1:8] @@ -178,7 +179,7 @@ rollout_eval = RolloutEvaluation( policy_state=:realized, ) -train_multistage(policy, initial_state, det, state_in_det, state_out_det, uncertainty_sampler; +train_multistage(policy, initial_state, det, state_params_in, state_params_out, uncertainty_samples_det; num_batches=100, record=(sample_log, iter, model) -> begin rollout_eval(iter, model) @@ -202,6 +203,57 @@ Each evaluation reports (a) the rollout objective **excluding** the target-slack Per-sample debugging hooks can be attached with `SampleLog(on_sample=(s, models, log) -> ...)`; the training loop calls the hook after each sample's solve with the live JuMP model(s). The previous `record_loss=(iter, model, loss, tag) -> ...` keyword keeps working as a deprecated adapter. +## Strict mode and reachable policies + +The standard TS-DDR target constraint uses slack: + +```math +x_t + \delta_t = \hat{x}_t, +\qquad +\text{objective} += C_\delta \|\delta_t\|. +``` + +Slack makes training robust to unreachable targets, but it also makes the dual +signal depend on the target-penalty calibration. Strict mode removes the slack: + +```math +x_t = \hat{x}_t. +``` + +The resulting dual is the clean shadow price of imposing the target. The price +of that cleaner signal is feasibility: every policy target must be reachable +from the state used to condition the policy. + +This is automatic in the hydro strict subproblem path because each stage is +solved sequentially and the policy receives the realized previous reservoir +state. It is also possible in regular deterministic equivalents when the target +trajectory is rolled out from the true initial state using a reachable policy: + +```math +\hat{x}_0 = x_0,\qquad +\hat{x}_t = \pi_\theta(w_t, \hat{x}_{t-1}),\qquad +\hat{x}_t \in R(\hat{x}_{t-1}, w_t). +``` + +By induction, all targets are feasible, and the strict equalities force the +realized trajectory to match that reachable path. See +[`examples/HydroPowerModels`](examples/HydroPowerModels) and the +DecisionRulesExa.jl companion for the GPU strict regular-DE implementation. + +The reachable map ``R(\hat x_{t-1}, w_t)`` depends on the state, and that +dependence **must be differentiated**. Treating the interval endpoints as +constants still trains and still lowers the loss while descending a materially +different direction — on the hydro case, a gradient carrying 6% of the true +magnitude and pointing 48 degrees away from it. Verify the complete actor +gradient against finite differences before trusting any hyperparameter +conclusion drawn on top of it. + +The policy helpers separate two architectural choices: + +- recurrent `layers` / `DR_ENCODER_LAYERS` process uncertainty history only; +- `combiner_layers` / `DR_HEAD_LAYERS` add a nonlinear feed-forward + state-to-target head without recurrence over the state input. + ## GPU acceleration with DecisionRulesExa.jl For large-scale problems where the inner NLP solve is the bottleneck (e.g., AC-OPF with hundreds of buses), [DecisionRulesExa.jl](https://github.com/LearningToOptimize/DecisionRulesExa.jl) provides a GPU-accelerated backend that replaces JuMP with [ExaModels.jl](https://github.com/exanauts/ExaModels.jl) and solves with [MadNLP.jl](https://github.com/MadNLP/MadNLP.jl) + CUDSS on GPU. @@ -222,6 +274,24 @@ Examples live in `examples/`. Run tests with: julia --project -e 'using Pkg; Pkg.test()' ``` +## Repository Map + +| Path | Purpose | +|---|---| +| `src/DecisionRules.jl` | Module entrypoint and exports | +| `src/dense_multilayer_nn.jl` | MLP helpers, state-conditioned recurrent policies, nonlinear target heads | +| `src/simulate_multistage.jl` | Stage-wise and deterministic-equivalent simulation logic | +| `src/multiple_shooting.jl` | Windowed multiple-shooting setup, simulation, and training | +| `src/utils.jl` | Target-parameter utilities, deficit construction, rollout evaluation | +| `src/integer_strategies.jl` | Strategies for extracting gradients from integer/mixed-integer models | +| `src/score_function.jl` | Score-function gradient correction for nonsmooth/integer problems | +| `src/parameter_duals.jl` | Dual/sensitivity helpers for parameterized JuMP models | +| `docs/src/` | Documenter.jl manual pages | +| `examples/inventory_control/` | Inventory-control example and dynamic-programming/SDDP comparisons | +| `examples/rocket_control/` | Rocket MPC/control example | +| `examples/Experimental/` | Research prototypes and robotics/control explorations | +| `test/runtests.jl` | Package test suite | + ## Citation If you use this package in academic work, please cite: diff --git a/docs/Project.toml b/docs/Project.toml index cadd9b3..6a67612 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,4 +1,6 @@ [deps] +CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" DecisionRules = "47937410-f832-486f-8300-12c95b225dfc" DiffOpt = "930fe3bc-9c6b-11ea-2d94-6184641e85e7" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" @@ -8,9 +10,13 @@ HiGHS = "87dc4568-4c63-4d18-b0c0-bb2238e4078b" Ipopt = "b6b21f68-93f8-5de0-b562-5493be1d77c9" JuMP = "4076af6c-e467-56ae-b986-b466b2749572" Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" +StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [compat] Documenter = "1" diff --git a/docs/make.jl b/docs/make.jl index 490de9b..d7655d5 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -2,15 +2,16 @@ using Documenter using Literate using DecisionRules -# Convert Literate.jl sources to markdown -examples_src = joinpath(@__DIR__, "src", "examples") -examples_out = joinpath(@__DIR__, "src", "examples") -for file in readdir(examples_src) - endswith(file, ".jl") || continue - Literate.markdown( - joinpath(examples_src, file), examples_out; - documenter=true, credit=false, - ) +# Convert Literate.jl sources to markdown, in place, wherever they live: a case +# study that owns a runnable walkthrough keeps it beside its prose rather than in +# a separate examples pile. +for dir in (joinpath(@__DIR__, "src", "examples"), + joinpath(@__DIR__, "src", "casestudies", "hydro")) + isdir(dir) || continue + for file in readdir(dir) + endswith(file, ".jl") || continue + Literate.markdown(joinpath(dir, file), dir; documenter=true, credit=false) + end end makedocs(; @@ -20,19 +21,35 @@ makedocs(; format=Documenter.HTML(; prettyurls=get(ENV, "CI", nothing) == "true", canonical="https://LearningToOptimize.github.io/DecisionRules.jl", + size_threshold=300 * 1024, ), pages=[ "Home" => "index.md", - "Algorithm" => "algorithm.md", - "Gradient Fallback" => "gradient_fallback.md", - "Uncertainty Sampling" => "sampling.md", - "GPU Acceleration" => "gpu_acceleration.md", - "Examples" => [ - "Hydropower Scheduling" => "examples/hydro.md", - "Rocket Control" => "examples/rocket.md", - "Stochastic Lot-Sizing (Integer Variables)" => "examples/inventory.md", + "Part I — Theory" => [ + "Multistage stochastic optimization" => "theory/multistage.md", + "The TS-DDR framework" => "algorithm.md", + "Stochastic dual dynamic programming" => "theory/sddp.md", + "Extensions: mixed gradients, critics, risk" => "theory/extensions.md", + ], + "Part II — Package Guide" => [ + "Getting started" => "guide/getting_started.md", + "Uncertainty sampling" => "sampling.md", + "Gradient fallback" => "gradient_fallback.md", + "GPU acceleration" => "gpu_acceleration.md", + "API reference" => "api.md", + ], + "Part III — Case Studies" => [ + "Battery-storage AC-OPF" => "casestudies/battery_storage_opf.md", + "Long-term hydrothermal planning" => [ + "Overview" => "casestudies/hydro/index.md", + "The problem" => "casestudies/hydro/problem.md", + "Valuing water: two approaches" => "casestudies/hydro/method.md", + "Results" => "casestudies/hydro/results.md", + "Walkthrough" => "casestudies/hydro/walkthrough.md", + ], + "Rocket control" => "examples/rocket.md", + "Stochastic lot-sizing (integer variables)" => "examples/inventory.md", ], - "API Reference" => "api.md", ], ) diff --git a/docs/src/algorithm.md b/docs/src/algorithm.md index 36c3d24..c2822ce 100644 --- a/docs/src/algorithm.md +++ b/docs/src/algorithm.md @@ -1,18 +1,23 @@ -# Algorithm +# The TS-DDR framework ```@meta CurrentModule = DecisionRules ``` -This page summarizes the TS-DDR (Two-Stage Deep Decision Rules) training algorithm. -For the full derivation, see [arXiv:2405.14973](https://arxiv.org/abs/2405.14973). +This chapter develops the TS-DDR (Two-Stage Deep Decision Rules) training +algorithm — the core of DecisionRules.jl. For the full derivation, see +[arXiv:2405.14973](https://arxiv.org/abs/2405.14973); for the general +problem class and where decision rules sit among solution methods, see +[Multistage stochastic optimization](@ref). ## Problem setting -Consider a ``T``-stage stochastic control problem where at each stage ``t`` we observe -an uncertainty realization ``w_t`` and must choose an action ``u_t`` that satisfies -stage constraints ``(u_t, x_t) \in \mathcal{X}_t(x_{t-1}, w_t)``. The goal is to -minimize the expected total cost: +Consider the ``T``-stage stochastic control problem of the previous +chapter: at each stage ``t`` we observe an uncertainty realization ``w_t`` +and must choose an action ``u_t`` that satisfies +stage constraints ``(u_t, x_t) \in \mathcal{X}_t(x_{t-1}, w_t)``. Restricting +attention to a parametric policy class, the goal is to +minimize the expected total cost over the parameters: ```math \min_\theta \; \mathbb{E}_{w_{1:T}} \left[ \sum_{t=1}^{T} c_t(x_t, u_t) \right] @@ -27,9 +32,13 @@ Instead of mapping observations directly to actions, the policy outputs **target states**: ```math -\hat{x}_{1:T} = \pi_\theta(w_{1:T}) +\hat{x}_t = \pi_\theta(w_t, \hat{x}_{t-1}), \qquad \hat{x}_0 = x_0, ``` +evaluated stage-wise with state feedback: each target is conditioned on the +previous target state during deterministic-equivalent training (or on the +realized state ``x_{t-1}`` in closed-loop rollouts). + A projection subproblem enforces feasibility by solving: ```math @@ -105,110 +114,172 @@ for k = 1, ..., ⌈T/W⌉: pass realized end-state to window k+1 ``` -**Pros**: balances coupling (within windows) with tractability; parallelizable windows. -**Cons**: continuity gaps between windows require penalty tuning. - -## Mixed gradient: score-function (REINFORCE) correction +**Pros**: balances coupling (within windows) with tractability; cheaper inner +solves than a full-horizon deterministic equivalent. +**Cons**: windows are chained sequentially during rollout/training because each +window needs the previous realized end-state; cross-window coupling is weaker +than in the full deterministic equivalent. + +## Beyond the pure dual gradient + +The dual gradient above is exact for smooth subproblems and unbiased over +fresh samples. Two extensions handle the situations where that is not +enough — **discrete decisions**, where the dual is local to a fixed +integer assignment and a score-function (REINFORCE) correction restores +the missing signal, and **small sample budgets**, where a control-variate +critic reduces the estimator's variance without moving its optimum. Both +are developed, together with a risk-averse change-of-measure variant, in +[Extensions: mixed gradients, critics, and risk](@ref); the score-function +correction is exercised in the +[Stochastic Lot-Sizing with Fixed Ordering Costs](@ref) case study. -For problems with integer variables or non-smooth subproblems, the dual -gradient can be biased — it is local to a fixed integer assignment and cannot -see the effect of discrete switches (e.g., opening a setup variable). +## Penalty annealing -DecisionRules provides a **score-function (REINFORCE)** correction that mixes -the dual gradient with a model-free policy gradient estimated from stage-wise -rollouts under perturbed targets. +The target penalty ``\lambda`` is critical: too small and the optimizer ignores +targets (no gradient); too large and the problem becomes ill-conditioned. DecisionRules.jl +supports a **penalty annealing schedule** that ramps ``\lambda`` during training: -### How the score-function estimator works +``` +Phase 1 (warmup): λ × 0.1 — let the policy explore +Phase 2 (nominal): λ × 1.0 — standard training +Phase 3 (tighten): λ × 10.0 — sharpen target tracking +Phase 4 (lock): λ × 30.0 — final precision +``` -1. **Perturb**: add Gaussian noise to the policy targets: - ``\tilde{x}_t = \hat{x}_t(\theta) + \delta_t``, where - ``\delta_t \sim \mathcal{N}(0, \sigma^2 I)``. +This is the `default_annealed` schedule, activated with `penalty_schedule=:default_annealed`. -2. **Rollout**: solve the stage-wise subproblems with the perturbed targets to - obtain realized costs ``R_m`` for ``m = 1, \ldots, M`` rollouts. These - rollouts solve the models exactly as built (MIPs stay MIPs), so the costs - reflect true integer-feasible decisions. +## Strict mode: penalty-free gradient signal -3. **Advantage**: center the costs ``A_m = R_m - \bar{R}`` (mean baseline - reduces variance without changing the expected gradient). +The standard TS-DDR formulation uses a penalty ``C_\delta \|\delta_t\|`` to +penalize deviations from the policy's targets. While effective, the penalty +introduces a trade-off: the dual ``\lambda_t`` conflates the **economic shadow +price** with a **penalty-correction term**. At high penalty, the gradient +signal tells the policy "reduce ``\delta``" rather than "be economically +optimal." -4. **Surrogate loss**: the differentiable scalar whose gradient recovers the - REINFORCE estimate: +**Strict mode** eliminates this coupling entirely by replacing the slack +constraint ``x_t + \delta_t = \hat{x}_t`` with a **hard equality**: ```math -L_{\text{sf}}(\theta) -\;=\; -\frac{1}{M} \sum_{m=1}^{M} - A_m - \sum_{t=1}^{T} - \left\langle - \frac{\delta_{m,t}}{\sigma^2},\; - \hat{x}_{t+1}(\theta) - \right\rangle. +x_t = \hat{x}_t \quad :\lambda_t ``` -This is the standard score-function estimator for Gaussian perturbations. -The key identity is -``\nabla_\theta \log p(\delta_t \mid \theta) = \delta_t / \sigma^2`` -for a Gaussian centered at ``\hat{x}_t(\theta)``. +There are no deficit variables, no penalty term, and no penalty to tune. The +dual ``\lambda_t`` is the **pure shadow price** ``\partial Q_t / \partial +\hat{x}_t`` — the marginal value of changing the target, uncontaminated by +any regularization. -### Mixed gradient +### The condition: target reachability -The final training gradient combines both signals: +A hard equality has no slack to absorb an unreachable target, so strict mode +is well-posed under exactly one condition: **every target the policy emits +must be attainable from the state the system is in when the corresponding +stage is solved.** Formally, let ```math -\nabla L -\;=\; -\alpha\, \nabla L_{\text{dual}} -+ (1 - \alpha)\, \nabla L_{\text{sf}}, +R(x, w) \;=\; \bigl\{\, x' \;:\; \exists\, u \text{ with } + (u, x') \in \mathcal{X}(x, w) \,\bigr\} ``` -where ``\alpha \in [0, 1]`` is the `dual_weight`. - -There are two separate solve paths in the mixed-gradient training loop: - -- **Dual path**: controlled by `integer_strategy`, which determines how local - dual information is read from the deterministic equivalent - (e.g., [`FixedDiscreteIntegerStrategy`](@ref) solves the MIP, fixes integers, - re-solves the LP, and reads LP duals). -- **Score-function path**: controlled by [`ScoreFunctionConfig`](@ref), which - owns separate rollout subproblems. These are solved exactly as built, and - their realized costs define the Monte Carlo score-function term. - -### Scheduled ramp-in +denote the **one-stage reachable set** — the states attainable from ``x`` +under realization ``w`` by some admissible action. Strict mode requires +``\hat{x}_t \in R(x_{t-1}, w_t)`` at every stage, where ``x_{t-1}`` is the +*realized* state. + +A **feasibility-guaranteeing policy** enforces this by construction: it +computes (an inner approximation of) ``R`` from its input state and maps the +network output into that set, typically by scaling a sigmoid-bounded output +across the reachable interval. The bounds carry no gradient; the gradient path +is solely through the network output, exactly as in the standard TS-DDR +pipeline. Constructing ``R`` is problem-specific. It is cheap whenever the +dynamics are linear in the controls with box bounds — resource-balance +equations are the canonical case. The +[battery-storage study](@ref "Stochastic battery-storage AC optimal power flow") +derives the battery-dynamic interval and explains why a network-constrained OPF +still needs an empirical strict-feasibility gate. + +### Validity in every formulation, by induction + +Reachability of each target from the *policy's input state* is enough to make +strict mode well-posed in **all** training formulations — stage-wise +subproblems, the embedded deterministic equivalent, and the regular +deterministic equivalent alike. The argument is one induction, and the strict +equality itself is what carries it: suppose ``\hat{x}_0 = x_0`` (the known +initial state) and every policy call returns a target reachable from the state +it conditioned on, -A [`ScoreFunctionSchedule`](@ref) can ramp ``\alpha`` from 1 (pure dual) to -its final value over a warmup period. Let ``k`` be the current iteration and -``\rho_k = \operatorname{clip}((k - k_0) / r,\, 0,\, 1)``. The effective -score-function weight is ``\rho_k (1 - \alpha)``. - -This lets the DE dual gradient establish a good initial policy before -introducing the higher-variance REINFORCE signal. - -See the [Stochastic Lot-Sizing with Fixed Ordering Costs](@ref) example for a -complete worked example with integer variables and mixed gradients. - -## Penalty annealing - -The target penalty ``\lambda`` is critical: too small and the optimizer ignores -targets (no gradient); too large and the problem becomes ill-conditioned. DecisionRules.jl -supports a **penalty annealing schedule** that ramps ``\lambda`` during training: - -``` -Phase 1 (warmup): λ × 0.1 — let the policy explore -Phase 2 (nominal): λ × 1.0 — standard training -Phase 3 (tighten): λ × 10.0 — sharpen target tracking -Phase 4 (lock): λ × 30.0 — final precision +```math +\hat{x}_t = \pi_\theta(w_t, \hat{x}_{t-1}) \in R(\hat{x}_{t-1}, w_t). ``` -This is the `default_annealed` schedule, activated with `penalty_schedule=:default_annealed`. +1. Stage 1 is feasible: ``\hat{x}_1`` is reachable from the true initial + state ``x_0 = \hat{x}_0``. +2. If stages ``1, \ldots, t`` are feasible, their strict equalities force + ``x_s = \hat{x}_s`` for ``s \le t``. The state the policy conditioned on + when producing ``\hat{x}_{t+1}`` is therefore *identical* to the realized + state ``x_t``, so ``\hat{x}_{t+1} \in R(x_t, w_{t+1})`` and stage ``t+1`` + is feasible. + +The formulations differ only in *which symbol* plays the policy input. In +stage-wise rollouts the policy reads the realized state ``x_{t-1}`` directly; +in the embedded DE it reads the solver's state variables; in the regular DE it +reads its own previous target ``\hat{x}_{t-1}``. Under strict equalities these +are the same object — the induction shows previous target ``\equiv`` previous +realized state — so no formulation is a special case and none needs a separate +argument. In particular, the regular DE is *not* an exception requiring extra +structure: the strict equality **closes the loop as a consequence**, it does +not presuppose a closed loop. + +### Information pattern: closed-loop vs. open-loop + +Distinct from the well-posedness question is the **information pattern**: does +the policy read the *realized* state (closed-loop feedback) or its *own +previous target* (open-loop target generation)? This axis matters +independently of strict mode: + +- In **non-strict** training, slack lets the realized state deviate from the + target, so the two inputs genuinely differ. A regular DE trains the policy + on target feedback while the optimizer realizes something else — a + train/deploy mismatch that shows up at evaluation (below). +- Under **strict equalities** the distinction collapses: realized state and + target are identical at every stage, so target feedback and realized + feedback are the same function evaluation, and training-time DE solves and + deployment-time stage-wise rollouts traverse identical trajectories. + +Keeping the two axes separate is the point: *strict-mode validity* is about +reachability of targets; *closed- vs. open-loop* is about what information the +policy consumes. Strict mode does not require closed-loop evaluation — it +makes the question moot by forcing the two information patterns to coincide. + +### When to use strict mode + +Whenever it is applicable — a feasible initial state and a policy constructed +to emit only **one-stage reachable** targets — strict mode is the preferred +formulation. The only reason to fall back to the penalty formulation is +numerical: some solvers degrade when the additional hard equality constraints +are imposed (the equalities remove the slack that otherwise absorbs small +constraint violations during intermediate iterates). + +Everything else follows as a beneficial side effect rather than a selection +criterion: there is no penalty hyperparameter to tune, no annealing schedule, +and the dual ``\lambda_t`` is the exact shadow price of the target — the +gradient signal is uncontaminated by a regularization term. + +In the +[battery-storage AC-OPF study](@ref "Stochastic battery-storage AC optimal power flow"), +strict mode is accepted only after representative true-ACP rollouts solve with +zero load shedding. This distinguishes dynamic reachability from full network +feasibility. ## Evaluation semantics -A policy trained on the deterministic equivalent generates targets using **target-state -feedback** (each target depends on the previous *predicted* target, not the realized -state). Evaluating such a policy with **realized-state feedback** (deployment semantics) -tests a different closed-loop path and will generally report higher cost. +A policy trained on the (non-strict) deterministic equivalent generates targets +using **target-state feedback** (each target depends on the previous *predicted* +target, not the realized state). Evaluating such a policy with **realized-state +feedback** (deployment semantics) tests a different closed-loop path and will +generally report higher cost. Under strict equalities the two modes coincide — +realized states equal targets identically — so the choice below is material +only when slack is present. [`RolloutEvaluation`](@ref) supports both modes via the `policy_state` keyword: - `:target` — matches DE training semantics (fair in-sample comparator) diff --git a/docs/src/assets/bolivia_inflow.svg b/docs/src/assets/bolivia_inflow.svg new file mode 100644 index 0000000..fc65532 --- /dev/null +++ b/docs/src/assets/bolivia_inflow.svg @@ -0,0 +1,27 @@ + + + +0 + +20 + +40 + +60 + +80 +wk 1 +wk 9 +wk 17 +wk 25 +wk 33 +wk 41 + + + +wet peak · wk 2 + +dry trough · wk 28 +Reservoir inflow across the year +energy-weighted · band = p10–p90 across 15 historical scenarios + \ No newline at end of file diff --git a/docs/src/assets/bolivia_map.svg b/docs/src/assets/bolivia_map.svg new file mode 100644 index 0000000..fc1a33a --- /dev/null +++ b/docs/src/assets/bolivia_map.svg @@ -0,0 +1,83 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Santa Cruz + +Cochabamba + +La Paz + +Oruro + +Potosí + +Sucre + +Tarija + +Trinidad +The Bolivian SIN — capacity, storage & load +satellite nodes fan each city's assets out from its bus so nothing overlaps + + +hydro + +gas thermal + +reservoir storage + +average load + +load · mining core + +230 kV corridor + +115 kV corridor + +69 kV corridor + + \ No newline at end of file diff --git a/docs/src/assets/hydro_cost_comparison.png b/docs/src/assets/hydro_cost_comparison.png deleted file mode 100644 index c806f5a..0000000 Binary files a/docs/src/assets/hydro_cost_comparison.png and /dev/null differ diff --git a/docs/src/assets/hydro_cost_distributions.png b/docs/src/assets/hydro_cost_distributions.png new file mode 100644 index 0000000..837d782 Binary files /dev/null and b/docs/src/assets/hydro_cost_distributions.png differ diff --git a/docs/src/assets/hydro_energy_price.png b/docs/src/assets/hydro_energy_price.png new file mode 100644 index 0000000..2709323 Binary files /dev/null and b/docs/src/assets/hydro_energy_price.png differ diff --git a/docs/src/assets/hydro_generation_comparison.png b/docs/src/assets/hydro_generation_comparison.png deleted file mode 100644 index b601e59..0000000 Binary files a/docs/src/assets/hydro_generation_comparison.png and /dev/null differ diff --git a/docs/src/assets/hydro_paired_differences.png b/docs/src/assets/hydro_paired_differences.png new file mode 100644 index 0000000..2deb0a7 Binary files /dev/null and b/docs/src/assets/hydro_paired_differences.png differ diff --git a/docs/src/assets/hydro_stagewise_physical.png b/docs/src/assets/hydro_stagewise_physical.png new file mode 100644 index 0000000..fd455f6 Binary files /dev/null and b/docs/src/assets/hydro_stagewise_physical.png differ diff --git a/docs/src/assets/hydro_training_convergence.png b/docs/src/assets/hydro_training_convergence.png deleted file mode 100644 index 3fd5455..0000000 Binary files a/docs/src/assets/hydro_training_convergence.png and /dev/null differ diff --git a/docs/src/assets/hydro_training_history.png b/docs/src/assets/hydro_training_history.png new file mode 100644 index 0000000..22252e0 Binary files /dev/null and b/docs/src/assets/hydro_training_history.png differ diff --git a/docs/src/assets/hydro_violation_share.png b/docs/src/assets/hydro_violation_share.png deleted file mode 100644 index 5f1d863..0000000 Binary files a/docs/src/assets/hydro_violation_share.png and /dev/null differ diff --git a/docs/src/assets/hydro_volume_comparison.png b/docs/src/assets/hydro_volume_comparison.png deleted file mode 100644 index 4e0f863..0000000 Binary files a/docs/src/assets/hydro_volume_comparison.png and /dev/null differ diff --git a/docs/src/assets/inventory_integer_results.png b/docs/src/assets/inventory_integer_results.png index 0f26dd2..9b80599 100644 Binary files a/docs/src/assets/inventory_integer_results.png and b/docs/src/assets/inventory_integer_results.png differ diff --git a/docs/src/assets/inventory_relaxed_results.png b/docs/src/assets/inventory_relaxed_results.png index 8a16f85..0556bfa 100644 Binary files a/docs/src/assets/inventory_relaxed_results.png and b/docs/src/assets/inventory_relaxed_results.png differ diff --git a/docs/src/casestudies/battery_storage_opf.md b/docs/src/casestudies/battery_storage_opf.md new file mode 100644 index 0000000..9475b34 --- /dev/null +++ b/docs/src/casestudies/battery_storage_opf.md @@ -0,0 +1,691 @@ +# Stochastic battery-storage AC optimal power flow + +```@meta +CurrentModule = DecisionRules +``` + +This chapter is the canonical specification of the battery-storage case study. +It defines the physical problem, information pattern, target-state formulations, +and comparison protocol. The full network equations are collected in +[Appendix A: AC polar formulation](@ref) and +[Appendix B: SOC-WR relaxation](@ref) so the main text can be read as an +experimental design. + +!!! note "Implementation status" + The deterministic ExaModels ACP foundation and reproducible PGLib battery + generator are implemented in the `BatteryStorageOPF` example of + [DecisionRulesExa.jl](https://github.com/LearningToOptimize/DecisionRulesExa.jl). + The stochastic TS-DDR and JuMP/SDDP implementations must conform to this + specification before they are treated as accepted results. + +## Purpose + +The experiment studies whether a policy trained with the true nonconvex AC +network can outperform an SDDP policy whose backward passes use a convex network +relaxation. The intended mechanism is **locational storage mispricing**: + +1. demand uncertainty moves scarcity between network regions; +2. congestion, losses, voltage constraints, and reactive-power limits make one + MWh of battery energy worth different amounts at different buses; +3. SOC-WR can assign different marginal values to those stored MWh than ACP; +4. the resulting SDDP policy can charge or discharge the wrong batteries even + when its algorithm has converged. + +The final comparison is therefore: + +- SDDP: SOC-WR backward passes and ACP forward simulation; +- TS-DDR: training and evaluation with ACP; +- perfect foresight (PF/WS): full-horizon ACP with the complete demand path known. + +Here **battery SoC** always means state of charge. **SOC-WR** always means the +second-order-cone relaxation in lifted voltage-product space. The abbreviation +“SOC” alone is avoided. + +## One-stage chronology + +At the start of stage ``t``: + +1. the previous battery state ``e_t`` is known; +2. the current demand atom ``\xi_t`` is observed; +3. future atoms ``\xi_{t+1:T}`` remain unknown; +4. the policy produces a target next state ``\hat e_{t+1}``; +5. an operational OPF chooses generation, network flows, charging, discharging, + the two-sided active recourse, and ``e_{t+1}``. + +Thus a nonanticipative policy has the form + +```math +\hat e_{t+1}=\pi_\theta(e_t,\xi_{1:t}), +``` + +implemented with a recurrent encoder. PF alone observes the complete path before +making its first decision. SDDP and TS-DDR receive exactly the same current +observation and state. + +The physical interstage state is the vector of battery energies. Generator +dispatch, voltages, branch flows, charge and discharge powers, and the two-sided active recourse +are stage decisions, not states. + +## Data, units, and reproducible case construction + +The network is an unmodified PGLib-OPF case. The initial public benchmark uses +`case300_ieee`; the same constructor works for every compatible PGLib case. + +All model power quantities are per unit on the case base ``S^{base}`` in MVA: + +| Quantity | Model unit | Physical conversion | +|:--|:--|:--| +| active/reactive power | pu | ``S^{base}`` MW/MVAr | +| apparent-power limit | pu | ``S^{base}`` MVA | +| battery energy | pu·h | ``S^{base}`` MWh | +| stage duration ``\Delta t`` | h | unchanged | +| voltage magnitude | pu | unchanged | +| voltage angle | rad | unchanged | + +Conversion occurs once in the data layer. Component identifiers from PGLib are +never assumed consecutive or equal to array positions. + +The default battery fleet is constructed by: + +1. selecting in-service buses with positive active load; +2. sorting the eligible original bus identifiers; +3. sampling distinct buses without replacement with `StableRNG`; +4. sizing total fleet power as a declared fraction of base active demand; +5. splitting that power equally across batteries; +6. setting energy capacity from the declared duration in hours. + +The manifest records the exact MATPOWER filename and SHA-256, PGLib release and +license, package versions, placement rule and seed, ordered battery buses, every +battery parameter, demand process, horizons, scenario seeds, and hashes. The +source-network hash, ordered placement, and battery parameters are verified on +reconstruction. + +No generator price, generator limit, branch parameter, voltage limit, or network +topology may be changed during the declared candidate ladder. Any future overlay +is a new, separately approved experiment. + +## Demand uncertainty + +Demand is the only exogenous uncertainty in the first experiment. Batteries have +no inflow: they charge by purchasing energy from the grid. + +Each bus is assigned to one of ``R`` deterministic, topology-derived regions. +At stage ``t``, atom ``a_t`` supplies a system factor ``L^{a_t}`` and regional +factors ``R_r^{a_t}``. With deterministic daily shape ``h_t``, + +```math +\begin{aligned} +p^d_{t,i} + &= p^{d,0}_i\,h_t L^{a_t}R_{r(i)}^{a_t},\\ +q^d_{t,i} + &= q^{d,0}_i\,h_t L^{a_t}R_{r(i)}^{a_t}. +\end{aligned} +``` + +The same multiplier is applied to active and reactive demand, preserving the +base power factor at every bus. The finite joint support contains a calm atom +and regional-scarcity atoms, so the location of high demand changes without +changing network data or prices. The initial process is stagewise independent, +which permits ordinary finite-support SDDP backward passes. + +Scenario generation is a pure seeded operation. Training and evaluation use +different seeds. Paired evaluation stores the stage-major atom-index matrix; +every method reads that same matrix rather than regenerating scenarios. + +## Battery model + +For battery ``b`` at stage ``t``, charge and discharge powers are continuous and +nonnegative: + +```math +0\le p^{ch}_{t,b}\le \bar p^{ch}_b,\qquad +0\le p^{dis}_{t,b}\le \bar p^{dis}_b. +``` + +The active injection at its host bus is + +```math +p^{bat}_{t,b}=p^{dis}_{t,b}-p^{ch}_{t,b}. +``` + +The first model uses unity power factor: a battery neither injects nor absorbs +reactive power. Its energy balance is + +```math +e_{t+1,b} +=(1-\sigma_b\Delta t)e_{t,b} ++\eta^{ch}_b\Delta t\,p^{ch}_{t,b} +-\frac{\Delta t}{\eta^{dis}_b}p^{dis}_{t,b}, +``` + +with + +```math +\underline e_b\le e_{t,b}\le\bar e_b. +``` + +Here ``\eta^{ch}_b,\eta^{dis}_b\in(0,1]`` are efficiencies and ``\sigma_b`` is +the hourly self-discharge rate; the data must satisfy +``0\le\sigma_b\Delta t<1``. + +The continuous formulation has no binary charge/discharge mode. A nonnegative +throughput price penalizes +``p^{ch}_{t,b}+p^{dis}_{t,b}``, making simultaneous operation economically +dominated when the remaining costs are well formed. Simultaneous operation must +still be measured and reported; binary mode variables are introduced only if +that audit invalidates the continuous model. + +### One-stage reachable energy + +Ignoring the network but enforcing battery power and energy limits, the reachable +interval from ``e_{t,b}`` is + +```math +\begin{aligned} +\ell_{t,b} +&=\max\left\{\underline e_b,\, + (1-\sigma_b\Delta t)e_{t,b} + -\frac{\Delta t}{\eta^{dis}_b}\bar p^{dis}_b\right\},\\ +u_{t,b} +&=\min\left\{\bar e_b,\, + (1-\sigma_b\Delta t)e_{t,b} + +\eta^{ch}_b\Delta t\,\bar p^{ch}_b\right\}. +\end{aligned} +``` + +These bounds prove battery-dynamic reachability only. They do **not** prove that +the associated charge or discharge is feasible under generator, voltage, +reactive-power, or branch limits. + +## Two-sided active recourse — the complete-recourse slack + +The physical model carries a **two-sided active nodal slack**: a bounded-below, +unbounded-above nonnegative pair at **every** bus, not a fraction of local demand. +For each bus ``i`` and stage ``t``, + +```math +d^{+}_{t,i}\ge 0,\qquad d^{-}_{t,i}\ge 0, +``` + +entering only the **active** balance: + +```math +p^{d}_{t,i}-d^{+}_{t,i}+d^{-}_{t,i}+g^{s}_i v_{t,i}^2 +-\!\!\sum_{g\in i}\! p^{g}_{t,g}-\!\!\sum_{b\in i}\!(p^{dis}_{t,b}-p^{ch}_{t,b}) ++\!\!\sum_{\text{from }i}\! p^{fr}+\!\!\sum_{\text{to }i}\! p^{to}=0 . +``` + +``d^{+}`` (active deficit / injection) covers an active-power **shortfall**; +``d^{-}`` (active surplus / absorption) absorbs an active-power **excess**. Because ``d^{+}`` +can inject and ``d^{-}`` can absorb arbitrary local power, the stage subproblem +has **relatively complete recourse**: it is feasible for every incoming SoC and +every dynamically reachable battery target, in **both** the charging and +discharging directions. A target that forces a battery to *charge* at a +network-constrained bus is served by local ``d^{+}``; a target that forces it to +*discharge* into a bus with saturated outgoing branches is absorbed by local +``d^{-}``. This is the classical multistage load-deficit device that lets any +non-anticipative algorithm converge without hitting an infeasible subproblem. + +The slack is **active-only**: it does not touch reactive power, so reactive KCL +remains a **hard equality** with no reactive slack. Batteries are unity-power- +factor, so the battery target moves only active injection; reactive feasibility +is a property of the base network and the feasible demand process, independent of +the target. + +The value of lost load is ``c^{VOLL}=10{,}000`` USD/MWh, giving stage cost + +```math +C^{shed}_t +=c^{VOLL}S^{base}\Delta t +\sum_i \bigl(d^{+}_{t,i}+d^{-}_{t,i}\bigr). +``` + +Both slacks are included in physical operating cost. They are a safety valve: an +accepted scientific run leaves both at zero within its declared numerical +tolerance. A nonzero deficit or surplus on an otherwise sensible target is a +case-design signal (the target is not network-deliverable at that operating +point)—not a solver failure. **The strict stage always solves**; the cost, not +the solver status, reports whether the target was deliverable. + +### The nodal slack is not target slack + +The two mechanisms have different meanings: + +| Mechanism | Relaxes | Unit | Included in reported physical cost? | +|:--|:--|:--|:--| +| nodal slack ``d^{\pm}`` | active nodal power balance | pu; cost from MWh | yes | +| target slack ``\delta^\pm`` | agreement with policy target | pu·h | no | + +Code, output schemas, and prose use `active_deficit`/`active_surplus` for the +first (``d^{+}``/``d^{-}``) and `target_slack` for the second. The active +recourse is an artificial active-balance device, **not** curtailed customer load: +``d^{+}`` may exceed local demand and may be positive where ``p^d = 0``, so it is +never named "load shedding" nor reported as a per-load fraction. A scientific +candidate path requires **both** directions numerically zero within tolerance. + +## Target-state projection + +The policy outputs the desired outgoing battery energy +``\hat e_{t+1,b}``; the OPF determines whether and how to realize it. + +### Strict mode + +Strict mode adds the hard equality + +```math +\hat e_{t+1,b}-e_{t+1,b}=0. +``` + +It has no target slack and no target penalty. With this orientation, the +equality multiplier is defined and finite-difference tested as + +```math +\lambda_{t,b} +=\frac{\partial Q_t}{\partial\hat e_{t+1,b}}, +``` + +up to the solver interface's documented dual convention. The implementation +must test the sign and magnitude rather than infer them from a convention. + +Strict mode is the **primary, default** production and training target because +its multiplier is an economic shadow price uncontaminated by a penalty. Backed by +the two-sided active nodal slack, strict has **complete recourse**: for every +supported PGLib case and every dynamically reachable target—including the exact +reachable endpoints—the strict stage NLP solves and reproduces the target to +within ``10^{-5}``. Battery reachability alone is not a network-feasibility proof, +but the nodal slack makes the strict solve feasible regardless: an +under-deliverable target simply carries a deficit/surplus cost. + +### Soft diagnostic mode + +Soft mode uses two nonnegative target slacks: + +```math +\hat e_{t+1,b}-e_{t+1,b} +-\delta^+_{t,b}+\delta^-_{t,b}=0,\qquad +\delta^+_{t,b},\delta^-_{t,b}\ge0. +``` + +A documented training-only penalty may combine L1 and L2 terms: + +```math +C^{target}_t +=\rho_1\sum_b(\delta^+_{t,b}+\delta^-_{t,b}) ++\frac{\rho_2}{2}\sum_b +\left[(\delta^+_{t,b})^2+(\delta^-_{t,b})^2\right]. +``` + +Soft mode is for diagnosis, warm starts, and penalty sensitivity studies. Its +penalty is excluded from physical operating cost and final policy comparisons; +target violations are reported separately. + +### Reachable target policy + +For a raw network output ``z_{t,b}``, the normalized output is mapped into the +battery interval: + +```math +\hat e_{t+1,b} +=\ell_{t,b}+(u_{t,b}-\ell_{t,b})\,y_{t,b}. +``` + +The canonical default is the project-tested stretched sigmoid + +```math +y_{t,b} +=\operatorname{clamp}\left( +\frac{\operatorname{sigmoid}(z_{t,b})-0.03}{0.94}, +0,\;1-10^{-3}\right). +``` + +It can reach the lower edge while keeping a small margin below the exact upper +edge, which avoids a known interior-point degeneracy at a store-max strict +target. A separately named `hardsigmoidsafe` activation may be retained as an +option, but must use the same safe upper margin and be tested independently. + +Reachability bounds are physical projection data, not learned functions. The +canonical gradient stops through ``\ell`` and ``u``; gradients flow through the +normalized policy output. Recurrent state is reset at every scenario boundary. + +## Stage objective and cost accounting + +For quadratic generator costs expressed in USD/hour at per-unit dispatch, the +physical stage cost is + +```math +\begin{aligned} +C^{phys}_t +={}&\Delta t\sum_g +\left(c_{2g}(p^g_{t,g})^2+c_{1g}p^g_{t,g}+c_{0g}\right)\\ +&+S^{base}\Delta t\sum_b c^{cycle}_b +\left(p^{ch}_{t,b}+p^{dis}_{t,b}\right) ++C^{shed}_t. +\end{aligned} +``` + +Every term, including ``c_{0g}``, is duration-scaled. The complete training +objective is ``C^{phys}_t+C^{target}_t`` in soft mode and ``C^{phys}_t`` in +strict mode. + +Every result reports at least: + +- generator cost; +- battery throughput cost; +- VOLL nodal-slack cost, deficit and surplus MWh; +- target penalty and target violation, if soft; +- physical operating cost; +- reporting-window and look-ahead physical costs separately. + +No target penalty is mixed into PF, SDDP-ACP, or TS-DDR-ACP operating cost. + +## Horizon and terminal treatment + +The total horizon is + +```math +T=T^{report}+T^{lookahead}. +``` + +Every method optimizes both portions. Statistical comparisons use physical cost +only over stages ``1:T^{report}``; the look-ahead cost and terminal battery SoC +are reported separately. The look-ahead buffer discourages end-of-horizon +depletion without adding a salvage value or terminal target. + +There is no default terminal salvage term or terminal energy constraint. If +either is introduced later, it must be fixed before production and identical in +PF, SDDP, and TS-DDR. + +## Methods + +### TS-DDR + +TS-DDR trains a recurrent policy for next-battery-SoC targets. Each training +sample embeds those targets in ACP projection problems. Strict training uses the +target-constraint multipliers as envelope gradients; soft training additionally +requires the declared target-penalty derivatives. + +Training and evaluation use true ACP. Checkpoints contain model parameters, +normalization, architecture, activation and upper margin, battery/process +manifests, horizon, stage duration, seeds, and source hashes. + +### SDDP + +The SDDP baseline uses the battery SoC as the resource state: + +- backward subproblems use SOC-WR and ordinary stock cuts; +- forward simulations use ACP; +- cuts are rebuilt for each frozen candidate; +- no custom cut acceptance, tolerance ladder, or penalty rewrite is allowed. + +The SDDP bound belongs to the relaxed backward model. It is not the SDDP policy's +ACP operating cost and is not the room available for TS-DDR. + +### Perfect foresight + +For each evaluation path, PF solves the full-horizon ACP after seeing every +demand atom. It provides an information-relaxation benchmark: + +```math +\operatorname{room} +=\frac{\mathbb E[C^{SDDP\text{-}ACP}] +-\mathbb E[C^{PF\text{-}ACP}]} +{\mathbb E[C^{SDDP\text{-}ACP}]}. +``` + +This room is only an upper bound on the improvement a nonanticipative policy +could attain. Because ACP is nonconvex, a locally solved PF model is an empirical +benchmark, not a rigorous mathematical lower bound unless global optimality is +certified. + +## Experimental acceptance + +A candidate proceeds to production only if: + +1. ExaModels and JuMP agree with an independent PowerModels ACP reference on + deterministic cases; +2. battery balances, AC residuals, bounds, and target equations close within + declared tolerances; +3. strict case14 and case300 paths solve (always feasible via the nodal slack) + with zero active deficit and zero active surplus within tolerance; +4. simultaneous charge/discharge is negligible; +5. ACP and SOC-WR assign materially different marginal values to energy at + relevant battery buses, and that difference changes battery behavior; +6. SDDP runs normally with clean ACP forward passes; +7. paired PF room is large enough to justify training. + +Final evaluation uses frozen manifests and the same stored scenario matrix for +all methods. It reports per-path PF, SDDP-ACP, and TS-DDR-ACP physical costs, +paired differences, a 95% confidence interval, solver failures, active recourse +(deficit and surplus), battery trajectories, binding network constraints, runtime, hardware, seeds, +and hashes. Scientific success requires the paired TS-DDR-minus-SDDP mean to be +negative with a 95% paired confidence interval excluding zero. + +## Implementation invariants + +The following are model requirements rather than tunable choices: + +- one shared ACP constraint implementation underlies deterministic, stochastic, + training, and evaluation builders; +- active and reactive demand use the same atom multiplier (power factor preserved); +- reactive balance has no independent slack; +- the active nodal slack is two-sided (deficit ``d^{+}`` and surplus ``d^{-}``), + nonnegative, unbounded above at every bus, and priced at VOLL — it gives the + strict target formulation relatively complete recourse; +- strict and soft target formulations have different variable sets; +- target penalties never enter reported physical cost; +- generator and network data remain the original PGLib values; +- CPU and GPU builders represent the same equations; +- no solver status is relabeled and failed paths are never silently discarded. + +## Appendix A: AC polar formulation + +This appendix states the complete per-stage ACP projection. Time subscripts are +omitted where unambiguous. + +### Sets and voltage variables + +Let ``N`` be buses, ``G_i`` generators at bus ``i``, ``B_i`` batteries at bus +``i``, and ``A_i`` directed branch ends leaving bus ``i``. Complex bus voltage is + +```math +V_i=v_i e^{\mathrm j\theta_i}, +\qquad \underline v_i\le v_i\le\bar v_i. +``` + +One angle is fixed in each connected reference component: + +```math +\theta_i=0,\qquad i\in N^{ref}. +``` + +### Generator limits + +```math +\underline p^g_g\le p^g_g\le\bar p^g_g,\qquad +\underline q^g_g\le q^g_g\le\bar q^g_g. +``` + +The limits and polynomial costs are taken directly from PGLib after the single +per-unit conversion. + +### General branch model + +For branch ``k=(i,j)`` let its pi-model—including series admittance, asymmetric +line charging, complex transformer tap, and phase shift—be represented by + +```math +\begin{bmatrix}I_{ij}\\I_{ji}\end{bmatrix} += +\begin{bmatrix} +Y^{ff}_k & Y^{ft}_k\\ +Y^{tf}_k & Y^{tt}_k +\end{bmatrix} +\begin{bmatrix}V_i\\V_j\end{bmatrix}. +``` + +The two complex branch flows are + +```math +S_{ij}=p_{ij}+\mathrm jq_{ij}=V_i I_{ij}^*,\qquad +S_{ji}=p_{ji}+\mathrm jq_{ji}=V_j I_{ji}^*. +``` + +These equations are the four real ACP branch-flow equalities. They retain +transformer taps and shifts and both end shunts; replacing them with a lossless +or single-ended approximation changes the model. + +For clarity, if ``Y^{ff}=a+\mathrm jb`` and +``Y^{ft}=c+\mathrm jd``, the from-end equations are + +```math +\begin{aligned} +p_{ij} +&=a v_i^2+v_iv_j[c\cos(\theta_i-\theta_j) + +d\sin(\theta_i-\theta_j)],\\ +q_{ij} +&=-b v_i^2+v_iv_j[c\sin(\theta_i-\theta_j) + -d\cos(\theta_i-\theta_j)]. +\end{aligned} +``` + +The to-end equations follow identically from ``Y^{tt}``, ``Y^{tf}``, and +``\theta_j-\theta_i``. + +### Branch limits + +Apparent-power limits are enforced at both ends: + +```math +p_{ij}^2+q_{ij}^2\le(\bar s_k)^2,\qquad +p_{ji}^2+q_{ji}^2\le(\bar s_k)^2. +``` + +Voltage angle differences satisfy + +```math +\underline\theta^\Delta_k +\le\theta_i-\theta_j +\le\bar\theta^\Delta_k. +``` + +An absent PGLib thermal limit adds no artificial finite bound. + +### Nodal power balance + +Let bus shunt admittance be ``Y_i^s=g_i^s+\mathrm jb_i^s``. Using branch flows +directed away from the bus, active and reactive KCL are + +```math +\begin{aligned} +\sum_{g\in G_i}p^g_g ++\sum_{b\in B_i}(p^{dis}_b-p^{ch}_b) +-p^{served}_i-g_i^s v_i^2 +&=\sum_{(i,j,k)\in A_i}p_{ij},\\ +\sum_{g\in G_i}q^g_g +-q^{served}_i+b_i^s v_i^2 +&=\sum_{(i,j,k)\in A_i}q_{ij}. +\end{aligned} +``` + +The battery energy equation, power and energy bounds, two-sided active-recourse +terms, target equation for the selected mode, and stage objective from the main +text complete the model. + +## Appendix B: SOC-WR relaxation + +SOC-WR retains the OPF specification—generation, batteries, demand, two-sided +active recourse, costs, KCL, thermal limits, and target equations—but replaces +the nonconvex voltage representation. + +Define the Hermitian voltage-product matrix + +```math +W=VV^*,\qquad +W_{ii}=w_i,\qquad +W_{ij}=w^R_{ij}+\mathrm jw^I_{ij}. +``` + +Voltage bounds become + +```math +(\underline v_i)^2\le w_i\le(\bar v_i)^2. +``` + +Branch flows are affine in ``W``: + +```math +\begin{aligned} +S_{ij} +&=(Y^{ff}_k)^*W_{ii}+(Y^{ft}_k)^*W_{ij},\\ +S_{ji} +&=(Y^{tt}_k)^*W_{jj}+(Y^{tf}_k)^*W_{ji},\\ +W_{ji}&=W_{ij}^*. +\end{aligned} +``` + +The exact lifted ACP model requires + +```math +(w^R_{ij})^2+(w^I_{ij})^2=w_iw_j +``` + +for every branch, together with globally consistent voltage angles around all +network cycles. SOC-WR relaxes the rank-one equality to + +```math +(w^R_{ij})^2+(w^I_{ij})^2\le w_iw_j, +``` + +which is second-order-cone representable, and does not impose global rank-one +cycle consistency. When the angle-difference interval lies inside +``(-\pi/2,\pi/2)``, its standard lifted form is + +```math +\tan(\underline\theta^\Delta_k)w^R_{ij} +\le w^I_{ij}\le +\tan(\bar\theta^\Delta_k)w^R_{ij}, +``` + +with the corresponding valid-domain conditions and strengthening used by +PowerModels. Apparent-power limits at both ends and nodal KCL are unchanged and +remain convex in the lifted variables. + +The canonical implementation is PowerModels' `SOCWRPowerModel`; independent +hand-written versions must match it on objective, bounds, flows, and storage +marginal values before use. SOC-WR is a relaxation, not an AC-feasible network +model. Its solution must be evaluated by a separate ACP forward solve. + +## Appendix C: symbols and references + +| Symbol | Meaning | +|:--|:--| +| ``p^g,q^g`` | generator active/reactive power | +| ``v,\theta,V`` | voltage magnitude, angle, complex voltage | +| ``p_{ij},q_{ij},S_{ij}`` | directed branch-end power flow | +| ``p^d,q^d`` | realized active/reactive demand | +| ``d^{+},d^{-}`` | two-sided active recourse: deficit / surplus (pu) | +| ``p^{ch},p^{dis}`` | battery charge/discharge power | +| ``e`` | battery energy state | +| ``\hat e`` | policy target for outgoing battery energy | +| ``\delta^+,\delta^-`` | soft target slacks | +| ``\Delta t`` | stage duration in hours | +| ``S^{base}`` | network power base in MVA | +| ``W`` | lifted voltage-product matrix | + +Primary references: + +- C. Coffrin et al., + [“PowerModels.jl: An Open-Source Framework for Exploring Power Flow Formulations”](https://doi.org/10.23919/PSCC.2018.8442948), + PSCC 2018. +- S. Babaeinejadsarookolaee et al., + [“The Power Grid Library for Benchmarking AC Optimal Power Flow Algorithms”](https://doi.org/10.48550/arXiv.1908.02788), + IEEE PES Task Force report. +- R. A. Jabr, + [“Radial Distribution Load Flow Using Conic Programming”](https://doi.org/10.1109/TPWRS.2006.879234), + IEEE Transactions on Power Systems, 2006. +- M. V. F. Pereira and L. M. V. G. Pinto, + [“Multi-stage Stochastic Optimization Applied to Energy Planning”](https://doi.org/10.1007/BF01582895), + Mathematical Programming, 1991. +- A. Rosemberg et al., + [“Efficiently Training Deep-Learning Parametric Policies Using Lagrangian Duality”](https://arxiv.org/abs/2405.14973), + 2024. diff --git a/docs/src/casestudies/hydro/index.md b/docs/src/casestudies/hydro/index.md new file mode 100644 index 0000000..9288eca --- /dev/null +++ b/docs/src/casestudies/hydro/index.md @@ -0,0 +1,67 @@ +# Long-term hydrothermal planning + +```@meta +CurrentModule = DecisionRules +``` + +A hydrothermal power system is operated by deciding, every week for years, how +much water to release and how much fuel to burn. Water is free but finite; fuel +is expensive but available. The whole problem is the price of water — a price +that no market quotes and that has to be inferred from what the water will be +worth later, elsewhere on the network, under inflows nobody has seen yet. + +This case study puts two ways of inferring that price against each other on a +real system, under the full nonconvex AC power flow, on identical inflow +scenarios. + +## The question + +**Stochastic dual dynamic programming** is the standard answer, and a very good +one. It builds an explicit value function from cutting planes. But cuts are only +valid if the stage problem is convex, and AC power flow is not — so in practice +the value of water is computed against a *relaxed* network and then applied to +the real one. + +**TS-DDR** needs no such relaxation. It trains a policy that outputs a target +reservoir level for each stage; the stage problem projects that target onto the +true AC feasible set, and the multiplier of the target constraint *is* the +marginal value of water, handed over by the solver at no extra cost. There is no +value function to build and nothing to convexify. + +So: **can a policy learned this way operate a real system as well as a converged +SDDP policy?** + +## The answer + +Trained from random initialisation in about eleven GPU-hours, and evaluated +against SDDP on 500 shared inflow scenarios under true AC physics: + +| | operating cost | +|---|---| +| SDDP | **313,546** | +| TS-DDR, from scratch | **314,023** | + +A difference of **+0.152%** — statistically unambiguous, practically small, and +in SDDP's favour. Not a tie, and not a win: the [Results](@ref "Results: TS-DDR versus SDDP") page says so in +those words and refuses the three obvious overstatements. + +The interesting part is not the number but the mechanism. The learned policy +**under-hedges**: it carries less water than SDDP, runs cheaper for most of the +horizon, and pays the difference back in the closing weeks when it arrives short. +That is visible stage by stage, in storage, in thermal dispatch, and in the +marginal price of energy — which is what makes the result diagnosable rather +than merely reported. + +## How to read this case study + +| page | what it covers | +|---|---| +| [The problem](@ref "The long-term hydrothermal planning problem") | the planning problem itself: reservoir dynamics, cascades, AC network physics, and why the value of water is both locational and temporal | +| [Valuing water: two approaches](@ref) | how SDDP and TS-DDR each arrive at a price for water, what each assumes, and what is held identical so the comparison is about the methods | +| [Results](@ref "Results: TS-DDR versus SDDP") | the measured comparison, its statistics, the physical mechanism behind the difference, and an honest reading | +| [Walkthrough](@ref "Walkthrough") | a runnable, few-minute version on CPU: build the stage problems, construct the policy, roll out, read the value of water, take some gradient steps | + +Everything needed to reproduce the published numbers — the case, both trained +policies, the evaluation protocol and the figures — ships with +`examples/HydroPowerModels` in this package and its companion, +DecisionRulesExa.jl. Their READMEs carry the commands. diff --git a/docs/src/casestudies/hydro/method.md b/docs/src/casestudies/hydro/method.md new file mode 100644 index 0000000..bd0cd26 --- /dev/null +++ b/docs/src/casestudies/hydro/method.md @@ -0,0 +1,180 @@ +# Valuing water: two approaches + +```@meta +CurrentModule = DecisionRules +``` + +Both methods solve the same planning problem and differ in exactly one respect: +how they decide what stored water is worth. Everything else — the network, the +demand, the inflow scenarios, the cost of shedding load, the horizon, the solver +tolerances, and the requirement that the reported dispatch satisfy the true AC +equations — is held identical, so the measured difference is attributable to the +method rather than to the setup. + +## SDDP: build the value function, but convexify to do it + +Stochastic dual dynamic programming approximates the future cost of leaving +water behind by an outer envelope of cutting planes, refined by sweeping forward +and backward through the horizon. It is the standard method for this problem and +it converges to a genuine lower bound on the cost. + +The bound is only valid if each stage problem is **convex in the incoming +state**, which AC power flow is not. The universal practical compromise, used +here, is to run the backward pass — the one that generates cuts — on a +second-order-cone relaxation of the network, and to simulate the resulting policy +forward on the true AC model. + +That compromise has a price, and naming it is the point of comparing at all. The +value of water is computed against a network that is easier to deliver power +across than the real one. Where the relaxation is loose — meshed corridors, +binding voltage limits, load far from generation — delivery is underpriced, and +storage decisions inherit the mispricing exactly where they matter most. + +Nothing else about the baseline is tuned in TS-DDR's favour or against it: cut +generation is stock SDDP, with no regularisation ladder and no retry-until-optimal +loop that would change which duals become cuts. + +## TS-DDR: skip the value function, read the price off the solver + +TS-DDR trains a policy that emits a **target** reservoir level for each stage. +The stage problem is then solved with the outgoing storage pinned to that target +by a hard equality — no slack, no penalty term. Two things follow. + +First, the dual of that equality is precisely ``\partial Q_t / \partial \hat v_t``: +the marginal value of water, delivered by the solver as a by-product of solving +the stage. There is no value function to construct, and therefore nothing to +convexify. Training and evaluation both use the **true AC model** end to end. + +Second, hard equalities are only well posed if every target the policy emits is +actually attainable in one stage. That is what the reachable policy guarantees: +a network output is mapped into the one-stage reachable interval of each +reservoir, and then clamped down the cascade so a downstream unit can never be +told to hold water that the unit above it did not release. The reachable +interval itself is a property of the water balance and is derived in +[The problem](@ref "One-stage reachable sets"); what matters for training is that +its endpoints depend on the state, and that the derivative has to go through +them. That is the next section. + +## The gradient must flow through the reachable map + +Strict mode makes the stage a projection: the policy emits a target +``\hat v_t`` and the stage problem is solved with ``v_t = \hat v_t`` enforced as +an equality. The multiplier ``\lambda_t`` of that equality is exactly +``\partial Q_t / \partial \hat v_t`` — the marginal value of water, delivered by +the solver at no extra cost. TS-DDR's actor gradient is then + +```math +\nabla_\theta \; \sum_t \bigl\langle \lambda_t,\; \hat v_t(\theta) \bigr\rangle , +``` + +so everything hinges on differentiating the map ``\theta \mapsto \hat v_t`` +**completely**. That map is not just the network and a sigmoid: it is + +```math +\hat v_{r,t} + \;=\; \ell_r(v_{t-1}, w_t) \;+\; + \bigl(u_r(v_{t-1}, w_t) - \ell_r(v_{t-1}, w_t)\bigr)\,\sigma(z_r), +``` + +followed by the cascade clamp. The bounds carry the state, so +``\partial \hat v_t / \partial v_{t-1}`` is nonzero *through them* even when the +network output ``z`` is held fixed — and since ``v_{t-1}`` is itself the previous +stage's target, this term is precisely what couples the stages. + +This is worth spelling out because getting it wrong is silent. Declaring the +bounds non-differentiable still produces a gradient, still trains, and still +reduces the loss; it simply descends a different direction. Measured on this +case over the full horizon: + +| | truncated | complete | +|---|---|---| +| ``\cos(\nabla_{\text{AD}}, \nabla_{\text{FD}})`` | 0.93 | **1.000000** | +| ``\|\nabla_{\text{AD}}\| / \|\nabla_{\text{FD}}\|`` | 0.059 | **1.000000** | +| ``\|\nabla\|`` at the same point | 28,991 | **491,674** | + +The truncated gradient is a 17-times-too-short vector pointing 48 degrees off. +The practical consequence was not a failure to train but a *wrong conclusion +about the method*: with a gradient carrying 6% of the magnitude, raising the +learning rate could not help, and a learning-rate sweep duly reported "learning +rate is not the lever". On the repaired gradient the ordering inverts and the +learning rate becomes the dominant lever. **Verify the complete actor gradient +against finite differences before spending a campaign on hyperparameters.** + +Two properties make the finite-difference check trustworthy here. The map is +piecewise affine in ``z``, so a difference taken across a kink is meaningless; +the check therefore measures the distance to the nearest kink and asserts that +the perturbation stays inside it. And the forward map must be *unchanged* by the +repair — a gradient fix that moves the policy's output is a different policy, so +the forward value is pinned bit-for-bit before the derivative is compared. + +## Training, and why it is staged + +The policy is trained from a random initialisation. Training runs in **phases**, +each a separate process that restarts from the policy the previous phase +selected. A restart is the point, not an artefact: the optimiser state, the +learning-rate schedule and its warm-up all begin again. + +The phases move two knobs in opposite directions: + +| phase | sampling per gradient step | learning rate | role | +|---|---|---|---| +| 1 | low | high | bulk descent — a noisy, cheap gradient is enough to make fast progress | +| 2 | low | high | continued descent from a better initialisation | +| 3 | raised | dropped | convergence — a precise gradient and a small step | + +The rule behind this is worth stating because it generalises: a **small sample +gives a noisy but cheap gradient**, which is what bulk descent wants; a **large +sample gives a precise one**, which is what final convergence wants. Pairing a +large sample with a small learning rate from the start is the flat quadrant — it +buys precision the optimiser cannot yet use and makes almost no progress. + +The schedule is declared as configuration and executed by a driver, so the +published run is reproduced by running the declared schedule rather than by +following a narrative. + +## Selection, and why this is not overfitting + +Learned policies invite a fair suspicion: that the reported number is the best of +many attempts on the data it was chosen with. Three properties of this study are +designed to answer it. + +**Checkpoints are selected on a small fixed panel, never on the training loss.** +The panel is a handful of scenarios with common random numbers, evaluated over +the reported horizon. Two requirements are enforced in code rather than by +convention: the evaluation must be **complete** — a mean over a scenario that +failed to solve is a mean over a different denominator, and no tolerance makes +that comparable — and it must show **no load shedding**. + +**The published claim is measured on a different, much larger protocol** — 500 +paired scenarios that no checkpoint was ever selected against. The selection +panel turns out to have been directionally right and slightly optimistic about +the level, which is what a small screening set should be expected to be, and is +why the claim does not rest on it. + +**The discarded work is reported.** One additional phase was attempted and +produced no selectable policy; it is excluded from the lineage and its cost is +included in the honest accounting of how long the result took. A time-to-policy +figure that quietly omits the attempts that failed is not a time-to-policy +figure. + +## What is held identical + +| | SDDP | TS-DDR | +|---|---|---| +| network, hydro topology, inflow scenarios | same | same | +| demand profile | same | same | +| uncertainty | inflow only | inflow only | +| initial reservoir state | same | same | +| water balance | same | same | +| price of shedding load | same | same | +| reactive balance | hard, no slack | hard, no slack | +| branch limits | apparent power, both ends | apparent power, both ends | +| horizon simulated and reported | same | same | +| evaluation scenarios | the shared paired protocol | the same scenarios | +| **model the dispatch must satisfy** | **true AC** | **true AC** | +| | | | +| model used to *value water* | convex relaxation | true AC | +| how the future enters | cutting planes | a learned target | + +The line in the middle is the whole experiment. Above it, the two are the same +problem; below it, they are two different answers to what water is worth. diff --git a/docs/src/casestudies/hydro/problem.md b/docs/src/casestudies/hydro/problem.md new file mode 100644 index 0000000..b8de6df --- /dev/null +++ b/docs/src/casestudies/hydro/problem.md @@ -0,0 +1,301 @@ +# The long-term hydrothermal planning problem + +```@meta +CurrentModule = DecisionRules +``` + +**Long-term hydrothermal dispatch (LTHD)** is the coordinated operation of +hydro reservoirs and thermal generation on an AC transmission network over +a multi-year horizon under inflow and demand uncertainty — an instance of +the general problem of [Multistage stochastic optimization](@ref) with the +state given by stored water, the uncertainty by river inflows, and the +stage feasibility set by a nonconvex AC optimal power flow. The system the +case study runs on is presented in +[Results](@ref "Results: TS-DDR versus SDDP"); how each policy arrives at a price for water is +[Valuing water: two approaches](@ref); a runnable version is +[Walkthrough](@ref). + +## Stages, state, uncertainty, decisions + +Time is discretized into **weekly stages** ``t = 1, \ldots, T`` (each stage +the case study trains on ``T`` stages spanning several years, and reports over +a shorter window so the reported horizon is free of end-of-horizon effects). At +each stage: + +- **State** — the vector of reservoir volumes + ``x_t = (v_{r,t})_{r \in \mathcal{R}} \in \mathbb{R}^{n_{\mathrm{hyd}}}``, + the only quantity carried between stages. +- **Uncertainty** — the vector of river inflows + ``w_t = (w_{r,t})_{r \in \mathcal{R}}``, revealed at the start of the + stage. Inflows are strongly seasonal and spatially correlated across the + basin, so realizations are drawn as *joint* scenarios (see + [Uncertainty Sampling](@ref)). Demand follows a fixed profile: inflow is the + only uncertainty, which keeps the comparison a statement about how the two + methods value **water**. +- **Decisions** — the stage dispatch ``u_t``: thermal generation + ``p_{g,t}`` (and reactive ``q_{g,t}``), turbined outflow ``q_{r,t}``, + spillage ``s_{r,t}``, load-shedding (deficit) variables, and the AC + power-flow variables (bus voltage magnitudes and angles). + +## Reservoir dynamics and cascades + +Volumes evolve by **water balance**. Rivers form **cascades**: water +released by an upstream plant arrives at its downstream neighbour within +the same weekly stage (travel times are short relative to the stage +length). For each reservoir ``r``, + +```math +v_{r,t} \;=\; v_{r,t-1} + + K \Bigl( w_{r,t} - q_{r,t} + + \sum_{u \in \mathcal{U}_r} q_{u,t} \Bigr) + - s_{r,t} + \sum_{u \in \mathcal{S}_r} s_{u,t}, +\qquad +v_{r,t} \in [\underline{v}_r,\, \overline{v}_r], +``` + +where + +- ``K`` is the **flow-to-volume conversion factor**: the volume accumulated by + a unit flow sustained over one stage. It therefore scales with the stage + duration, which for long-term planning is long — the case study uses weekly + stages — and the whole water balance is proportional to it; +- note that **turbine flow is scaled by ``K`` and spill is not**: inflow and + turbined outflow are rates (m³/s) while spill is already carried as a volume + in this formulation. The asymmetry is HydroPowerModels' convention and is + reproduced exactly by every engine here; +- ``\mathcal{U}_r`` is the set of plants whose **turbined** water feeds + ``r``, and ``\mathcal{S}_r`` the set whose **spilled** water does — the + two sets need not coincide (some diversions bypass the downstream + turbine intake); +- turbine flow and spill obey their own bounds, + ``q_{r,t} \in [\underline{q}_r, \overline{q}_r]`` and + ``s_{r,t} \ge 0``. + +In the Bolivian system three cascade links are active: **COR → SIS** +(turbine-only: only COR's turbined water reaches SIS) and +**ZON → CHU** and **TAQ1 → TAQ2** (turbine *and* spill). COR is the +system's one large seasonal reservoir; SIS, immediately downstream, is a +run-of-river plant with negligible storage, so every hectometre COR +releases is worth SIS's production factor *in addition to* COR's own — +storage decisions at the head of a cascade are leveraged decisions. + +The water balance is the **only intertemporal coupling** in the problem: +water not released this week is available next week. Everything else — +power flow, generation limits — is contained within the stage. + +## Hydro-to-electric coupling + +A hydro plant converts outflow to active power through its +**production factor** ``\rho_r`` (MW per unit of turbined flow): + +```math +p_{r,t} \;=\; \rho_r \, q_{r,t}, +\qquad 0 \le p_{r,t} \le \rho_r\, \overline{q}_r . +``` + +The production factor differs by an order of magnitude across plants +(in the case study from ``\rho = 1.2`` to ``9.7``), which is why *where* +the system stores and releases water matters as much as *how much*: a +hectometre of water is not a fungible commodity but a location- and +plant-specific quantity of energy. + +## Network physics: AC optimal power flow + +Within each stage, the dispatch must satisfy the full **AC power-flow** +equations on the transmission network ``(\mathcal{N}, \mathcal{E})``. In +polar form, with complex voltage ``V_i = |V_i| e^{j\theta_i}`` at bus +``i`` and admittances ``G, B``: + +```math +\begin{aligned} +&\sum_{g \in \mathcal{G}_i} p_{g,t} + \sum_{r \in \mathcal{R}_i} p_{r,t} + - P^{d}_{i,t} + \Delta_{i,t} + = |V_i| \sum_{k} |V_k| \bigl( G_{ik} \cos\theta_{ik} + B_{ik} \sin\theta_{ik} \bigr), + \\[2pt] +&\sum_{g \in \mathcal{G}_i} q_{g,t} - Q^{d}_{i,t} + = |V_i| \sum_{k} |V_k| \bigl( G_{ik} \sin\theta_{ik} - B_{ik} \cos\theta_{ik} \bigr), +\end{aligned} +\qquad \forall i \in \mathcal{N}, +``` + +with ``\theta_{ik} = \theta_i - \theta_k``, together with voltage bands +``|V_i| \in [\underline{V}_i, \overline{V}_i]``, branch thermal (apparent +power) limits, and generator capability bounds. ``P^d_{i,t}, Q^d_{i,t}`` +are the stage-``t`` bus loads and ``\Delta_{i,t} \ge 0`` is the **deficit** +(unserved load) at bus ``i``. + +These equations are **nonconvex** in the voltage variables. That single +fact drives the methodological fork of this case study: the true cost of +delivering power across a stressed network — losses, reactive support, +voltage margin — is a property of this nonconvex set, and any method that +replaces it with a convex surrogate is pricing delivery on a network that +does not quite exist. We write the whole within-stage feasible set +compactly as ``(x_t, u_t) \in \mathcal{F}_t(w_t)``. + +## Stage cost and objective + +The stage cost is thermal fuel plus a penalty on unserved load: + +```math +c_t(x_t, u_t) \;=\; +\sum_{g \in \mathcal{G}} C_g\bigl(p_{g,t}\bigr) +\;+\; C_{\Delta} \sum_{i \in \mathcal{N}} \Delta_{i,t}, +``` + +with ``C_g`` the (convex, typically affine or quadratic) fuel cost of +thermal unit ``g`` and ``C_\Delta`` the deficit cost, set well above the +most expensive generator so that shedding load is always the last resort. +Hydro production itself is free at the stage level — its cost is +*opportunity cost*, visible only through the intertemporal coupling. + +The planning problem is then exactly the general problem of +[Multistage stochastic optimization](@ref): + +```math +\min_{\pi \in \Pi} \;\; +\mathbb{E}_{w_{1:T}} \Bigl[ \sum_{t=1}^{T} c_t\bigl(x_t^\pi, u_t^\pi\bigr) \Bigr] +\quad \text{s.t.} \quad +\text{water balance},\;\; +(x^\pi_t, u^\pi_t) \in \mathcal{F}_t(w_t) \;\; \forall t, +``` + +over nonanticipative policies ``\pi``. + +## Why this is a planning problem: the value of water + +The economics of LTHD are concentrated in one quantity: the marginal +**value of water**, +``-\partial\, \mathbb{E}[V_{t+1}]/\partial v_{r,t}`` — the expected future +fuel cost avoided by holding one more unit of volume in reservoir ``r`` +now. A *myopic* (greedy) operator, minimizing each week in isolation, +implicitly sets this value to zero and fails in three distinct ways: + +1. **Water has a time value.** Free hydro spent to shave this week's fuel + bill is hydro missing at the seasonal demand peak, when its replacement + is the most expensive thermal unit on the system. When the demand peak + falls in the *dry* season — as in the Bolivian case — the mistake is + maximal: the water most tempting to spend is exactly the water that + will be scarcest when needed. +2. **Relief is locational.** Stored hydro relieves network stress only if + it is stored *upstream of the right plants* and released *in the right + weeks*; through the production factors and the cascade topology, the + same volume is worth different energy in different places. A greedy + dispatch cannot see the future congestion it should be positioning + against. +3. **The forecast is a fan.** Each release is committed before the next + inflow is known. A planning policy hedges across the scenario + distribution; a greedy rule effectively bets on a point forecast and + is caught out by dry sequences. + +The case study does not train a greedy baseline — these failure modes are +structural, not empirical claims — but they explain what any competent +method must accomplish: **bank wet-season inflow, carry it across the +network, and release it against the dry-season peak, hedged across +scenarios.** + +## One-stage reachable sets + +A concept used throughout the strict TS-DDR formulation +(see [Strict mode: penalty-free gradient signal](@ref)) is the +**one-stage reachable set** of the water balance: the set of next-stage +volume vectors attainable from state ``x_{t-1}`` under inflow ``w_t`` by +*some* admissible choice of turbine flows and spills, + +```math +R(x_{t-1}, w_t) \;=\; +\Bigl\{ x_t \;:\; \exists\, (q_t, s_t) \in + [\underline{q}, \overline{q}] \times [0, \overline{s}] + \;\text{ s.t. water balance holds and } x_t \in [\underline{v}, \overline{v}] +\Bigr\}. +``` + +Because the water balance is *linear* in ``(q_t, s_t)``, the per-reservoir +reachable set is an interval whose endpoints follow from substituting the +extreme releases — the property that makes penalty-free (strict) training +practical for hydro. + +### Per-unit reachable bounds + +For reservoir ``r`` at state ``v_{r}`` under inflow ``w_{r}``, the highest +attainable next volume corresponds to minimum outflow plus the worst-case +(maximal) upstream contribution, and the lowest to maximum outflow: + +```math +u_r \;=\; \min\Bigl(\overline{v}_r,\; + v_r + K w_r - K \underline{q}_r + + \sum_{u \in \mathcal{U}_r} K \overline{q}_u\Bigr), +\qquad +\ell_r \;=\; \max\bigl(\underline{v}_r,\; + v_r + K w_r - K \overline{q}_r - \overline{s}_r\bigr), +``` + +with ``\overline{s}_r`` the spill bound; when spillage is unbounded, +``\ell_r = \underline{v}_r`` — the reservoir can always be drawn down to its +physical minimum. A policy that must emit attainable targets therefore has a +natural construction available: squash an unconstrained output into this +interval, + +```math +\hat{v}_r \;=\; \ell_r + (u_r - \ell_r)\,\sigma(z_r), +``` + +The bounds ``\ell_r, u_r`` are functions of the incoming state and the realized +inflow, and they **are differentiated**. An earlier implementation declared them +non-differentiable, which silently truncated ``\partial \hat v_r / \partial +v_r`` to the ``\sigma`` term alone; measured against finite differences over the +full 126-stage horizon, that truncated gradient carried 5.9% of the true +magnitude and pointed 48 degrees away from it, and the error compounds with the +horizon. Restoring the path through ``\ell_r`` and ``u_r`` reproduces the finite +difference to `cos = 1.000000` and `‖AD‖/‖FD‖ = 1.000000`. See +[The gradient must flow through the reachable map](@ref). + +### Cascade-aware clamping + +The fixed upstream term ``\sum_u K \overline{q}_u`` in ``u_r`` is an +**overestimate** whenever an upstream unit stores water: its actual release +is then smaller than ``K \overline{q}_u``, so the fixed bound can exceed the +true reachable set and render a strict subproblem infeasible. After +computing the raw sigmoid targets for all units, the policy therefore clamps +downstream targets against the release actually implied upstream. For each +cascade link ``u \to d``, the implied upstream release is + +```math +R_u \;=\; K w_u + v_u - \hat{v}_u , +``` + +and the maximum contribution reaching ``d`` is ``\max(0, R_u)`` for +turbine-plus-spill links and ``\min(K \overline{q}_u,\, \max(0, R_u))`` for +turbine-only links. The downstream target is clamped to + +```math +\hat{v}_d \;\le\; \min\bigl(\overline{v}_d,\; + v_d + K w_d - K \underline{q}_d + \text{max\_contrib}\bigr). +``` + +Two assumptions are documented for this scheme: + +- **Single-level cascades**: the release formula ``R_u`` omits the upstream + unit's own incoming cascade contribution, which is conservative + (underestimates the release) for multi-level chains — and exact for the + Bolivian topology, whose three links are all single-level. +- **The clamp is a real dependence, not a projection.** Where it binds, the + downstream reachable set genuinely moves with the upstream decision — one more + unit released above is one more unit the unit below can hold — and where the + turbine cap binds instead, it does not. Both branches matter to any method that + differentiates through this map; see + [Valuing water: two approaches](@ref "The gradient must flow through the reachable map"). + +With these bounds and clamps, a target chosen inside the interval is reachable in +one stage from the state it was conditioned on — the condition under which a hard +target equality is well posed at all (see +[Validity in every formulation, by induction](@ref)). + +## Further reading + +- Molzahn & Hiskens, *A survey of relaxations and approximations of the + power flow equations*, Foundations and Trends in Electric Energy + Systems (2019) — where and why conic relaxations of AC power flow are + (in)exact. +- Pereira & Pinto, *Multi-stage stochastic optimization applied to energy + planning*, Mathematical Programming 52 (1991) — the origin of SDDP, in + exactly this application domain. diff --git a/docs/src/casestudies/hydro/results.md b/docs/src/casestudies/hydro/results.md new file mode 100644 index 0000000..6602bda --- /dev/null +++ b/docs/src/casestudies/hydro/results.md @@ -0,0 +1,212 @@ +# Results: TS-DDR versus SDDP + +```@meta +CurrentModule = DecisionRules +``` + +The system is the Bolivian national grid — 28 buses, 31 branches, 34 generators +and 11 hydro units in three cascades — operated over weekly stages under the full +AC power-flow equations. + +```@raw html +The Bolivian interconnected system +``` + +Two features of the instance make the planning problem bite. The cascades are +leveraged: water released by the large seasonal reservoir at the head of a chain +is worth its own production factor *plus* that of the run-of-river plant +immediately below it, so where water is stored matters as much as how much. And +the demand peak falls in the **dry** season, so the water most tempting to spend +is exactly the water that will be scarcest when it is needed. + +```@raw html +Seasonal inflow +``` + +Both policies were evaluated on the same 500 inflow scenarios, drawn once and +shared by every engine. Pairing is what makes the comparison decidable: the +spread of cost across scenarios is about 6,000, while the quantity being measured +is a mean difference of about 480. Comparing unpaired distributions of that shape +would need orders of magnitude more scenarios. + +## The comparison + +| policy | mean operating cost | standard deviation | +|---|---|---| +| SDDP | **313,546.09** | 5,925.42 | +| TS-DDR, trained from scratch | **314,023.62** | 5,998.37 | + +Paired difference, TS-DDR − SDDP, over 500 scenarios: + +| | | +|---|---| +| mean | **+477.53** | +| standard error | 16.25 | +| *t* | 29.38 | +| 95% confidence interval | **[+445.60, +509.46]** | +| relative | **+0.152299%**, CI **[+0.142115%, +0.162483%]** | +| scenarios where TS-DDR is cheaper | **29 of 500** | +| load shed, either policy | none | +| scenarios solved | 500 / 500, both | + +Time to policy, from random initialisation on a single GPU: **10.97 hours** +across three phases and 890 gradient updates. A fourth phase was attempted, +produced no selectable policy, and is excluded from the lineage; including its +cost, the whole search took 11.82 hours. + +```@raw html +From-scratch training history +``` + +The training figure keeps three quantities apart on purpose, because they are +routinely conflated. The **stochastic training loss** is one noisy sample per +update, drawn faintly and smoothed over a fixed number of sampled trajectories — +not a fixed number of updates, since the sample size changes between phases and a +fixed-update window would change the curve's noise for reasons unrelated to +learning. The smoothing resets at each restart. The **fixed-panel evaluation** +that actually selects checkpoints lives on its own axis below, because it differs +in horizon, in sampling and in level; plotting the two together invites reading a +dip in a noisy sample as progress. Evaluations that failed to complete are marked +as refused rather than quietly averaged in. + +## What the difference means + +**TS-DDR is more expensive than SDDP here, by a small but statistically +unambiguous margin.** The gap is 0.15% of operating cost. Its significance — +*t* = 29.4 — is a property of the paired design and the sample size, not of the +effect's size: the difference is fifteen times smaller than the standard +deviation of either policy's own cost distribution. + +Three statements would be wrong, and are not made: + +- **not** that the two policies are equal. They are distinguishable, decisively; +- **not** that TS-DDR beat SDDP. It did not, on the mean. It is cheaper on 29 + scenarios and its best case beats SDDP by 1,657, but the confidence interval + excludes zero by a wide margin; +- **not** that 0.15% is negligible. Whether it matters is an operational question + about the system being planned, not a statistical one. + +What can be said is the honest claim, and it is still an interesting one: **a +policy learned from scratch in eleven GPU-hours, with no value function and no +convex relaxation anywhere in its path, operates this system within 0.15% of a +converged SDDP policy that was given a relaxation to build its cuts with.** + +```@raw html +Cost distributions +Paired differences +``` + +The absolute distributions overlap almost completely — which is the point of the +paired design, since the difference between them is far smaller than either one's +spread. The paired differences resolve what the overlay cannot. + +## Where the difference comes from + +The aggregate hides the mechanism. Cumulatively, TS-DDR runs **cheaper** than +SDDP through most of the horizon — by about 1,700 at its widest — and the entire +final difference is incurred in the closing weeks. + +| stage | cumulative Δcost | thermal MW T/S | hydro MW T/S | reservoir 2 storage T/S | +|---|---|---|---|---| +| 1 | −40 | 204.3 / 207.3 | 285.1 / 283.6 | 6.8 / 7.5 | +| 12 | −577 | 208.5 / 211.6 | 280.0 / 277.4 | 102.5 / 104.0 | +| 48 | −860 | 209.8 / 212.7 | 277.6 / 274.8 | 2.6 / 5.8 | +| 62 | −1,689 | 201.9 / 208.4 | 286.3 / 279.6 | 110.5 / 115.7 | +| 90 | −2 | 214.1 / 212.3 | 272.9 / 274.8 | 3.3 / 5.3 | +| 96 | **+478** | 211.8 / 195.3 | 275.7 / 292.6 | 0.2 / 0.4 | + +The policy **under-hedges**. It carries persistently less water than SDDP — +concentrated in the system's large seasonal reservoir — spends it to run cheaper +early, and arrives at the closing weeks short, substituting thermal generation +for hydro exactly when hydro is most valuable. + +```@raw html +Stagewise physical comparison +``` + +The same story appears in the price of energy, and it appears *cyclically* rather +than as a single drift. The difference in marginal cost tracks the reservoir +cycle: TS-DDR prices energy **below** SDDP while the reservoirs are refilling, +and **above** SDDP in the weeks just after each storage peak, when it is drawing +down a stock it did not build as high. + +Every run of at least three consecutive stages of one sign, with its mean +difference — shorter flips are sampling noise on a ten-scenario mean and are not +listed: + +| stages | sign | mean difference | +|---|---|---| +| 1–5 | TS-DDR cheaper | −25.1 | +| 7–15 | TS-DDR cheaper | −33.5 | +| 20–30 | **TS-DDR dearer** | +14.0 | +| 34–39 | TS-DDR cheaper | −11.9 | +| 41–44 | TS-DDR cheaper | −20.0 | +| 46–48 | TS-DDR cheaper | −11.4 | +| 50–59 | TS-DDR cheaper | −38.8 | +| 61–64 | TS-DDR cheaper | −29.7 | +| 65–83 | **TS-DDR dearer** | +10.8 | +| 88–90 | **TS-DDR dearer** | +12.8 | +| 93–96 | **TS-DDR dearer** | +23.5 | + +Reservoir storage peaks near stages 15 and 63, and the two long dear bands open +at 20 and 65 — just after each peak, when the policy is drawing down a stock it +did not build as high. The final band is the largest, and it is where the +cumulative cost difference is actually paid. + +This table is printed by `plot_hydro_results.jl` alongside the figure, so it is +regenerated from the evidence rather than transcribed once. + +The two price *levels* differ by well under a percent and are not distinguishable +by eye, which is why the difference is drawn on its own axis below them rather +than left to the reader to infer from two overlaid curves. + +```@raw html +Marginal cost of energy +``` + +This also means a **short-horizon evaluation would have ranked TS-DDR ahead of +SDDP**. Only the full reported horizon exposes the under-hedge — a good reason to +fix the reporting window before running the comparison rather than after seeing +it. + +## An aside on the initial state + +The reservoirs start empty. This was very nearly "repaired" to a fraction of +capacity, on the assumption that an empty start would force load shedding and +make the comparison vacuous. The assumption was tested and is false: across the +full evaluation the per-bus load-shedding slack sits at its zero bound at every +stage for both policies, within interior-point tolerance and never above it. The +system is operable from empty, so the inputs were left alone. + +It is a small thing, but it is the kind of assumption that quietly becomes a +modelling change if nobody measures it. + +## What transfers + +Stated without pretending these are universal hyperparameters: + +1. **Verify the complete policy gradient against finite differences before a long + run.** A truncated gradient still trains and still lowers the loss; it simply + descends the wrong direction, and every hyperparameter conclusion drawn on top + of it is wrong too. This study cost itself a campaign's worth of conclusions + that way; the correction is + [documented](@ref "The gradient must flow through the reachable map"). +2. **Keep the training signal and the selection signal apart** — different + horizon, different sampling, different level. Select on one of them only. +3. **Select on complete evaluations.** A silently dropped scenario changes the + denominator, and no tolerance makes the result comparable. +4. **Treat restarts as transients.** Raising the learning rate at a restart makes + things worse before better; a stopping rule that does not know this will kill + the phase inside the dip. +5. **Couple the sample size and the learning rate**, and move both. +6. **Finish on one fresh, larger protocol** the policy was never selected on. +7. **Report the discarded work.** + +## Reproducing this + +Both trained policies ship with the packages, so the comparison can be verified +without retraining either one: verify the case, regenerate the stage models, +evaluate both policies on the shared protocol, merge, and plot. Retraining from +scratch runs the declared schedule; retraining the baseline runs SDDP to +convergence. Both take hours. The commands are in the example READMEs of +`DecisionRules.jl` and `DecisionRulesExa.jl`. diff --git a/docs/src/casestudies/hydro/walkthrough.jl b/docs/src/casestudies/hydro/walkthrough.jl new file mode 100644 index 0000000..8d2501f --- /dev/null +++ b/docs/src/casestudies/hydro/walkthrough.jl @@ -0,0 +1,199 @@ +# # Walkthrough +# +# This walkthrough builds the Bolivian hydrothermal planning problem, constructs +# the feasibility-guaranteeing policy that strict TS-DDR needs, and takes a few +# gradient steps — on CPU, in a few minutes, on a horizon short enough to watch. +# +# It is deliberately *not* the published run. That took eleven GPU-hours over 126 +# stages. The point here is that every piece of it is visible in a script you can +# execute, and that the pieces are the ones the result depends on. Where this +# simplifies, it says so. +# +# The case, the numbers and the honest reading of the comparison are in +# [Results](@ref "Results: TS-DDR versus SDDP"); the mathematics is in +# [The long-term hydrothermal planning problem](@ref). + +# ## Setup +# +# Run from `examples/HydroPowerModels`, whose `Project.toml` carries everything +# used below. + +using DecisionRules +using JuMP, DiffOpt, Ipopt +using Flux +using Random +using Statistics + +HYDRO_DIR = joinpath(pkgdir(DecisionRules), "examples", "HydroPowerModels") #hide +nothing #hide + +# ## 1. The system +# +# The case is the Bolivian national grid operated over weekly stages: a +# transmission network with thermal units, and a set of reservoirs linked into +# cascades, driven by historical inflow scenarios. Three files describe it — the +# network, the hydro topology, and the inflows. + +CASE_DIR = joinpath(HYDRO_DIR, "bolivia") + +# ## 2. Build the stage problems +# +# `build_hydropowermodels` reads one serialized stage model per stage — produced +# from the case by `export_subproblem_mof.jl` through HydroPowerModels — and +# re-parameterizes it: the incoming reservoir state becomes a parameter, the +# inflow becomes a parameter, and in **strict** mode the outgoing state is bound +# to a target parameter by a hard equality. +# +# A short horizon keeps this runnable; the published run uses 126. + +include(joinpath(HYDRO_DIR, "load_hydropowermodels.jl")) +include(joinpath(HYDRO_DIR, "hydro_reachable_policy.jl")) + +NUM_STAGES = 3 + +diff_optimizer = () -> DiffOpt.diff_optimizer( + optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0), +) + +subproblems, state_params_in, state_params_out, uncertainty_samples, + initial_volumes, max_volume, hydro_meta = build_hydropowermodels( + CASE_DIR, "ACPPowerModel.mof.json"; + num_stages = NUM_STAGES, optimizer = diff_optimizer, strict = true, +) + +(stages = length(subproblems), reservoirs = hydro_meta.nHyd, + inflow_scenarios = length(uncertainty_samples[1]), K = hydro_meta.K) + +# In strict mode there are **no slack variables on the target** and no penalty +# term. The dual of `reservoir_out == target` is therefore the clean marginal +# value of water, ``\partial Q_t / \partial \hat v_t``, with no penalty noise +# mixed into it. That is the entire reason strict mode exists. +# +# It is only well posed if every target the policy emits is reachable in one +# stage — otherwise the equality makes the stage infeasible. Hence the policy. + +# ## 3. The reachable policy +# +# `hydro_reachable_policy` maps an unconstrained network output into the +# one-stage reachable interval of each reservoir, +# +# ```math +# \hat v_r \;=\; \ell_r(v, w) + \bigl(u_r(v, w) - \ell_r(v, w)\bigr)\,\sigma(z_r), +# ``` +# +# and then applies a cascade clamp, so a downstream target can never assume more +# water than the upstream unit actually released. +# +# Two details carry the published result: +# +# * the activation is a **stretched** sigmoid onto `[0, 1 - 1e-3]`, not a plain +# one. A plain sigmoid cannot attain the ends of the interval, and the good +# policy on this case puts a substantial share of its targets exactly at a +# feasibility extreme; +# * the bounds ``\ell_r, u_r`` depend on the incoming state, and that dependence +# **is differentiated**. Treating it as constant still trains and still lowers +# the loss, while descending a direction 48 degrees off the true gradient. + +Random.seed!(42) +policy = hydro_reachable_policy(hydro_meta, [128, 128]; combiner_layers = [256, 256]) +nothing #hide + +# The encoder is an LSTM over the **inflow** sequence only; the reservoir state +# enters through the state-conditioned head, not through the recurrence. + +# ## 4. One rollout +# +# A rollout threads the realized state: the policy sees the state it actually +# reached, emits a target, the stage problem projects that target onto the +# feasible set, and the realized outgoing state becomes the next stage's input. + +# Written as a function rather than a bare loop: at top level (and inside a +# documentation `@example` block) a `for` introduces its own scope, so a +# loop-carried `total_cost += ...` would fail with `UndefVarError`. This is a +# recurring Julia trap in exactly this kind of script. + +function rollout(policy, scenario) + state = Float64.(initial_volumes) + total = 0.0 + for t in 1:NUM_STAGES + for (j, param) in enumerate(state_params_in[t]) + set_parameter_value(param, state[j]) + end + for (param, val) in scenario[t] + set_parameter_value(param, val) + end + + w = Float32.([val for (_, val) in scenario[t]]) + target = policy(vcat(w, Float32.(state))) + for j in 1:hydro_meta.nHyd + set_parameter_value(state_params_out[t][j][1], Float64(target[j])) + end + + optimize!(subproblems[t]) + @assert termination_status(subproblems[t]) in + (MOI.LOCALLY_SOLVED, MOI.OPTIMAL, MOI.ALMOST_LOCALLY_SOLVED) + total += objective_value(subproblems[t]) + + for j in 1:hydro_meta.nHyd + state[j] = value(state_params_out[t][j][2]) + end + end + return total, state +end + +scenario = [uncertainty_samples[t][1] for t in 1:NUM_STAGES] +total_cost, final_state = rollout(policy, scenario) +total_cost + +# Every stage solved, from an untrained policy: the reachable map did its job and +# no emitted target was infeasible. That property is what makes hard target +# equalities usable at all. + +# ## 5. The marginal value of water +# +# The multiplier of the target equality is what TS-DDR differentiates, and it +# comes straight off the solved stage — no extra machinery: + +lambda = [DecisionRules.pdual(state_params_out[NUM_STAGES][j][1]) + for j in 1:hydro_meta.nHyd] +round.(lambda; digits = 3) + +# A negative entry means holding one more unit in that reservoir *lowers* future +# cost — water has value there. The spread across reservoirs is the locational +# content that the production factors and the cascade topology create: a cubic +# metre of water is not a fungible commodity. + +# ## 6. A few training steps +# +# `train_multistage` assembles the loop: sample a scenario, roll out, collect the +# multipliers, backpropagate through the policy, step the optimizer. Three +# iterations here; the published run took 890 across three restarted stages. + +# `uncertainty_samples` is passed DIRECTLY: `DecisionRules.sample` has an +# overload for it that draws one inflow scenario per stage, which is exactly the +# per-stage joint sampling this case needs. + +DecisionRules.train_multistage( + policy, initial_volumes, subproblems, + state_params_in, state_params_out, uncertainty_samples; + num_train_per_batch = 2, + num_batches = 3, + optimizer = Flux.Adam(1e-3), +) + +# Three updates on three stages will not produce a good policy, and the number +# above is not meaningful on its own — it is one noisy sample of a stochastic +# objective, and the trap this case study keeps returning to: the training loss, +# the training objective and the fixed-panel evaluation are three different +# quantities, and only the last one selects a policy. + +# ## Where to go from here +# +# The published run is the same construction at full scale — a longer horizon, +# a training schedule of several phases, and a GPU. How each method arrives at a +# price for water is [Valuing water: two approaches](@ref); what the comparison +# measured, and what it does and does not say, is [Results](@ref "Results: TS-DDR versus SDDP"). +# +# To actually run it, the example READMEs of `DecisionRules.jl` and +# `DecisionRulesExa.jl` carry the commands, from verifying the case through to +# regenerating the figures. diff --git a/docs/src/casestudies/hydro/walkthrough.md b/docs/src/casestudies/hydro/walkthrough.md new file mode 100644 index 0000000..e9b99a5 --- /dev/null +++ b/docs/src/casestudies/hydro/walkthrough.md @@ -0,0 +1,218 @@ +```@meta +EditURL = "walkthrough.jl" +``` + +# Walkthrough + +This walkthrough builds the Bolivian hydrothermal planning problem, constructs +the feasibility-guaranteeing policy that strict TS-DDR needs, and takes a few +gradient steps — on CPU, in a few minutes, on a horizon short enough to watch. + +It is deliberately *not* the published run. That took eleven GPU-hours over 126 +stages. The point here is that every piece of it is visible in a script you can +execute, and that the pieces are the ones the result depends on. Where this +simplifies, it says so. + +The case, the numbers and the honest reading of the comparison are in +[Results](@ref "Results: TS-DDR versus SDDP"); the mathematics is in +[The long-term hydrothermal planning problem](@ref). + +## Setup + +Run from `examples/HydroPowerModels`, whose `Project.toml` carries everything +used below. + +````@example walkthrough +using DecisionRules +using JuMP, DiffOpt, Ipopt +using Flux +using Random +using Statistics + +HYDRO_DIR = joinpath(pkgdir(DecisionRules), "examples", "HydroPowerModels") #hide +nothing #hide +```` + +## 1. The system + +The case is the Bolivian national grid operated over weekly stages: a +transmission network with thermal units, and a set of reservoirs linked into +cascades, driven by historical inflow scenarios. Three files describe it — the +network, the hydro topology, and the inflows. + +````@example walkthrough +CASE_DIR = joinpath(HYDRO_DIR, "bolivia") +```` + +## 2. Build the stage problems + +`build_hydropowermodels` reads one serialized stage model per stage — produced +from the case by `export_subproblem_mof.jl` through HydroPowerModels — and +re-parameterizes it: the incoming reservoir state becomes a parameter, the +inflow becomes a parameter, and in **strict** mode the outgoing state is bound +to a target parameter by a hard equality. + +A short horizon keeps this runnable; the published run uses 126. + +````@example walkthrough +include(joinpath(HYDRO_DIR, "load_hydropowermodels.jl")) +include(joinpath(HYDRO_DIR, "hydro_reachable_policy.jl")) + +NUM_STAGES = 3 + +diff_optimizer = () -> DiffOpt.diff_optimizer( + optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0), +) + +subproblems, state_params_in, state_params_out, uncertainty_samples, + initial_volumes, max_volume, hydro_meta = build_hydropowermodels( + CASE_DIR, "ACPPowerModel.mof.json"; + num_stages = NUM_STAGES, optimizer = diff_optimizer, strict = true, +) + +(stages = length(subproblems), reservoirs = hydro_meta.nHyd, + inflow_scenarios = length(uncertainty_samples[1]), K = hydro_meta.K) +```` + +In strict mode there are **no slack variables on the target** and no penalty +term. The dual of `reservoir_out == target` is therefore the clean marginal +value of water, ``\partial Q_t / \partial \hat v_t``, with no penalty noise +mixed into it. That is the entire reason strict mode exists. + +It is only well posed if every target the policy emits is reachable in one +stage — otherwise the equality makes the stage infeasible. Hence the policy. + +## 3. The reachable policy + +`hydro_reachable_policy` maps an unconstrained network output into the +one-stage reachable interval of each reservoir, + +```math +\hat v_r \;=\; \ell_r(v, w) + \bigl(u_r(v, w) - \ell_r(v, w)\bigr)\,\sigma(z_r), +``` + +and then applies a cascade clamp, so a downstream target can never assume more +water than the upstream unit actually released. + +Two details carry the published result: + +* the activation is a **stretched** sigmoid onto `[0, 1 - 1e-3]`, not a plain + one. A plain sigmoid cannot attain the ends of the interval, and the good + policy on this case puts a substantial share of its targets exactly at a + feasibility extreme; +* the bounds ``\ell_r, u_r`` depend on the incoming state, and that dependence + **is differentiated**. Treating it as constant still trains and still lowers + the loss, while descending a direction 48 degrees off the true gradient. + +````@example walkthrough +Random.seed!(42) +policy = hydro_reachable_policy(hydro_meta, [128, 128]; combiner_layers = [256, 256]) +nothing #hide +```` + +The encoder is an LSTM over the **inflow** sequence only; the reservoir state +enters through the state-conditioned head, not through the recurrence. + +## 4. One rollout + +A rollout threads the realized state: the policy sees the state it actually +reached, emits a target, the stage problem projects that target onto the +feasible set, and the realized outgoing state becomes the next stage's input. + +Written as a function rather than a bare loop: at top level (and inside a +documentation `@example` block) a `for` introduces its own scope, so a +loop-carried `total_cost += ...` would fail with `UndefVarError`. This is a +recurring Julia trap in exactly this kind of script. + +````@example walkthrough +function rollout(policy, scenario) + state = Float64.(initial_volumes) + total = 0.0 + for t in 1:NUM_STAGES + for (j, param) in enumerate(state_params_in[t]) + set_parameter_value(param, state[j]) + end + for (param, val) in scenario[t] + set_parameter_value(param, val) + end + + w = Float32.([val for (_, val) in scenario[t]]) + target = policy(vcat(w, Float32.(state))) + for j in 1:hydro_meta.nHyd + set_parameter_value(state_params_out[t][j][1], Float64(target[j])) + end + + optimize!(subproblems[t]) + @assert termination_status(subproblems[t]) in + (MOI.LOCALLY_SOLVED, MOI.OPTIMAL, MOI.ALMOST_LOCALLY_SOLVED) + total += objective_value(subproblems[t]) + + for j in 1:hydro_meta.nHyd + state[j] = value(state_params_out[t][j][2]) + end + end + return total, state +end + +scenario = [uncertainty_samples[t][1] for t in 1:NUM_STAGES] +total_cost, final_state = rollout(policy, scenario) +total_cost +```` + +Every stage solved, from an untrained policy: the reachable map did its job and +no emitted target was infeasible. That property is what makes hard target +equalities usable at all. + +## 5. The marginal value of water + +The multiplier of the target equality is what TS-DDR differentiates, and it +comes straight off the solved stage — no extra machinery: + +````@example walkthrough +lambda = [DecisionRules.pdual(state_params_out[NUM_STAGES][j][1]) + for j in 1:hydro_meta.nHyd] +round.(lambda; digits = 3) +```` + +A negative entry means holding one more unit in that reservoir *lowers* future +cost — water has value there. The spread across reservoirs is the locational +content that the production factors and the cascade topology create: a cubic +metre of water is not a fungible commodity. + +## 6. A few training steps + +`train_multistage` assembles the loop: sample a scenario, roll out, collect the +multipliers, backpropagate through the policy, step the optimizer. Three +iterations here; the published run took 890 across three restarted stages. + +`uncertainty_samples` is passed DIRECTLY: `DecisionRules.sample` has an +overload for it that draws one inflow scenario per stage, which is exactly the +per-stage joint sampling this case needs. + +````@example walkthrough +DecisionRules.train_multistage( + policy, initial_volumes, subproblems, + state_params_in, state_params_out, uncertainty_samples; + num_train_per_batch = 2, + num_batches = 3, + optimizer = Flux.Adam(1e-3), +) +```` + +Three updates on three stages will not produce a good policy, and the number +above is not meaningful on its own — it is one noisy sample of a stochastic +objective, and the trap this case study keeps returning to: the training loss, +the training objective and the fixed-panel evaluation are three different +quantities, and only the last one selects a policy. + +## Where to go from here + +The published run is the same construction at full scale — a longer horizon, +a training schedule of several phases, and a GPU. How each method arrives at a +price for water is [Valuing water: two approaches](@ref); what the comparison +measured, and what it does and does not say, is [Results](@ref "Results: TS-DDR versus SDDP"). + +To actually run it, the example READMEs of `DecisionRules.jl` and +`DecisionRulesExa.jl` carry the commands, from verifying the case through to +regenerating the figures. + diff --git a/docs/src/examples/hydro.jl b/docs/src/examples/hydro.jl deleted file mode 100644 index e7bf3fd..0000000 --- a/docs/src/examples/hydro.jl +++ /dev/null @@ -1,481 +0,0 @@ -# # Hydropower Scheduling -# -# This example trains target-setting decision rules for the Bolivia -# long-term hydrothermal dispatch (LTHD) problem — both **TS-DDR** (deep, -# LSTM-based) and **TS-LDR** (linear) — and compares them against an SDDP -# baseline with inconsistent formulations. -# -# The Bolivia system has **10 hydro plants**, **96 monthly stages**, and -# **AC power flow** constraints. Inflow uncertainty is sampled from 47 -# historical scenarios. -# -# ## Overview of the TS-DDR approach -# -# Classical stochastic programming (e.g., SDDP) constructs piecewise-linear -# value-function approximations. TS-DDR takes a different route: a neural -# network policy ``\pi_\theta`` maps observations to **target states**, and a -# projection subproblem at each stage enforces physical feasibility while -# tracking those targets as closely as possible. -# -# The key insight is that the gradient of the projection subproblem with -# respect to the target parameters is available through Lagrange duality -# (or equivalently, implicit differentiation of the KKT conditions). -# This avoids differentiating through the full optimization solver. -# -# ## Problem formulation -# -# At each stage ``t``, the operator observes inflows ``w_t`` and the current -# reservoir state ``x_{t-1}``. The policy predicts target volumes: -# -# ```math -# \hat{x}_t = \pi_\theta(w_{1:t},\, x_{t-1}). -# ``` -# -# A stage subproblem projects onto the feasible set: -# -# ```math -# \begin{aligned} -# q_t(x_{t-1},\, w_t;\; \hat{x}_t) -# \;=\; -# \min_{x_t, u_t, \delta_t} -# \quad & -# c_t(x_t, u_t) + C_\delta\, \|\delta_t\| \\ -# \text{s.t.}\quad -# & x_t = x_{t-1} + w_t - \text{turbined}_t - \text{spilled}_t, -# && \text{(reservoir balance)} \\ -# & x_t + \delta_t = \hat{x}_t, -# && : \lambda_t \quad \text{(target constraint)} \\ -# & \text{AC-OPF}(u_t), -# && \text{(power flow)} \\ -# & x_t \in [0, \bar{x}],\; u_t \ge 0. -# \end{aligned} -# ``` -# -# The slack variable ``\delta_t`` absorbs infeasible targets; ``\lambda_t`` is -# the dual multiplier that provides the gradient signal. -# -# ## Gradient computation: the envelope theorem -# -# By the envelope theorem, the sensitivity of the optimal value with respect -# to the target parameter is simply the dual: -# -# ```math -# \frac{\partial q_t}{\partial \hat{x}_t} -# \;=\; -\lambda_t. -# ``` -# -# Combined with backpropagation through the policy network, the full gradient -# of the expected cost is: -# -# ```math -# \nabla_\theta \mathbb{E}[Q] -# \;\approx\; -# \frac{1}{S} \sum_{s=1}^{S} \sum_{t=1}^{T} -# \lambda_t^s \odot \nabla_\theta \hat{x}_t^s(\theta), -# ``` -# -# where ``S`` is the number of sampled trajectories per batch and ``\odot`` -# denotes elementwise multiplication. - -# ## Problem setup -# -# The JuMP subproblems are built from a MOF file (exported from PowerModels.jl) -# plus hydro data (reservoir limits, inflow scenarios). Each subproblem contains: -# - AC optimal power flow constraints -# - Reservoir balance: `vol_out = vol_in + inflow - turbined - spilled` -# - Target-slack deficit variables penalizing deviation from the policy's targets -# -# The helper `build_hydropowermodels` reads the case data, creates one JuMP model -# per stage, and parameterizes the initial volumes and inflows so they can be set -# at each training sample. - -using DecisionRules -using JuMP, DiffOpt, Ipopt -using Flux -using Statistics, Random - -# Load the problem builder (reads MOF + hydro JSON + inflow CSV). -# -# ```julia -# include("load_hydropowermodels.jl") -# ``` - -# ## Building the stage-wise subproblems -# -# Each subproblem is wrapped with `DiffOpt.diff_optimizer` so that Lagrange duals -# and implicit sensitivities are available for training. - -# ```julia -# diff_optimizer = () -> DiffOpt.diff_optimizer( -# optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0, "linear_solver" => "mumps") -# ) -# -# subproblems, state_params_in, state_params_out, uncertainty_samples, initial_state, max_volume = -# build_hydropowermodels( -# "bolivia", "ACPPowerModel.mof.json"; -# num_stages=96, -# optimizer=diff_optimizer, -# penalty_l1=:auto, penalty_l2=:auto, -# ) -# ``` - -# ## Policy architecture -# -# The policy is a [`StateConditionedPolicy`](@ref) with two components: -# -# 1. **Encoder** — a stack of LSTM cells that processes only the uncertainty -# (inflow) sequence, capturing temporal dependencies across stages. -# 2. **Combiner** — a Dense layer that merges the encoded uncertainty with the -# previous state to produce the next target. -# -# At each stage the policy receives ``[w_t;\; x_{t-1}]`` and outputs -# target reservoir volumes ``\hat{x}_t``: -# -# ``` -# ┌─────────┐ ┌────────────────┐ ┌──────────────┐ -# │ w_t │─────▶│ LSTM encoder │─────▶│ │ -# └─────────┘ └────────────────┘ │ Dense │──▶ x̂_t -# ┌─────────┐ │ combiner │ -# │ x_{t-1} │─────────────────────────────▶│ │ -# └─────────┘ └──────────────┘ -# ``` -# -# The LSTM carries hidden state across stages, giving the policy memory of -# past inflows. The activation is `sigmoid` (bounding outputs to ``[0,1]``, -# which is then scaled by the feasibility mapping). - -# ```julia -# models = state_conditioned_policy( -# num_uncertainties, num_hydro, num_hydro, [128, 128]; -# activation=sigmoid, encoder_type=Flux.LSTM, -# ) -# ``` - -# ## TS-LDR: Linear Decision Rules -# -# As a baseline, we also train a **linear** policy (TS-LDR). This uses -# `dense_multilayer_nn` with identity activation — a composition of linear -# layers equivalent to a single affine map: -# -# ```math -# \hat{x}_t = W [w_{1:t};\; x_{t-1}] + b. -# ``` -# -# TS-LDR uses the same target-setting framework and training pipeline as -# TS-DDR. The only difference is the policy class: linear maps have fewer -# parameters and cannot capture nonlinear inflow patterns, but they are a -# natural baseline from the classical LDR literature. - -# ```julia -# num_inputs = DecisionRules.policy_input_dim(num_uncertainties, num_hydro) -# models = dense_multilayer_nn(num_inputs, num_hydro, [64, 64]; activation=identity) -# ``` - -# ## Training pipeline 1: Deterministic Equivalent -# -# The deterministic equivalent (DE) couples all 96 stages into a **single NLP** -# for each sampled trajectory. This is the most direct formulation: the policy -# generates the full target trajectory ``\hat{x}_{1:T}`` in one forward pass, -# and a single coupled solve determines all realized states simultaneously. -# -# ### How it works -# -# ``` -# ┌──────────────────────────────────────────────────────────┐ -# │ For each sampled trajectory w_{1:T}: │ -# │ │ -# │ 1. Forward pass: x̂_{1:T} = π_θ(w_{1:T}, x_0) │ -# │ │ -# │ 2. Solve coupled NLP: │ -# │ min Σ_t c_t(x_t, u_t) + C_δ Σ_t ‖δ_t‖ │ -# │ s.t. dynamics + AC-OPF for ALL stages simultaneously │ -# │ x_t + δ_t = x̂_t(θ) ∀t (target constraint) │ -# │ │ -# │ 3. Read duals λ_t of target constraints │ -# │ Gradient: Σ_t λ_t ⊙ ∇_θ x̂_t(θ) │ -# └──────────────────────────────────────────────────────────┘ -# ``` -# -# ### Mathematical formulation -# -# ```math -# \begin{aligned} -# Q(w;\, \theta) -# \;=\; -# \min_{\{x_t, u_t, \delta_t\}_{t=1}^{T}} -# \quad & -# \sum_{t=1}^{T} c_t(x_t, u_t) -# + C_\delta \sum_{t=1}^{T} \|\delta_t\| \\ -# \text{s.t.}\quad -# & x_t = T_t(w_t,\, u_t,\, x_{t-1}), -# && t=1,\ldots,T \\ -# & x_t + \delta_t = \hat{x}_t(\theta), -# && : \lambda_t,\quad t=1,\ldots,T \\ -# & h_t(x_t, u_t) \ge 0, -# && t=1,\ldots,T -# \end{aligned} -# ``` -# -# The gradient is exact by the envelope theorem: -# -# ```math -# \nabla_\theta Q -# \;=\; -# \sum_{t=1}^{T} -# \lambda_t \odot \nabla_\theta \hat{x}_t(\theta). -# ``` -# -# **Advantages**: strongest gradient signal — full cross-stage coupling -# captures how a target at stage 3 affects costs at stage 50. -# -# **Disadvantage**: the NLP has ``96 \times (\text{AC-OPF variables})`` -# decision variables; the policy generates targets without seeing realized -# states (open-loop target generation). - -# ```julia -# det_equivalent, uncertainty_samples_det = DecisionRules.deterministic_equivalent!( -# det_model, subproblems_de, state_params_in, state_params_out, -# Float64.(initial_state), uncertainty_samples, -# ) -# -# train_multistage( -# models, initial_state, det_equivalent, -# state_params_in, state_params_out, uncertainty_samples; -# num_batches=4000, optimizer=Flux.Adam(), -# penalty_schedule=[(1,100,0.1), (101,210,1.0), (211,300,10.0), (301,4000,30.0)], -# ) -# ``` - -# ## Training pipeline 2: Stage-wise Decomposition (Single Shooting) -# -# Stage-wise decomposition solves one subproblem per stage sequentially. -# Unlike the DE, the policy operates in **closed loop**: after each stage -# solve, the realized state ``x_t`` (not the predicted target) is fed back -# as input to the next stage. -# -# ### How it works -# -# ``` -# ┌─────────────────────────────────────────────────────────────┐ -# │ For each sampled trajectory w_{1:T}: │ -# │ │ -# │ x_0 = initial state │ -# │ for t = 1, ..., T: │ -# │ x̂_t = π_θ(w_t, x_{t-1}) ← predict target │ -# │ solve stage-t subproblem ← project to feasible│ -# │ x_t = realized state from solver ← closed-loop │ -# │ accumulate c_t + C_δ ‖δ_t‖ │ -# │ │ -# │ Gradient: chain rule through all stage solves │ -# └─────────────────────────────────────────────────────────────┘ -# ``` -# -# ### Gradient chain -# -# The gradient must account for how the realized state at stage ``t`` -# depends on the targets at all earlier stages. By the chain rule: -# -# ```math -# \frac{\partial Q}{\partial \hat{x}_t} -# \;=\; -# \lambda_t -# + \sum_{k>t} -# \frac{\partial q_k}{\partial x_{k-1}} -# \cdot \prod_{j=t+1}^{k-1} -# \frac{\partial x_j}{\partial x_{j-1}} -# \cdot \frac{\partial x_t}{\partial \hat{x}_t}. -# ``` -# -# In practice, automatic differentiation (Zygote + ChainRules `rrule`s -# defined on each stage solve) handles this chain automatically. -# The `rrule` for each stage solve reads the dual ``\lambda_t`` for the -# target constraint and uses DiffOpt's implicit differentiation for the -# state-transition sensitivities. -# -# **Advantages**: closed-loop — the policy sees realized states, matching -# deployment semantics. Each solve is small (single-stage AC-OPF). -# -# **Disadvantage**: gradients weaken over long horizons because the -# chain rule multiplies many Jacobians; sequential solve prevents -# parallelism. - -# ```julia -# train_multistage( -# models, initial_state, subproblems, -# state_params_in, state_params_out, uncertainty_samples; -# num_batches=3000, optimizer=Flux.Adam(), -# penalty_schedule=:default_annealed, -# ) -# ``` - -# ## Training pipeline 3: Multiple Shooting -# -# Multiple shooting partitions the ``T``-stage horizon into ``K`` windows of -# ``W`` stages each. Within each window, a local deterministic equivalent -# couples the stages (strong gradient signal). Between windows, the realized -# end-state is passed to the next window (closed-loop continuity). -# -# ### How it works -# -# ``` -# ┌────────────────────────────────────────────────────────────────┐ -# │ Partition T=96 stages into K=⌈96/12⌉=8 windows of W=12 │ -# │ │ -# │ x_0 = initial state │ -# │ for k = 1, ..., K: │ -# │ stages = [(k-1)W+1, ..., kW] │ -# │ x̂_{stages} = π_θ(w_{stages}, x_{start_k}) │ -# │ solve window-k DE (12-stage coupled NLP) │ -# │ x_{end_k} = realized end-state from window solve │ -# │ x_{start_{k+1}} = x_{end_k} │ -# │ │ -# │ Gradient: │ -# │ Within window: duals from the coupled solve (like full DE) │ -# │ Across windows: DiffOpt chain rule through end-states │ -# └────────────────────────────────────────────────────────────────┘ -# ``` -# -# ### Gradient structure -# -# Let ``Q_k`` be the cost of window ``k``. The total cost is -# ``Q = \sum_k Q_k``. Within a window, the gradient is identical to the -# DE case (duals of the target constraints in the coupled model). Across -# windows, the chain rule threads through the realized end-state: -# -# ```math -# \frac{dQ}{d\theta} -# \;=\; -# \sum_{k=1}^{K} -# \left( -# \frac{\partial Q_k}{\partial \hat{x}_k} -# \cdot \frac{\partial \hat{x}_k}{\partial \theta} -# \;+\; -# \frac{\partial Q_k}{\partial x_{\text{start}_k}} -# \cdot \frac{d x_{\text{start}_k}}{d\theta} -# \right), -# ``` -# -# where ``\frac{d x_{\text{start}_k}}{d\theta}`` involves the chain -# through all prior windows via ``x_{\text{end}_{k-1}}``. -# -# **Advantages**: balances gradient quality (12-stage coupling) with -# tractability (8 small DEs instead of one large one); inter-window -# chain provides some closed-loop signal. -# -# **Disadvantage**: window boundaries introduce gradient discontinuities; -# the full-horizon coupling is weaker than the single DE. - -# ```julia -# windows = DecisionRules.setup_shooting_windows( -# subproblems, state_params_in, state_params_out, -# Float64.(initial_state), uncertainty_samples; -# window_size=12, -# model_factory=() -> DiffOpt.nonlinear_diff_model(ipopt_attrs), -# ) -# -# train_multiple_shooting( -# models, initial_state, windows, () -> uncertainty_samples; -# num_batches=3000, optimizer=Flux.Adam(), -# penalty_schedule=:default_annealed, -# ) -# ``` - -# ## Penalty annealing -# -# The target penalty ``C_\delta`` controls the trade-off between following -# the policy's targets and minimizing operational cost. DecisionRules -# supports a **penalty annealing schedule** that ramps the penalty multiplier -# during training: -# -# | Phase | Multiplier | Purpose | -# |:------|:----------:|:--------| -# | Warmup | ``0.1 \times C_\delta`` | Let the policy explore freely | -# | Nominal | ``1.0 \times C_\delta`` | Standard training | -# | Tighten | ``10.0 \times C_\delta`` | Sharpen target tracking | -# | Lock | ``30.0 \times C_\delta`` | Final precision | -# -# This is activated with `penalty_schedule=:default_annealed` or by passing -# an explicit list of `(start_iter, end_iter, multiplier)` tuples. - -# ## Evaluation -# -# After training, we evaluate the policy using stage-wise rollout on held-out -# scenarios. Two modes: -# - **Target feedback** (`policy_state=:target`): the policy receives its own -# predicted target as input, matching DE training semantics. -# - **Realized feedback** (`policy_state=:realized`): the policy receives the -# realized state from the solver, matching deployment semantics. -# -# The **target-violation share** measures how much cost comes from the slack -# penalty rather than actual operations — it should be small (``\le 5\%``) for -# a well-trained policy. - -# ```julia -# rollout_eval = RolloutEvaluation( -# subproblems, state_params_in, state_params_out, initial_state, eval_scenarios; -# stride=1, policy_state=:realized, -# ) -# rollout_eval(1, models) -# println("Operational cost: ", rollout_eval.last_objective_no_deficit) -# println("Violation share: ", rollout_eval.last_violation_share) -# ``` - -# ## SDDP baseline -# -# For comparison, we also train an SDDP policy using -# [SDDP.jl](https://github.com/odow/SDDP.jl) with **inconsistent -# formulations**: a convex SOC-WR relaxation for the backward pass -# (cut generation) and the nonconvex ACP formulation for the forward -# pass (simulation). This is a pragmatic approach when the true problem -# (AC-OPF) is nonconvex — SDDP requires convexity for valid cuts, so a -# convex relaxation approximates the value function while the forward pass -# evaluates under the true physics. -# -# The SDDP policy is trained for up to 2000 iterations and the learned -# cuts are saved to a JSON file, which can be loaded to simulate the -# policy under the ACP formulation. - -# ## Results -# -# The plots below compare the TS-DDR and TS-LDR training formulations and -# the SDDP baseline on the Bolivia case. Training curves, out-of-sample -# cost distributions, reservoir volume trajectories, and thermal generation -# profiles are shown. -# -# ### Training convergence (TS-DDR methods) -# -# ![Training convergence](../assets/hydro_training_convergence.png) -# -# ### Out-of-sample cost (TS-DDR methods) -# -# ![Out-of-sample cost comparison](../assets/hydro_cost_comparison.png) -# -# ### Target-violation share (TS-DDR methods) -# -# ![Violation share](../assets/hydro_violation_share.png) -# -# ### Reservoir volume comparison (all methods) -# -# ![Volume comparison](../assets/hydro_volume_comparison.png) -# -# ### Thermal generation comparison (all methods) -# -# ![Generation comparison](../assets/hydro_generation_comparison.png) -# -# ### Summary -# -# | Method | Policy | Mean Cost | Std | N | -# |:-------|:------:|----------:|----:|--:| -# | TS-DDR (DE) | LSTM | 325 540 | 6 266 | 100 | -# | TS-DDR (DE, anneal) | LSTM | 324 445 | 6 134 | 100 | -# | TS-DDR (shooting w=12) | LSTM | 323 289 | 5 593 | 100 | -# | TS-DDR (shooting w=12, anneal) | LSTM | 322 812 | 6 081 | 100 | -# | TS-DDR (stage-wise, anneal) | LSTM | 321 543 | 6 214 | 100 | -# | SDDP (SOC-WR / ACP) | cuts | 303 684 | — | 100 | -# -# All three TS-DDR methods with penalty annealing converge to similar -# costs (321K–325K). SDDP trains on 126 stages (96 + 30 margin). -# -# !!! note "Preliminary results" -# These numbers reflect the current default training scripts. -# They will be updated as the package evolves. diff --git a/docs/src/examples/hydro.md b/docs/src/examples/hydro.md deleted file mode 100644 index 6c23019..0000000 --- a/docs/src/examples/hydro.md +++ /dev/null @@ -1,488 +0,0 @@ -```@meta -EditURL = "hydro.jl" -``` - -# Hydropower Scheduling - -This example trains target-setting decision rules for the Bolivia -long-term hydrothermal dispatch (LTHD) problem — both **TS-DDR** (deep, -LSTM-based) and **TS-LDR** (linear) — and compares them against an SDDP -baseline with inconsistent formulations. - -The Bolivia system has **10 hydro plants**, **96 monthly stages**, and -**AC power flow** constraints. Inflow uncertainty is sampled from 47 -historical scenarios. - -## Overview of the TS-DDR approach - -Classical stochastic programming (e.g., SDDP) constructs piecewise-linear -value-function approximations. TS-DDR takes a different route: a neural -network policy ``\pi_\theta`` maps observations to **target states**, and a -projection subproblem at each stage enforces physical feasibility while -tracking those targets as closely as possible. - -The key insight is that the gradient of the projection subproblem with -respect to the target parameters is available through Lagrange duality -(or equivalently, implicit differentiation of the KKT conditions). -This avoids differentiating through the full optimization solver. - -## Problem formulation - -At each stage ``t``, the operator observes inflows ``w_t`` and the current -reservoir state ``x_{t-1}``. The policy predicts target volumes: - -```math -\hat{x}_t = \pi_\theta(w_{1:t},\, x_{t-1}). -``` - -A stage subproblem projects onto the feasible set: - -```math -\begin{aligned} -q_t(x_{t-1},\, w_t;\; \hat{x}_t) - \;=\; - \min_{x_t, u_t, \delta_t} - \quad & - c_t(x_t, u_t) + C_\delta\, \|\delta_t\| \\ -\text{s.t.}\quad - & x_t = x_{t-1} + w_t - \text{turbined}_t - \text{spilled}_t, - && \text{(reservoir balance)} \\ - & x_t + \delta_t = \hat{x}_t, - && : \lambda_t \quad \text{(target constraint)} \\ - & \text{AC-OPF}(u_t), - && \text{(power flow)} \\ - & x_t \in [0, \bar{x}],\; u_t \ge 0. -\end{aligned} -``` - -The slack variable ``\delta_t`` absorbs infeasible targets; ``\lambda_t`` is -the dual multiplier that provides the gradient signal. - -## Gradient computation: the envelope theorem - -By the envelope theorem, the sensitivity of the optimal value with respect -to the target parameter is simply the dual: - -```math -\frac{\partial q_t}{\partial \hat{x}_t} -\;=\; -\lambda_t. -``` - -Combined with backpropagation through the policy network, the full gradient -of the expected cost is: - -```math -\nabla_\theta \mathbb{E}[Q] -\;\approx\; -\frac{1}{S} \sum_{s=1}^{S} \sum_{t=1}^{T} - \lambda_t^s \odot \nabla_\theta \hat{x}_t^s(\theta), -``` - -where ``S`` is the number of sampled trajectories per batch and ``\odot`` -denotes elementwise multiplication. - -## Problem setup - -The JuMP subproblems are built from a MOF file (exported from PowerModels.jl) -plus hydro data (reservoir limits, inflow scenarios). Each subproblem contains: -- AC optimal power flow constraints -- Reservoir balance: `vol_out = vol_in + inflow - turbined - spilled` -- Target-slack deficit variables penalizing deviation from the policy's targets - -The helper `build_hydropowermodels` reads the case data, creates one JuMP model -per stage, and parameterizes the initial volumes and inflows so they can be set -at each training sample. - -````@example hydro -using DecisionRules -using JuMP, DiffOpt, Ipopt -using Flux -using Statistics, Random -```` - -Load the problem builder (reads MOF + hydro JSON + inflow CSV). - -```julia -include("load_hydropowermodels.jl") -``` - -## Building the stage-wise subproblems - -Each subproblem is wrapped with `DiffOpt.diff_optimizer` so that Lagrange duals -and implicit sensitivities are available for training. - -```julia -diff_optimizer = () -> DiffOpt.diff_optimizer( - optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0, "linear_solver" => "mumps") -) - -subproblems, state_params_in, state_params_out, uncertainty_samples, initial_state, max_volume = - build_hydropowermodels( - "bolivia", "ACPPowerModel.mof.json"; - num_stages=96, - optimizer=diff_optimizer, - penalty_l1=:auto, penalty_l2=:auto, - ) -``` - -## Policy architecture - -The policy is a [`StateConditionedPolicy`](@ref) with two components: - -1. **Encoder** — a stack of LSTM cells that processes only the uncertainty - (inflow) sequence, capturing temporal dependencies across stages. -2. **Combiner** — a Dense layer that merges the encoded uncertainty with the - previous state to produce the next target. - -At each stage the policy receives ``[w_t;\; x_{t-1}]`` and outputs -target reservoir volumes ``\hat{x}_t``: - -``` - ┌─────────┐ ┌────────────────┐ ┌──────────────┐ - │ w_t │─────▶│ LSTM encoder │─────▶│ │ - └─────────┘ └────────────────┘ │ Dense │──▶ x̂_t - ┌─────────┐ │ combiner │ - │ x_{t-1} │─────────────────────────────▶│ │ - └─────────┘ └──────────────┘ -``` - -The LSTM carries hidden state across stages, giving the policy memory of -past inflows. The activation is `sigmoid` (bounding outputs to ``[0,1]``, -which is then scaled by the feasibility mapping). - -```julia -models = state_conditioned_policy( - num_uncertainties, num_hydro, num_hydro, [128, 128]; - activation=sigmoid, encoder_type=Flux.LSTM, -) -``` - -## TS-LDR: Linear Decision Rules - -As a baseline, we also train a **linear** policy (TS-LDR). This uses -`dense_multilayer_nn` with identity activation — a composition of linear -layers equivalent to a single affine map: - -```math -\hat{x}_t = W [w_{1:t};\; x_{t-1}] + b. -``` - -TS-LDR uses the same target-setting framework and training pipeline as -TS-DDR. The only difference is the policy class: linear maps have fewer -parameters and cannot capture nonlinear inflow patterns, but they are a -natural baseline from the classical LDR literature. - -```julia -num_inputs = DecisionRules.policy_input_dim(num_uncertainties, num_hydro) -models = dense_multilayer_nn(num_inputs, num_hydro, [64, 64]; activation=identity) -``` - -## Training pipeline 1: Deterministic Equivalent - -The deterministic equivalent (DE) couples all 96 stages into a **single NLP** -for each sampled trajectory. This is the most direct formulation: the policy -generates the full target trajectory ``\hat{x}_{1:T}`` in one forward pass, -and a single coupled solve determines all realized states simultaneously. - -### How it works - -``` - ┌──────────────────────────────────────────────────────────┐ - │ For each sampled trajectory w_{1:T}: │ - │ │ - │ 1. Forward pass: x̂_{1:T} = π_θ(w_{1:T}, x_0) │ - │ │ - │ 2. Solve coupled NLP: │ - │ min Σ_t c_t(x_t, u_t) + C_δ Σ_t ‖δ_t‖ │ - │ s.t. dynamics + AC-OPF for ALL stages simultaneously │ - │ x_t + δ_t = x̂_t(θ) ∀t (target constraint) │ - │ │ - │ 3. Read duals λ_t of target constraints │ - │ Gradient: Σ_t λ_t ⊙ ∇_θ x̂_t(θ) │ - └──────────────────────────────────────────────────────────┘ -``` - -### Mathematical formulation - -```math -\begin{aligned} -Q(w;\, \theta) - \;=\; - \min_{\{x_t, u_t, \delta_t\}_{t=1}^{T}} - \quad & - \sum_{t=1}^{T} c_t(x_t, u_t) - + C_\delta \sum_{t=1}^{T} \|\delta_t\| \\ -\text{s.t.}\quad - & x_t = T_t(w_t,\, u_t,\, x_{t-1}), - && t=1,\ldots,T \\ - & x_t + \delta_t = \hat{x}_t(\theta), - && : \lambda_t,\quad t=1,\ldots,T \\ - & h_t(x_t, u_t) \ge 0, - && t=1,\ldots,T -\end{aligned} -``` - -The gradient is exact by the envelope theorem: - -```math -\nabla_\theta Q -\;=\; -\sum_{t=1}^{T} -\lambda_t \odot \nabla_\theta \hat{x}_t(\theta). -``` - -**Advantages**: strongest gradient signal — full cross-stage coupling -captures how a target at stage 3 affects costs at stage 50. - -**Disadvantage**: the NLP has ``96 \times (\text{AC-OPF variables})`` -decision variables; the policy generates targets without seeing realized -states (open-loop target generation). - -```julia -det_equivalent, uncertainty_samples_det = DecisionRules.deterministic_equivalent!( - det_model, subproblems_de, state_params_in, state_params_out, - Float64.(initial_state), uncertainty_samples, -) - -train_multistage( - models, initial_state, det_equivalent, - state_params_in, state_params_out, uncertainty_samples; - num_batches=4000, optimizer=Flux.Adam(), - penalty_schedule=[(1,100,0.1), (101,210,1.0), (211,300,10.0), (301,4000,30.0)], -) -``` - -## Training pipeline 2: Stage-wise Decomposition (Single Shooting) - -Stage-wise decomposition solves one subproblem per stage sequentially. -Unlike the DE, the policy operates in **closed loop**: after each stage -solve, the realized state ``x_t`` (not the predicted target) is fed back -as input to the next stage. - -### How it works - -``` - ┌─────────────────────────────────────────────────────────────┐ - │ For each sampled trajectory w_{1:T}: │ - │ │ - │ x_0 = initial state │ - │ for t = 1, ..., T: │ - │ x̂_t = π_θ(w_t, x_{t-1}) ← predict target │ - │ solve stage-t subproblem ← project to feasible│ - │ x_t = realized state from solver ← closed-loop │ - │ accumulate c_t + C_δ ‖δ_t‖ │ - │ │ - │ Gradient: chain rule through all stage solves │ - └─────────────────────────────────────────────────────────────┘ -``` - -### Gradient chain - -The gradient must account for how the realized state at stage ``t`` -depends on the targets at all earlier stages. By the chain rule: - -```math -\frac{\partial Q}{\partial \hat{x}_t} -\;=\; -\lambda_t -+ \sum_{k>t} - \frac{\partial q_k}{\partial x_{k-1}} - \cdot \prod_{j=t+1}^{k-1} - \frac{\partial x_j}{\partial x_{j-1}} - \cdot \frac{\partial x_t}{\partial \hat{x}_t}. -``` - -In practice, automatic differentiation (Zygote + ChainRules `rrule`s -defined on each stage solve) handles this chain automatically. -The `rrule` for each stage solve reads the dual ``\lambda_t`` for the -target constraint and uses DiffOpt's implicit differentiation for the -state-transition sensitivities. - -**Advantages**: closed-loop — the policy sees realized states, matching -deployment semantics. Each solve is small (single-stage AC-OPF). - -**Disadvantage**: gradients weaken over long horizons because the -chain rule multiplies many Jacobians; sequential solve prevents -parallelism. - -```julia -train_multistage( - models, initial_state, subproblems, - state_params_in, state_params_out, uncertainty_samples; - num_batches=3000, optimizer=Flux.Adam(), - penalty_schedule=:default_annealed, -) -``` - -## Training pipeline 3: Multiple Shooting - -Multiple shooting partitions the ``T``-stage horizon into ``K`` windows of -``W`` stages each. Within each window, a local deterministic equivalent -couples the stages (strong gradient signal). Between windows, the realized -end-state is passed to the next window (closed-loop continuity). - -### How it works - -``` - ┌────────────────────────────────────────────────────────────────┐ - │ Partition T=96 stages into K=⌈96/12⌉=8 windows of W=12 │ - │ │ - │ x_0 = initial state │ - │ for k = 1, ..., K: │ - │ stages = [(k-1)W+1, ..., kW] │ - │ x̂_{stages} = π_θ(w_{stages}, x_{start_k}) │ - │ solve window-k DE (12-stage coupled NLP) │ - │ x_{end_k} = realized end-state from window solve │ - │ x_{start_{k+1}} = x_{end_k} │ - │ │ - │ Gradient: │ - │ Within window: duals from the coupled solve (like full DE) │ - │ Across windows: DiffOpt chain rule through end-states │ - └────────────────────────────────────────────────────────────────┘ -``` - -### Gradient structure - -Let ``Q_k`` be the cost of window ``k``. The total cost is -``Q = \sum_k Q_k``. Within a window, the gradient is identical to the -DE case (duals of the target constraints in the coupled model). Across -windows, the chain rule threads through the realized end-state: - -```math -\frac{dQ}{d\theta} -\;=\; -\sum_{k=1}^{K} -\left( - \frac{\partial Q_k}{\partial \hat{x}_k} - \cdot \frac{\partial \hat{x}_k}{\partial \theta} - \;+\; - \frac{\partial Q_k}{\partial x_{\text{start}_k}} - \cdot \frac{d x_{\text{start}_k}}{d\theta} -\right), -``` - -where ``\frac{d x_{\text{start}_k}}{d\theta}`` involves the chain -through all prior windows via ``x_{\text{end}_{k-1}}``. - -**Advantages**: balances gradient quality (12-stage coupling) with -tractability (8 small DEs instead of one large one); inter-window -chain provides some closed-loop signal. - -**Disadvantage**: window boundaries introduce gradient discontinuities; -the full-horizon coupling is weaker than the single DE. - -```julia -windows = DecisionRules.setup_shooting_windows( - subproblems, state_params_in, state_params_out, - Float64.(initial_state), uncertainty_samples; - window_size=12, - model_factory=() -> DiffOpt.nonlinear_diff_model(ipopt_attrs), -) - -train_multiple_shooting( - models, initial_state, windows, () -> uncertainty_samples; - num_batches=3000, optimizer=Flux.Adam(), - penalty_schedule=:default_annealed, -) -``` - -## Penalty annealing - -The target penalty ``C_\delta`` controls the trade-off between following -the policy's targets and minimizing operational cost. DecisionRules -supports a **penalty annealing schedule** that ramps the penalty multiplier -during training: - -| Phase | Multiplier | Purpose | -|:------|:----------:|:--------| -| Warmup | ``0.1 \times C_\delta`` | Let the policy explore freely | -| Nominal | ``1.0 \times C_\delta`` | Standard training | -| Tighten | ``10.0 \times C_\delta`` | Sharpen target tracking | -| Lock | ``30.0 \times C_\delta`` | Final precision | - -This is activated with `penalty_schedule=:default_annealed` or by passing -an explicit list of `(start_iter, end_iter, multiplier)` tuples. - -## Evaluation - -After training, we evaluate the policy using stage-wise rollout on held-out -scenarios. Two modes: -- **Target feedback** (`policy_state=:target`): the policy receives its own - predicted target as input, matching DE training semantics. -- **Realized feedback** (`policy_state=:realized`): the policy receives the - realized state from the solver, matching deployment semantics. - -The **target-violation share** measures how much cost comes from the slack -penalty rather than actual operations — it should be small (``\le 5\%``) for -a well-trained policy. - -```julia -rollout_eval = RolloutEvaluation( - subproblems, state_params_in, state_params_out, initial_state, eval_scenarios; - stride=1, policy_state=:realized, -) -rollout_eval(1, models) -println("Operational cost: ", rollout_eval.last_objective_no_deficit) -println("Violation share: ", rollout_eval.last_violation_share) -``` - -## SDDP baseline - -For comparison, we also train an SDDP policy using -[SDDP.jl](https://github.com/odow/SDDP.jl) with **inconsistent -formulations**: a convex SOC-WR relaxation for the backward pass -(cut generation) and the nonconvex ACP formulation for the forward -pass (simulation). This is a pragmatic approach when the true problem -(AC-OPF) is nonconvex — SDDP requires convexity for valid cuts, so a -convex relaxation approximates the value function while the forward pass -evaluates under the true physics. - -The SDDP policy is trained for up to 2000 iterations and the learned -cuts are saved to a JSON file, which can be loaded to simulate the -policy under the ACP formulation. - -## Results - -The plots below compare the TS-DDR and TS-LDR training formulations and -the SDDP baseline on the Bolivia case. Training curves, out-of-sample -cost distributions, reservoir volume trajectories, and thermal generation -profiles are shown. - -### Training convergence (TS-DDR methods) - -![Training convergence](../assets/hydro_training_convergence.png) - -### Out-of-sample cost (TS-DDR methods) - -![Out-of-sample cost comparison](../assets/hydro_cost_comparison.png) - -### Target-violation share (TS-DDR methods) - -![Violation share](../assets/hydro_violation_share.png) - -### Reservoir volume comparison (all methods) - -![Volume comparison](../assets/hydro_volume_comparison.png) - -### Thermal generation comparison (all methods) - -![Generation comparison](../assets/hydro_generation_comparison.png) - -### Summary - -| Method | Policy | Mean Cost | Std | N | -|:-------|:------:|----------:|----:|--:| -| TS-DDR (DE) | LSTM | 325 540 | 6 266 | 100 | -| TS-DDR (DE, anneal) | LSTM | 324 445 | 6 134 | 100 | -| TS-DDR (shooting w=12) | LSTM | 323 289 | 5 593 | 100 | -| TS-DDR (shooting w=12, anneal) | LSTM | 322 812 | 6 081 | 100 | -| TS-DDR (stage-wise, anneal) | LSTM | 321 543 | 6 214 | 100 | -| SDDP (SOC-WR / ACP) | cuts | 303 684 | — | 100 | - -All three TS-DDR methods with penalty annealing converge to similar -costs (321K–325K). SDDP trains on 126 stages (96 + 30 margin). - -!!! note "Preliminary results" - These numbers reflect the current default training scripts. - They will be updated as the package evolves. - diff --git a/docs/src/gpu_acceleration.md b/docs/src/gpu_acceleration.md index 5cbf498..a56764c 100644 --- a/docs/src/gpu_acceleration.md +++ b/docs/src/gpu_acceleration.md @@ -138,21 +138,6 @@ The key requirements are: 3. **Return** a struct with fields `.core`, `.model`, `.horizon`, and `.target_con_range`. -The `HydroPowerModels` example in DecisionRulesExa.jl demonstrates this -pattern for a full AC-OPF problem with reservoir dynamics: - -```julia -# In examples/HydroPowerModels/hydro_power_exa.jl -prob = build_hydro_de( - data; - num_stages = 96, - backend = CUDABackend(), - formulation = :ac_polar, - deficit_cost = 1e5, - target_penalty = :auto, -) -``` - ## Parallel GPU solves When training samples are independent, multiple NLP instances can be @@ -236,18 +221,150 @@ target-deficit penalty, and target-violation share. | Stage-wise decomposition | — | JuMP only | | Multiple shooting | — | JuMP only | -## Full example: HydroPowerModels - -The `examples/HydroPowerModels/` directory in DecisionRulesExa.jl contains -a complete AC-OPF hydrothermal scheduling example for the Bolivia test case -— the same problem solved by DecisionRules.jl in the -[Hydropower Scheduling](@ref) tutorial. It demonstrates: - -- Parsing PowerModels.jl network data and hydro reservoir parameters -- Building a multi-stage deterministic-equivalent NLP in ExaModels - (DC or AC polar OPF formulations) -- L1 + L2 penalty on target slack (δ⁺/δ⁻ splitting for smooth NLP) -- GPU training with parallel MadNLP solves -- Warm-start caching to prevent cascade solver failures -- Penalty and sample-count annealing schedules -- W&B metric logging +## Embedded deterministic equivalent + +The standard `DeterministicEquivalentProblem` treats the policy's target +trajectory as an external parameter: the training loop generates +``\hat{x}_{1:T}`` outside the NLP and passes it in via `set_targets!`. +This is **open-loop** — the policy does not see the realized states +from the coupled solve. + +`EmbeddedDeterministicEquivalentProblem` embeds the policy *inside* +the NLP via a `VectorNonlinearOracle`. The NLP constraint becomes: + +```math +\pi_\theta(w_t,\, x_{t-1}^*) - x_t - \delta_t = 0 \quad \forall t +``` + +where ``x_{t-1}^*`` is the solver's realized state. This is +**closed-loop**: the policy sees realized states from the coupled solve, +and the duals ``\lambda_t`` reflect the joint (policy + physics) system. + +```julia +prob = build_embedded_deterministic_equivalent( + policy; + horizon = T, + nx = nx, + nu = nu, + nw = nw, + dynamics_eq = my_dynamics, + stage_cost = my_cost, + backend = CUDABackend(), +) + +train_tsddr_embedded( + policy, x0, prob, sampler; + num_batches = 500, + num_train_per_batch = 4, + optimizer = Flux.Adam(1f-3), + madnlp_kwargs = (print_level = MadNLP.ERROR, tol = 1e-6), +) +``` + +The oracle closures capture the policy **by reference** — updating Flux +parameters between solves automatically changes the NLP without +rebuilding it. Use `invalidate_policy_cache!` if your oracle caches +policy-dependent intermediates. + +### Strict reachable targets + +When the policy is guaranteed to produce feasible targets (e.g., via a +reachable-set mapping), the slack variables ``\delta_t`` can be removed +entirely. This is strict mode: target constraints are hard equalities, the duals +are pure shadow prices, and there is no target penalty to tune. + +There are two strict deterministic-equivalent paths. + +**Embedded strict DE** evaluates the policy inside the NLP against realized +state decision variables. + +Its constraint is simply ``x_t = \pi_\theta(w_t, x_{t-1}^*)``. Because the +policy receives the realized previous state, a reachable-set map can guarantee +that the next strict equality is dynamically feasible. + +**Regular strict DE** keeps the policy outside the NLP but rolls out targets +from the known initial state: + +```math +\hat{x}_0 = x_0,\qquad +\hat{x}_t = \pi_\theta(w_t, \hat{x}_{t-1}). +``` + +If the policy returns ``\hat{x}_t \in R(\hat{x}_{t-1}, w_t)`` at every stage, +then the entire strict DE target trajectory is feasible by induction. The solve +then enforces ``x_t = \hat{x}_t`` for every stage, so the realized state path is +exactly the reachable target path. + +For battery storage, the charge/discharge and energy bounds give a cheap +one-stage battery-dynamic interval. It is not the complete reachable set of an +AC-OPF: a target can still conflict with generation, branch, voltage, or +reactive-power limits. The +[battery-storage specification](@ref "Stochastic battery-storage AC optimal power flow") +therefore requires true-ACP zero-shedding tests before strict mode becomes the +production default. + +## Sequential rollout evaluation + +`train_tsddr` solves the full deterministic equivalent in one shot. For +deployment diagnostics, DecisionRulesExa.jl provides `RolloutEvaluation`, which +solves a one-stage ExaModels problem sequentially over a materialized scenario: + +```julia +eval = RolloutEvaluation( + stage_problem, + x0, + eval_scenarios; + horizon = T, + n_uncertainty = nw, + set_stage_parameters! = my_setter!, + realized_state = my_state_reader, + policy_state = :realized, +) +``` + +This mirrors deployment semantics: the policy can be evaluated with the +realized previous state (`policy_state = :realized`) or with its previous target +(`policy_state = :target`) to match regular-DE target-generation semantics. + +## Critic control variate + +`train_tsddr` optionally trains a scalar critic ``C(w, \hat{x})`` that +provides a learned control variate for the dual gradient signal. The +critic does not replace the NLP solve — dual multipliers remain the +primary actor gradient. The critic reduces gradient variance by +subtracting a correlated baseline. + +```julia +critic = Chain(Dense(input_dim => 128, tanh), Dense(128 => 128, tanh), Dense(128 => 1)) + +cv = ScalarCriticControlVariate(critic; + featurizer = default_critic_featurizer, + value_loss_weight = 1.0, + gradient_loss_weight = 0.0, +) + +critic_target = RolloutCriticTarget(stage_problem; + horizon = T, + n_uncertainty = nw, + set_stage_parameters! = my_setter!, + realized_state = my_state_reader, + policy_state = :target, +) + +train_tsddr(policy, x0, prob, prob.p_x0, prob.p_target, prob.p_w, sampler; + control_variate = cv, + critic_training_target = critic_target, + actor_gradient_mode = :control_variate, + critic_cv_weight = 1.0, + critic_optimizer = Flux.Adam(1f-3), +) +``` + +Two actor modes are supported: + +- `:control_variate` — subtracts ``\nabla_{\hat{x}} C`` from the dual + signal and adds it back as a differentiable surrogate. Unbiased when + the critic is exact; reduces variance otherwise. +- `:surrogate` — blends dual and critic actor gradients via explicit + weights (`dual_actor_weight`, `critic_actor_weight`). Useful when raw + duals are noisy, but no longer strictly unbiased. diff --git a/docs/src/guide/getting_started.md b/docs/src/guide/getting_started.md new file mode 100644 index 0000000..88af943 --- /dev/null +++ b/docs/src/guide/getting_started.md @@ -0,0 +1,115 @@ +# Getting started + +```@meta +CurrentModule = DecisionRules +``` + +## Installation + +```julia +using Pkg +Pkg.add("DecisionRules") +``` + +DecisionRules.jl builds policies with [Flux.jl](https://fluxml.ai) and +subproblems with [JuMP](https://jump.dev); training requires a +DiffOpt-compatible solver for the stage problems (Ipopt for smooth NLPs, +HiGHS for LPs/MIPs). For GPU-accelerated training of large NLPs, install +the companion package +[DecisionRulesExa.jl](https://github.com/LearningToOptimize/DecisionRulesExa.jl) +(see [GPU Acceleration with DecisionRulesExa.jl](@ref)). + +## Anatomy of a training run + +Every TS-DDR training run assembles the same five ingredients: + +1. **Stage subproblems** — one JuMP model per stage, wrapped with + `DiffOpt.diff_optimizer` so Lagrange duals and sensitivities are + available. The incoming state, the uncertainty, and the policy's + target state enter as *parameters*. +2. **A policy** — a Flux model mapping ``[w_t;\, x_{t-1}]`` to a target + state ``\hat{x}_t`` (see [Target-state policies](@ref)). +3. **An uncertainty sampler** — how trajectories ``w_{1:T}`` are drawn; + the three supported formats (independent pools, joint-scenario pools, + trajectory samplers) are the subject of [Uncertainty Sampling](@ref). +4. **A training formulation** — deterministic equivalent, stage-wise, + multiple shooting, or strict (see + [Three training formulations](@ref) and + [Strict mode: penalty-free gradient signal](@ref)). +5. **An evaluation protocol** — out-of-sample stage-wise rollout via + [`RolloutEvaluation`](@ref), with target- or realized-state feedback + (see [Evaluation semantics](@ref)). + +## Quick start + +```julia +using DecisionRules, JuMP, DiffOpt, Flux, Ipopt + +# Build per-stage subproblems in JuMP (DiffOpt-enabled) +# subproblems, state_params_in, state_params_out, uncertainty_samples, initial_state = ... + +# Define a policy: maps [uncertainty; state] → target state +policy = Chain( + Dense(policy_input_dim(num_uncertainties, num_states), 64, relu), + Dense(64, num_states), +) + +# Train via stage-wise decomposition +train_multistage( + policy, initial_state, subproblems, + state_params_in, state_params_out, uncertainty_samples; + num_batches=100, optimizer=Flux.Adam(1e-3), +) +``` + +## Choosing a training formulation + +| Formulation | Horizon coupling | Gradient source | +|:---|:---|:---| +| **Deterministic Equivalent** | Full horizon, one large NLP | Duals on the coupled problem | +| **Stage-wise (single shooting)** | Sequential rollout | Duals + DiffOpt per stage | +| **Multiple Shooting** | Windowed sub-horizons | DiffOpt per window, continuity penalties | +| **Strict subproblems** | Sequential rollout, no slack | Pure shadow-price duals | + +As a rule of thumb: + +- start with **stage-wise** training — closed-loop, smallest solves, + fewest assumptions; +- move to the **deterministic equivalent** (or its GPU implementation) + when the per-stage solves are large and horizon-coupled gradient signal + pays off; +- use **multiple shooting** as the middle ground on long horizons; +- switch to **strict** mode whenever you can construct a + feasibility-guaranteeing policy — one that bounds its output to the + one-stage reachable set ``R(x, w)`` of the dynamics. Constructing ``R`` + is problem-specific (cheap for resource-balance dynamics with box + bounds). The + [battery-storage AC-OPF specification](@ref "Stochastic battery-storage AC optimal power flow") + shows both the battery-dynamic interval and the additional network-feasibility + gate needed before strict mode is accepted. Strict mode eliminates the + target-slack penalty and its tuning entirely, and the dual ``\lambda_t`` + becomes the pure shadow price. + +## Robustness and hardware + +- Solver or differentiation failures during training are handled by the + pluggable [gradient fallback](@ref "Gradient Fallback") system — + by default a failed iteration logs a warning and is skipped. +- Problems with large stage NLPs train an order of magnitude + faster on GPU through + [DecisionRulesExa.jl](@ref "GPU Acceleration with DecisionRulesExa.jl"), + which implements the strict deterministic equivalent with + ExaModels + MadNLP/cuDSS. + +## Where to go next + +- Theory: + [multistage stochastic optimization](@ref "Multistage stochastic optimization"), + [the TS-DDR framework](@ref "The TS-DDR framework"), + [SDDP and inconsistent formulations](@ref "Stochastic dual dynamic programming"), + [extensions](@ref "Extensions: mixed gradients, critics, and risk"). +- Worked problems: + [stochastic battery-storage AC-OPF](@ref "Stochastic battery-storage AC optimal power flow"), + [rocket control](@ref "Rocket Control"), and + [stochastic lot-sizing](@ref "Stochastic Lot-Sizing with Fixed Ordering Costs"). +- [API Reference](@ref): every exported symbol. diff --git a/docs/src/index.md b/docs/src/index.md index 7051de0..c767864 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -4,27 +4,61 @@ CurrentModule = DecisionRules ``` -DecisionRules.jl trains parametric decision rules through multi-stage optimization, -implementing the **Two-Stage Deep Decision Rules (TS-DDR)** framework from -[arXiv:2405.14973](https://arxiv.org/abs/2405.14973). - -## How it works - -In multi-stage stochastic control, the feasible action at each stage comes from solving -a constrained optimization problem (OPF, MPC, hydrothermal dispatch, …). Rather than -outputting actions directly, the neural-network policy outputs **target states**. -An optimization subproblem then projects these targets onto the feasible set defined by -dynamics and constraints. Lagrange duals and implicit differentiation (via -[DiffOpt.jl](https://github.com/jump-dev/DiffOpt.jl)) provide the gradient signal to -update the policy end-to-end. - -Three training formulations are supported: - -| Formulation | Horizon coupling | Gradient source | -|:---|:---|:---| -| **Deterministic Equivalent** | Full horizon, one large NLP | Duals on the coupled problem | -| **Stage-wise (single shooting)** | Sequential rollout | Duals + DiffOpt per stage | -| **Multiple Shooting** | Windowed sub-horizons | DiffOpt per window, continuity penalties | +DecisionRules.jl trains parametric decision rules — from affine policies +to deep recurrent networks — for multistage stochastic optimization +problems whose actions come from constrained optimization subproblems +(optimal power flow, MPC, inventory control, …). It implements the +**Two-Stage Deep Decision Rules (TS-DDR)** framework of +[arXiv:2405.14973](https://arxiv.org/abs/2405.14973): the policy outputs +**target states**, a projection subproblem restores exact feasibility, and +Lagrange duals (with implicit differentiation via +[DiffOpt.jl](https://github.com/jump-dev/DiffOpt.jl) where needed) provide +the end-to-end training gradient — no differentiation through solver +iterations, no feasibility violations at deployment. + +In the **strict** formulation, the target constraints are hard equalities +and the policy is built to emit only reachable targets: no slack, no +penalty hyperparameter, and the dual ``\lambda_t`` is the pure shadow +price of the target. A GPU companion package, +[DecisionRulesExa.jl](https://github.com/LearningToOptimize/DecisionRulesExa.jl), +trains the same policies through full-horizon deterministic equivalents +with ExaModels + MadNLP/cuDSS. + +## The documentation + +**Theory.** The +[multistage stochastic optimization problem](@ref "Multistage stochastic optimization") +and where decision rules sit among solution methods; +[the TS-DDR framework](@ref "The TS-DDR framework") — target-state +policies, dual gradients, the training formulations, and strict mode with +its reachability-based feasibility guarantee; +[stochastic dual dynamic programming](@ref "Stochastic dual dynamic programming"), +including the inconsistent-formulation variant for nonconvex stage problems and +the bound-versus-forward gap; and +[extensions](@ref "Extensions: mixed gradients, critics, and risk") — +score-function corrections for integer decisions, control-variate +critics, risk-averse objectives. + +**Package guide.** [Getting started](@ref); +[uncertainty sampling formats](@ref "Uncertainty Sampling"); +[gradient fallback](@ref "Gradient Fallback"); +[GPU acceleration](@ref "GPU Acceleration with DecisionRulesExa.jl"); +[API Reference](@ref). + +**Case studies.** The flagship is +[stochastic battery-storage AC optimal power flow](@ref "Stochastic battery-storage AC optimal power flow"), which defines the +PGLib case generator, demand information pattern, battery physics, strict and +soft target projections, true ACP model, SOC-WR backward relaxation, and paired +PF/SDDP/TS-DDR evaluation protocol. [Long-term hydrothermal planning](@ref) asks whether a learned policy can +value water as well as a method built to do exactly that: on a real grid under +full AC physics, a policy trained from random initialisation in eleven GPU-hours, +with no value function and no convex relaxation anywhere in its path, operates +the system within **0.152%** of a converged SDDP baseline over 500 shared inflow +scenarios — close, measurably more expensive, and diagnosably so. Two further +studies, +[rocket control](@ref "Rocket Control") and +[stochastic lot-sizing](@ref "Stochastic Lot-Sizing with Fixed Ordering Costs"), +exercise continuous control and mixed-integer recourse. ## Installation @@ -33,32 +67,8 @@ using Pkg Pkg.add("DecisionRules") ``` -## Quick start - -```julia -using DecisionRules, JuMP, DiffOpt, Flux, Ipopt - -# Build per-stage subproblems in JuMP (DiffOpt-enabled) -# subproblems, state_params_in, state_params_out, uncertainty_samples, initial_state = ... - -# Define a policy: maps [uncertainty; state] → target state -policy = Chain( - Dense(policy_input_dim(num_uncertainties, num_states), 64, relu), - Dense(64, num_states), -) - -# Train via stage-wise decomposition -train_multistage( - policy, initial_state, subproblems, - state_params_in, state_params_out, uncertainty_samples; - num_batches=100, optimizer=Flux.Adam(1e-3), -) -``` - -See the [Algorithm](@ref) page for the mathematical formulation, the -[Uncertainty Sampling](@ref) guide for how to prepare your scenario data, the -[GPU Acceleration with DecisionRulesExa.jl](@ref) page for GPU-accelerated training, -and the examples for complete worked problems. +[Getting started](@ref) covers solver requirements, a quick-start example, +and how to choose among the training formulations. ## Citation diff --git a/docs/src/sampling.md b/docs/src/sampling.md index bacdfad..3e07a41 100644 --- a/docs/src/sampling.md +++ b/docs/src/sampling.md @@ -251,35 +251,36 @@ For callables, `sample(f::Function)` simply calls `f()`. ## Demonstrating the difference -Consider 3 hydro reservoirs with 4 historical inflow scenarios: +Consider 3 demand regions with 4 historical load-factor scenarios: ``` -Historical inflow data (columns = scenarios): +Historical load factors (columns = scenarios): ω=1 ω=2 ω=3 ω=4 -Res 1: 10 20 15 25 -Res 2: 80 120 90 110 -Res 3: 5 8 6 9 +Reg 1: 0.90 1.00 0.95 1.10 +Reg 2: 0.85 1.15 0.90 1.05 +Reg 3: 0.92 1.08 0.97 1.12 ``` **Independent sampling** draws one value per row independently. A sample -might be `(10, 120, 9)` — reservoir 1 from ω=1, reservoir 2 from ω=2, -reservoir 3 from ω=4. This combination never occurred historically and -may violate the drought-affects-all-basins correlation. +might be `(0.90, 1.15, 1.12)` — region 1 from ω=1, region 2 from ω=2, +region 3 from ω=4. This combination never occurred historically and +may violate spatial demand correlation. -**Joint sampling** picks one column: `(10, 80, 5)` or `(25, 110, 9)` — +**Joint sampling** picks one column: `(0.90, 0.85, 0.92)` or +`(1.10, 1.05, 1.12)` — always a historically observed combination. **Trajectory sampling** can additionally model temporal persistence: -if ω=1 (dry year) was drawn at stage 1, the AR(1) sampler will likely -produce below-average inflows at stage 2 as well. +if a low-demand atom was drawn at stage 1, the AR(1) sampler will likely +produce below-average demand at stage 2 as well. ``` Joint sampling (k=4 possible outcomes per stage): - Res 1 ──┐ - Res 2 ──┼── same ω ──→ one of 4 historical vectors - Res 3 ──┘ + Reg 1 ──┐ + Reg 2 ──┼── same ω ──→ one of 4 historical vectors + Reg 3 ──┘ Independent sampling (k³=64 possible outcomes per stage): @@ -307,7 +308,7 @@ maintainability. parameters from an uncertainty pool, discarding the scenario values. Used by `setup_shooting_windows` for multiple-shooting training. -## API Reference +## Docstrings ```@docs sample diff --git a/docs/src/theory/extensions.md b/docs/src/theory/extensions.md new file mode 100644 index 0000000..b33bb2c --- /dev/null +++ b/docs/src/theory/extensions.md @@ -0,0 +1,165 @@ +# Extensions: mixed gradients, critics, and risk + +```@meta +CurrentModule = DecisionRules +``` + +The dual gradient of [The TS-DDR framework](@ref) is exact for smooth +subproblems and, over fresh samples, an unbiased estimator of the +expected-cost gradient. Two practical situations call for more: +**discrete decisions**, where the dual is blind to integer switches and a +score-function (REINFORCE) correction restores the missing signal, and +**small sample budgets**, where a control-variate critic cuts the +estimator's variance without moving its optimum. Both extensions, and a +risk-averse change-of-measure variant of the gradient, ship with the +package. + +!!! note "Scope" + The battery-storage case study is continuous and is designed to use the + pure strict dual gradient after its feasibility gates; it uses none of these + extensions. The score-function correction is exercised in + [Stochastic Lot-Sizing with Fixed Ordering Costs](@ref), whose + fixed-charge (binary) ordering decisions are exactly the situation it + addresses. + +## Mixed gradient: score-function (REINFORCE) correction + +For problems with integer variables or non-smooth subproblems, the dual +gradient can be biased — it is local to a fixed integer assignment and cannot +see the effect of discrete switches (e.g., opening a setup variable). + +DecisionRules provides a **score-function (REINFORCE)** correction that mixes +the dual gradient with a model-free policy gradient estimated from stage-wise +rollouts under perturbed targets. + +### How the score-function estimator works + +1. **Perturb**: add Gaussian noise to the policy targets: + ``\tilde{x}_t = \hat{x}_t(\theta) + \delta_t``, where + ``\delta_t \sim \mathcal{N}(0, \sigma^2 I)``. + +2. **Rollout**: solve the stage-wise subproblems with the perturbed targets to + obtain realized costs ``R_m`` for ``m = 1, \ldots, M`` rollouts. These + rollouts solve the models exactly as built (MIPs stay MIPs), so the costs + reflect true integer-feasible decisions. + +3. **Advantage**: center the costs ``A_m = R_m - \bar{R}``. Because the mean + baseline ``\bar{R}`` is computed from the same ``M`` rollouts, mean-centering + reduces variance but introduces a small ``O(1/M)`` bias (effectively scaling + the estimator by ``(M-1)/M``) that vanishes as `num_rollouts` grows; a + leave-one-out baseline would be exactly unbiased. + +4. **Surrogate loss**: the differentiable scalar whose gradient recovers the + REINFORCE estimate: + +```math +L_{\text{sf}}(\theta) +\;=\; +\frac{1}{M} \sum_{m=1}^{M} + A_m + \sum_{t=1}^{T} + \left\langle + \frac{\delta_{m,t}}{\sigma^2},\; + \hat{x}_{t+1}(\theta) + \right\rangle. +``` + +This is the standard score-function estimator for Gaussian perturbations. +The key identity is +``\nabla_\theta \log p(\delta_t \mid \theta) = \delta_t / \sigma^2`` +for a Gaussian centered at ``\hat{x}_t(\theta)``. + +### Mixed gradient + +The final training gradient combines both signals: + +```math +\nabla L +\;=\; +\alpha\, \nabla L_{\text{dual}} ++ (1 - \alpha)\, \nabla L_{\text{sf}}, +``` + +where ``\alpha \in [0, 1]`` is the `dual_weight`. + +There are two separate solve paths in the mixed-gradient training loop: + +- **Dual path**: controlled by `integer_strategy`, which determines how local + dual information is read from the deterministic equivalent + (e.g., [`FixedDiscreteIntegerStrategy`](@ref) solves the MIP, fixes integers, + re-solves the LP, and reads LP duals). +- **Score-function path**: controlled by [`ScoreFunctionConfig`](@ref), which + owns separate rollout subproblems. These are solved exactly as built, and + their realized costs define the Monte Carlo score-function term. + +### Scheduled ramp-in + +A [`ScoreFunctionSchedule`](@ref) can ramp ``\alpha`` from 1 (pure dual) to +its final value over a warmup period. Let ``k`` be the current iteration and +``\rho_k = \operatorname{clip}((k - k_0) / r,\, 0,\, 1)``. The effective +score-function weight is ``\rho_k (1 - \alpha)``. + +This lets the DE dual gradient establish a good initial policy before +introducing the higher-variance REINFORCE signal. + +See the [Stochastic Lot-Sizing with Fixed Ordering Costs](@ref) example for a +complete worked example with integer variables and mixed gradients. + +## Variance reduction: control-variate critic + +The dual gradient over a batch of ``N`` sampled trajectories is the +sample-average + +```math +g \;=\; \frac{1}{N}\sum_{s=1}^{N}\sum_{t=1}^{T} + \bigl\langle \lambda^{s}_t,\; \partial \hat{x}^{s}_t/\partial\theta \bigr\rangle , +\qquad \lambda^{s}_t = \partial Q_s/\partial \hat{x}^{s}_t . +``` + +With fresh independent samples each step this is an **unbiased** estimator of +``\nabla_\theta\,\mathbb{E}[Q]`` for any ``N``. A small batch does not bias it — +it only inflates its **variance**, which sets the SGD noise floor and keeps the +policy short of the optimum. Rather than paying for a large ``N``, a +`ScalarCriticControlVariate` subtracts a learned, state-conditioned +baseline ``b^{s}_t = \nabla_{\hat{x}_t} C \approx \lambda^{s}_t`` and adds it back +as an independent-sample expectation: + +```math +g_{\text{cv}} \;=\; + \frac{1}{N}\sum_{s}\sum_t \bigl\langle \lambda^{s}_t - b^{s}_t,\; \partial\hat{x}^{s}_t/\partial\theta\bigr\rangle + \;+\; + \frac{1}{M}\sum_{j}\sum_t \bigl\langle b^{j}_t,\; \partial\hat{x}^{j}_t/\partial\theta\bigr\rangle . +``` + +**Unbiasedness.** The subtracted and added terms are two Monte-Carlo estimates of +the same expectation ``\mathbb{E}\bigl[\sum_t\langle b_t,\partial\hat{x}_t/\partial\theta\rangle\bigr]``, +so ``\mathbb{E}[g_{\text{cv}}] = \mathbb{E}[g] = \nabla_\theta\mathbb{E}[Q]``: the +critic **cannot move the optimum**. (If the add-back reuses the same samples with +``M=N``, the two terms cancel and the critic is a no-op — a fresh add-back batch, +`num_cheap_critic_samples_per_batch > 0`, is what activates it.) + +**Variance.** The reduction is governed by how well the baseline tracks the dual, + +```math +\text{Var reduction} \;\approx\; \frac{1}{1 - R^2}, \qquad +R^2 = \text{explained variance of } \lambda_t \text{ by } b_t , +``` + +so the baseline must be trained to **match the dual** (`gradient_loss_weight > 0`), +not only the scalar value (`value_loss_weight`): a value-only critic leaves +``\nabla_{\hat{x}} C`` unconstrained, ``R^2\approx 0``, and yields no reduction. The +control-variate critic is therefore a **sample-efficiency lever** — it reaches the +same risk-neutral optimum a much larger batch would, at a modest batch size. + +## Risk-averse extension (nested change-of-measure) + +The same per-stage critic supports a **risk-averse** objective by replacing the +expectation with a nested, time-consistent coherent risk measure (e.g. conditional +``\mathrm{CVaR}_\alpha``). Each stage weight becomes ``\Xi^{s}_t\,\lambda^{s}_t`` with +``\Xi^{s}_t = \prod_{u\le t}\zeta^{s}_u``, the causal product of per-node +change-of-measure densities ``\zeta_u`` (``\zeta\equiv 1`` recovers the risk-neutral +gradient above). Because each ``\zeta_u`` depends only on the distribution +*conditional* on stage ``u``, the policy never hedges against a tail already +precluded by realized uncertainty — the time-consistency property that a global +scenario re-weighting would violate. This targets tail cost at the expense of the +mean, and is a distinct objective from the risk-neutral formulation above. diff --git a/docs/src/theory/multistage.md b/docs/src/theory/multistage.md new file mode 100644 index 0000000..f5e271d --- /dev/null +++ b/docs/src/theory/multistage.md @@ -0,0 +1,196 @@ +# Multistage stochastic optimization + +```@meta +CurrentModule = DecisionRules +``` + +Everything downstream — the TS-DDR framework, the SDDP baseline, the case +studies — is an attempt to solve one problem: sequential decision making +under uncertainty, with actions constrained by an optimization-level +feasible set. What follows fixes +notation, recalls the dynamic-programming view and why it is intractable +in general, and places **decision rules** among the classical solution +families; readers fluent in multistage stochastic programming can skim to +[Decision rules](@ref decision-rules-sec) and continue with +[The TS-DDR framework](@ref). + +## Problem statement + +Consider a sequential decision problem over a finite horizon of ``T`` +stages. At each stage ``t = 1, \ldots, T``: + +1. an exogenous **uncertainty realization** ``w_t \in \mathcal{W}_t`` is + revealed (river inflows, demands, prices, disturbances); +2. the decision maker, knowing the current **state** ``x_{t-1}`` and the + realization ``w_t`` (and, in general, the whole history + ``w_{1:t} = (w_1, \ldots, w_t)``), chooses a **control** + ``u_t`` and a next state ``x_t``; +3. the pair must satisfy the stage feasibility constraints, + ``(u_t, x_t) \in \mathcal{X}_t(x_{t-1}, w_t)``, which encode both the + **dynamics** (how the state evolves) and the **static constraints** of + the stage (a network, a budget, a capacity — whatever the application + imposes within a single period); +4. a **stage cost** ``c_t(x_t, u_t)`` is incurred. + +The objective is to choose a *policy* — a rule for making each decision +from the information available when it must be made — minimizing expected +total cost: + +```math +\min_{\pi \in \Pi} \; +\mathbb{E}_{w_{1:T}} \left[ \sum_{t=1}^{T} c_t\bigl(x_t^\pi, u_t^\pi\bigr) \right] +\qquad \text{s.t.} \quad +\bigl(u_t^\pi, x_t^\pi\bigr) \in \mathcal{X}_t\bigl(x_{t-1}^\pi, w_t\bigr) +\;\; \forall t, +``` + +where ``\Pi`` is the set of **nonanticipative** policies: ``u_t^\pi`` may +depend on ``w_{1:t}`` but not on future realizations ``w_{t+1:T}``. +Nonanticipativity is what makes the problem *stochastic control* rather +than a family of deterministic problems — every decision is a hedge +against a distribution of futures, committed before those futures are +revealed. + +Two structural features of this formulation deserve emphasis, because the +solution methods differ precisely in how they treat them: + +- **Intertemporal coupling through the state.** The only channel through + which stage ``t`` affects stage ``t+1`` is ``x_t``. A resource stored in + the state (energy in a battery, inventory on a shelf, fuel in a tank) + has an *opportunity cost* — the expected future cost avoided by carrying + it forward — that no single-stage view can price. +- **Constrained actions.** The feasible set ``\mathcal{X}_t`` is itself an + optimization-level object — possibly a full nonconvex program, as in + the battery-storage AC-OPF case study. Any learned policy must produce + decisions that *satisfy it exactly*, not approximately. + +## The dynamic-programming recursion + +Under the Markovian assumption that ``(x_{t-1}, w_t)`` summarizes the +history (stagewise-independent ``w_t``, or an augmented state otherwise), +the problem admits Bellman's recursion. Define the **cost-to-go** +(or *value*) **function** at the end of stage ``t``: + +```math +V_t(x_{t-1}, w_t) \;=\; +\min_{(u_t, x_t) \in \mathcal{X}_t(x_{t-1}, w_t)} +\; c_t(x_t, u_t) + \mathbb{E}_{w_{t+1}}\bigl[ V_{t+1}(x_t, w_{t+1}) \bigr], +``` + +with ``V_{T+1} \equiv 0``. The optimal policy acts greedily against the +expected cost-to-go: at each stage it trades the immediate cost +``c_t`` against the expected future cost +``\mathbb{E}[V_{t+1}(x_t, \cdot)]`` of the state it leaves behind. In +resource-storage problems this expected cost-to-go *is* the "value of +water" (or of inventory): its negative gradient with respect to the stored +quantity is the marginal price at which storing beats releasing. + +The recursion is conceptually complete and computationally hopeless in +general: ``V_t`` is a function on the full state space, and any grid-based +representation grows exponentially with the state dimension — Bellman's +*curse of dimensionality*. Every practical method is a way of +approximating either the value function or the policy. + +## Solution families + +Three broad families dominate practice; the third is the one this package +implements. + +### Scenario trees and the deterministic equivalent + +Discretize the uncertainty into a finite **scenario tree** and attach one +copy of the decision variables to every node. The result is a single — +typically enormous — mathematical program, the **deterministic +equivalent** (DE), whose solution is exact *for the tree*. The tree grows +exponentially in ``T``, so pure scenario-tree methods are confined to +short horizons or coarse discretizations. The DE returns in +[Three training formulations](@ref) in a different role — not as a +solution method but as a *differentiable training oracle* for a policy, +evaluated one sampled trajectory at a time, which sidesteps the +exponential growth entirely. + +### Value-function approximation: SDDP + +**Stochastic dual dynamic programming** (SDDP) exploits convexity: when +each stage problem is convex in ``x_{t-1}``, the cost-to-go +``\mathbb{E}[V_{t+1}]`` is convex and can be outer-approximated by +supporting hyperplanes ("cuts") generated from stage duals. SDDP is the +workhorse of long-horizon planning under uncertainty and the baseline the +case studies compare against; +[Stochastic dual dynamic programming](@ref) develops it in detail — +including what must be done, and what is silently given up, when the true +stage problem is *nonconvex*. + +### [Decision rules](@id decision-rules-sec) + +The third family approximates the **policy** directly: restrict ``\Pi`` to +a parametric class + +```math +u_t = \pi_\theta\bigl(w_{1:t}, x_{t-1}\bigr), \qquad \theta \in \Theta, +``` + +and optimize over the finite-dimensional parameter ``\theta`` instead of +over the space of all measurable policies. Nonanticipativity holds *by +construction* — the rule only ever reads the history. The classical +instance is the **linear decision rule** (LDR/affine policy), where +``\pi_\theta`` is affine in the observations: tractable, sometimes +provably near-optimal, but limited in expressiveness. Replacing the affine +map with a deep network gives a **deep decision rule** with the opposite +profile: expressive, but raising two difficulties that the naive +"learn a network that outputs actions" approach does not survive in +constrained physical systems: + +1. **Feasibility.** A network output has no reason to satisfy + ``\mathcal{X}_t`` — and in operations, constraint violation is not a + soft error. Penalizing violations reintroduces exactly the kind of + hyperparameter tuning decision rules were supposed to avoid. +2. **Gradient signal.** If feasibility is enforced by an optimization + layer, training requires differentiating through a solver — expensive + and fragile if done by unrolling or generic implicit differentiation at + scale. + +[The TS-DDR framework](@ref) resolves both at +once: the network outputs *target states* rather than actions, a +projection subproblem restores feasibility exactly, and Lagrangian duality +supplies the training gradient at the cost of the solve itself. The +[strict variant](@ref "Strict mode: penalty-free gradient signal") +sharpens this further when targets can be guaranteed reachable by +construction. + +## What "solving" means: bounds and simulation + +Because all practical methods approximate, empirical comparisons rest on +two complementary quantities, used throughout the case studies: + +- A **lower bound** (for minimization): SDDP's cut model provides a valid + lower bound on the expected cost *of the problem its cuts actually + model*. When the cut model is a convex relaxation of a nonconvex stage + problem, + the bound is a bound on the *relaxed* problem — an important subtlety + developed in [The bound and the forward cost](@ref). +- A **simulation (forward) cost**: the expected cost of a concrete policy, + estimated by rolling it out on the *true* stage problems over sampled + scenarios. This is the only number that treats every method — cuts, + linear rules, deep rules — on identical footing, and it is the primary + metric of the case studies (see the + paired evaluation protocol in the + [battery-storage AC-OPF study](@ref "Stochastic battery-storage AC optimal power flow")). + +The gap between the two jointly measures the suboptimality of the policy +*and* the fidelity of the model used to bound it — and keeping those two +contributions separate is a recurring theme, made precise for SDDP in +[The bound and the forward cost](@ref). + +## Further reading + +- Shapiro, Dentcheva, Ruszczyński, *Lectures on Stochastic Programming* + (SIAM) — the standard reference for the general theory. +- Bertsekas, *Dynamic Programming and Optimal Control* — the + control-theoretic view of the same recursion. +- Ben-Tal et al., *Adjustable robust solutions of uncertain linear + programs* (2004) — the origin of affine decision rules. +- Rosemberg, Street, Valladão, Van Hentenryck, + [*Efficiently Training Deep-Learning Parametric Policies using + Lagrangian Duality*](https://arxiv.org/abs/2405.14973) — the TS-DDR + paper this package implements. diff --git a/docs/src/theory/sddp.md b/docs/src/theory/sddp.md new file mode 100644 index 0000000..069dd17 --- /dev/null +++ b/docs/src/theory/sddp.md @@ -0,0 +1,182 @@ +# Stochastic dual dynamic programming + +```@meta +CurrentModule = DecisionRules +``` + +Stochastic dual dynamic programming (SDDP) is the industrial standard for +long-horizon planning under uncertainty and the baseline against which the +case studies are evaluated — a fair comparison requires both methods at +the same depth. Its guarantees rest on one structural assumption, +convexity of the stage problem in the state; making that assumption +precise leads directly to the **inconsistent-formulation** variant used +when the true stage problem is nonconvex, and to the +*bound-versus-forward gap*, the quantity that measures what the +convexification gives up. (DecisionRules.jl does not implement SDDP; the +baselines use [SDDP.jl](https://github.com/odow/SDDP.jl).) + +## Cutting-plane approximation of the cost-to-go + +Recall the dynamic-programming recursion of +[Multistage stochastic optimization](@ref): the optimal stage decision +trades immediate cost against the expected cost-to-go +``\mathcal{V}_{t+1}(x_t) := \mathbb{E}_{w_{t+1}}[V_{t+1}(x_t, w_{t+1})]``. +SDDP (Pereira & Pinto, 1991) replaces ``\mathcal{V}_{t+1}`` by a +polyhedral outer approximation built from **cuts**, + +```math +\mathcal{V}_{t+1}(x) \;\ge\; \underline{\mathcal{V}}_{t+1}(x) +\;=\; \max_{k = 1, \ldots, K} \;\alpha_k + \langle \beta_k,\, x \rangle , +``` + +and iterates two passes over a sampled scenario lattice: + +- **Forward pass.** Sample a trajectory ``w_{1:T}``; solve the stage + problems in sequence with ``\underline{\mathcal{V}}_{t+1}`` in place of + the true cost-to-go, recording the visited states ``x_t``. The + accumulated stage costs of many forward passes estimate the expected + cost of the *current cut policy* — an upper-bound estimator (in + expectation) for minimization. +- **Backward pass.** At each visited state ``x_t``, re-solve the + stage-``(t{+}1)`` problems for every uncertainty realization, and read + the **dual multipliers** of the constraints through which ``x_t`` + enters (the state-coupling rows). Averaging over realizations + yields a subgradient ``\beta`` of ``\underline{\mathcal{V}}_{t+1}`` at + ``x_t`` and an intercept ``\alpha`` — a new cut, appended to the model. + +The value of the first-stage problem under the current cuts is a valid +**lower bound** on the optimal expected cost, monotonically nondecreasing +as cuts accumulate. Under standard assumptions (finite support, +stagewise independence, relatively complete recourse), the bound and the +forward-cost estimate converge to the common optimal value. + +## Where convexity enters + +Every step above leans on convexity of the stage problem in the incoming +state ``x_{t-1}``: + +1. **Cut validity.** A cut is a supporting hyperplane; it under-estimates + ``\mathcal{V}_{t+1}`` everywhere only if ``\mathcal{V}_{t+1}`` is + convex. Convexity of ``V_{t+1}(\cdot, w)`` in the state follows from + convexity of the stage feasible set and cost — and is *inherited + backwards* through the recursion. +2. **Dual attainment.** The subgradient ``\beta`` is a Lagrange + multiplier; strong duality (no duality gap) is what makes the + multiplier a subgradient of the value function rather than merely a + local sensitivity. + +If the stage problem is **nonconvex** in the state, both properties +fail: duals of a nonconvex solve are local objects, and a "cut" built +from them can *cut off* the true value function. SDDP as stated simply +does not apply. + +## Inconsistent formulations: convex cuts, nonconvex stage problems + +The pragmatic and widely used response is to run the two passes on +**different formulations** of the same stage: + +- the **backward pass** (cut generation) uses a **convex relaxation** + ``\mathcal{X}_t^{\mathrm{rel}} \supseteq \mathcal{X}_t`` of the stage + feasible set; +- the **forward pass** (state sampling and policy simulation) uses the + **true nonconvex stage problem** ``\mathcal{X}_t``. + +We refer to this as SDDP with **inconsistent formulations**. It is +well defined: the relaxed stage problem is convex in the state, so the +cuts are valid *for the relaxed problem*, and the recursion converges on +that surrogate. The forward pass then evaluates the resulting +value-function approximation against the stage problem that will +actually be operated. Concretely, the operating policy is + +```math +u_t^{\mathrm{SDDP}}(x_{t-1}, w_t) \;\in\; +\arg\min_{(u_t, x_t) \in \mathcal{X}_t(x_{t-1}, w_t)} +\; c_t(x_t, u_t) + \underline{\mathcal{V}}_{t+1}^{\mathrm{rel}}(x_t) : +``` + +true feasibility inside the stage, *relaxation-priced* future outside +it. + +### What the surrogate misprices + +The quality of this policy hinges on how well the relaxed cost-to-go +``\underline{\mathcal{V}}^{\mathrm{rel}}`` prices the *true* marginal +value of the state. Relaxation only widens the stage feasible set, so +the surrogate can realize transitions the true system cannot — it +systematically **underestimates the cost of future operation** wherever +the relaxation is loose, and therefore undervalues precisely the states +whose worth derives from relieving that future stress. Whether the +resulting error is negligible or material is a property of the +*instance and its operating regime*, not of the algorithm. The +[battery-storage AC-OPF study](@ref "Stochastic battery-storage AC optimal power flow") +tests the concrete mechanism of a conic network relaxation mispricing the +locational value of stored energy. + +## The bound and the forward cost + +The inconsistent scheme produces two headline numbers with different +epistemic status: + +- ``\underline{z}^{\mathrm{rel}}`` — the converged **backward bound**: a + valid lower bound on the expected cost of the *relaxed* multistage + problem. Because relaxation only widens each stage's feasible set, it + is also a valid lower bound on the true problem — but a *slack* one: + it is attained (if at all) by relaxed trajectories that no feasible + policy can reproduce. +- ``\hat{z}`` — the **forward simulation cost**: the Monte Carlo + estimate of the expected cost of the actual operating policy on the + true stage problems. + +Their relative difference, + +```math +\mathrm{gap} \;=\; +\frac{\hat{z} - \underline{z}^{\mathrm{rel}}} + {\underline{z}^{\mathrm{rel}}}, +``` + +is the **bound-versus-forward gap**. It conflates ordinary SDDP +suboptimality, relaxation error, and finite-cut error. It is a diagnostic, +not recoverable policy headroom: the relaxed bound can be attained only by +network states that ACP cannot realize. + +The relevant empirical room compares the SDDP ACP forward cost with a +paired perfect-foresight ACP solve over the same full horizon. Even that +quantity is only an information-relaxation upper bound on possible +nonanticipative improvement. The +[battery-storage protocol](@ref "Stochastic battery-storage AC optimal power flow") +defines both quantities and keeps them separate. + +Two disciplines keep the comparison honest, and both are enforced in the +case studies: + +1. **Bounds are horizon-specific.** A bound computed on a + ``T``-stage problem does not bound a ``T' < T``-stage simulation + metric; training and evaluation horizons must be stated and matched. +2. **Policies are compared on the forward metric only.** The only number + comparable across SDDP, TS-DDR, and any other method is the simulated + expected cost under identical stage problems and identical scenarios — + hence the paired-scenario protocol of the case studies. + +## Complementarity with decision rules + +SDDP and TS-DDR occupy dual corners of the design space. SDDP +approximates the *value function* and recovers actions by re-solving a +stage problem at operation time; its strength is a self-certifying bound +and decades of industrial hardening, and its structural commitment is +convexity of the stage model that generates cuts. TS-DDR approximates the +*policy* and needs no convexity — the projection subproblem may be an +arbitrary NLP — but it certifies nothing by itself: its quality is +established empirically, by simulation against a baseline. This is why +the case studies always report both: SDDP supplies the yardstick (a bound +and a strong incumbent policy), and the decision rule is measured against +it on the true stage problems. + +## Further reading + +- Pereira & Pinto, *Multi-stage stochastic optimization applied to energy + planning*, Mathematical Programming 52 (1991) — the original SDDP paper. +- Dowson & Kapelevich, *SDDP.jl: a Julia package for stochastic dual + dynamic programming*, INFORMS Journal on Computing 33 (2021). +- Shapiro, *Analysis of stochastic dual dynamic programming method*, + EJOR 209 (2011) — convergence analysis and statistical stopping. diff --git a/examples/BatteryStorageOPF/.gitignore b/examples/BatteryStorageOPF/.gitignore new file mode 100644 index 0000000..4285044 --- /dev/null +++ b/examples/BatteryStorageOPF/.gitignore @@ -0,0 +1,29 @@ +# Cases are CONSTRUCTED, never committed. Every artifact under case/ is a pure +# function of the builder in `build_battery_case.jl` plus its recorded seeds, so +# committing it would ship a second, drifting source of truth for the case. +# The manifest hashes belong in the phase record, not in git. +case/ + +# SDDP.jl writes these into the working directory when a node fails: the whole +# subproblem in MathOptFormat and the cuts it carried. They are debugging +# evidence for one run — megabytes each — and are regenerated by rerunning it. +subproblem_*.mof.json +model_infeasible_*.cuts.json +SDDP.log + +# Figures written by the `plot_*` helpers in `battery_analysis.jl`. Their paths +# are chosen by the caller — the README's `value_curves.png` and `energy.png` +# are only examples — so the name cannot be enumerated. Scoped to this example +# directory, which tracks no image of its own. +*.png + +# Output of the README's own commands, run from this directory: the protocol +# descriptor `write_protocol_descriptor(…, "screening.toml")` writes, and the +# segment tree `portfolio_runner.jl --output run/seg001` writes. Both are +# regenerated per run. +/screening.toml +/run/ + +# Machine-local package preferences, written by Preferences.jl on the user's +# machine. Project.toml is the environment contract. +LocalPreferences.toml diff --git a/examples/BatteryStorageOPF/Project.toml b/examples/BatteryStorageOPF/Project.toml new file mode 100644 index 0000000..b0d11a1 --- /dev/null +++ b/examples/BatteryStorageOPF/Project.toml @@ -0,0 +1,40 @@ +[deps] +CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +Clarabel = "61c947e1-3e6d-4ee4-985a-eec8c727bd6e" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" +HiGHS = "87dc4568-4c63-4d18-b0c0-bb2238e4078b" +Ipopt = "b6b21f68-93f8-5de0-b562-5493be1d77c9" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +JuMP = "4076af6c-e467-56ae-b986-b466b2749572" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee" +PGLib = "07a8691f-3d11-4330-951b-3c50f98338be" +Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" +PowerModels = "c36e90e8-916a-50a6-bd94-075b64ef4655" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +SCS = "c946c3f1-0d1f-5ce8-9dea-7daa1f7e2d13" +SDDP = "f4570300-c277-11e8-125c-4912f86ce65d" +SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" +StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[compat] +CSV = "0.10" +Clarabel = "0.11" +DataFrames = "1" +Distributions = "0.25" +Ipopt = "1" +JSON = "0.21, 1" +JuMP = "1" +MathOptInterface = "1" +PGLib = "0.2" +Plots = "1" +PowerModels = "0.21" +SDDP = "1.14" +StableRNGs = "1" +julia = "1.11, 1.12" diff --git a/examples/BatteryStorageOPF/README.md b/examples/BatteryStorageOPF/README.md new file mode 100644 index 0000000..6c2f681 --- /dev/null +++ b/examples/BatteryStorageOPF/README.md @@ -0,0 +1,789 @@ +# Battery-storage AC-OPF — JuMP / PowerModels / SDDP engine + +This example is the CPU half of the multistage battery-storage study. It does +two things: + +1. it is a **reusable toolkit** for building and diagnosing battery cases on top + of any PGLib benchmark — acquire a case, place batteries, author a demand + process, freeze it into finite support, probe the physics and the value of + stored energy, solve perfect-foresight equivalents, and train the SDDP + baseline; +2. it is the **construction of this study's own benchmark**, built with exactly + those tools and nothing else. + +The scientific narrative — what the experiment asks and what it found — lives in +the documentation. This file says how to run things and what each file is for. + +The GPU half (the ExaModels true-ACP model, the strict reachable policy and the +TS-DDR trainer) lives in `DecisionRulesExa.jl/examples/BatteryStorageOPF`. The +two packages are independent: neither loads the other. They share the frozen +case bytes and two source files (`battery_case.jl`, `battery_solution_schema.jl`) +as copies whose byte identity is asserted whenever the case is rebuilt. + +Everything below runs from this directory with `julia --project=.`, needs no +cluster scheduler and writes no telemetry. + +## The boundary between the packages and this example + +| owned by | what | +|---|---| +| PGLib.jl | the benchmark case and its parser | +| PowerModels.jl | buses, generators, branches, voltage variables, reference angle, Ohm's law at both ends with taps and phase shifts, shunts, angle-difference limits, apparent-power limits at both ends, nodal active and reactive balances, generator bounds and costs, and the `ACPPowerModel` / `SOCWRConicPowerModel` / `DCPPowerModel` formulations themselves | +| SDDP.jl | policy graph, state variables, sampling, cut generation, training, simulation | +| Distributions.jl | the laws an authoring demand sampler is built from | +| this example | batteries: energy state, charge/discharge, state transition, unity-power-factor injection, throughput cost, the strict outgoing-energy target equality, the two-sided nodal active-power recourse, and the demand parameterization | + +No AC trigonometry, no SOC-WR lifted branch equation and no DC susceptance-times- +angle-difference flow is written here. The regression suite scans the +model-building files and the diagnostics and fails if one appears. + +## Files + +| file | role | +|---|---| +| `battery_case.jl` | the frozen case contract: canonical JSON, artifact hashes, battery parameters, the one-stage reachable interval, the **frozen finite demand support**, the battery-placement API, and the reproducible evaluation protocol. **Byte-identical copy in the Exa package.** | +| `battery_solution_schema.jl` | the shared long-format solution schema, and an engine-neutral recomputation of the physical residuals. **Byte-identical copy in the Exa package.** | +| `battery_demand.jl` | the **authoring** side of demand: the composable sampler abstraction, the deterministic profile, sampler validation, and `freeze_demand_support` | +| `build_battery_case.jl` | PGLib acquisition and validation, recourse pricing, the general case constructor, the correctness case, the research case, and the Exa mirror | +| `battery_powermodels.jl` | the battery layer on top of PowerModels: the shared problem specification, stage-model construction, the strict stage solve with its multipliers, nodal prices, binding limits, solution extraction, and the runtime provenance assertion | +| `battery_diagnostics.jl` | the diagnostic toolkit: incoming-energy sampling, the targetless probe, the fixed-outgoing-energy value curve, the multiperiod deterministic equivalent and the perfect-foresight panel | +| `battery_analysis.jl` | tables and figures over the shared schema | +| `battery_sddp.jl` | the SDDP baseline: a SELECTABLE convex backward graph (SOC-WR or DC), true-ACP forward graph, stock cuts, paired simulation on a protocol, the standard cost report, and the study's four method identifiers | +| `battery_portfolio.jl` | the **preregistered PGLib panel**: hash-keyed storage placement, PTDF-sensitivity regions, the finite joint demand support, the ACP headroom calibration, and the manifest a reader regenerates every case from | +| `battery_portfolio.json` | the frozen panel manifest. **Byte-identical copy in the Exa package.** | +| `portfolio_runner.jl` | the production runner: one preemptible SEGMENT of one long SDDP run (`sddp_soc` or `sddp_dc`), with identity binding, verified checkpoints that embed the cuts, resume and a stop protocol | +| `test/runtests.jl` | the consolidated regression suite | +| `case//` | the constructed artifacts: `network.json`, `batteries.json`, `demand.json`, `case_manifest.json`. **Not committed** — see below. | + + +## Running a long study: `portfolio_runner.jl` + +`train_battery_sddp` is one training call in one process. A study run is longer +than any queue reservation and can be killed at any moment, so it is executed as +a sequence of SEGMENTS, each a separate invocation of `portfolio_runner.jl` that +continues the previous one from a verified checkpoint. The runner adds no +science: the policy graphs, the stock `AlternativeForwardPass` / +`AlternativePostIterationCallback` pair, the cuts, the cost and the paired +protocol evaluation are the same objects `battery_sddp.jl` describes. + +```bash +julia --project=. portfolio_runner.jl \ + --case-manifest case/pglib_opf_case118_ieee/case_manifest.json \ + --method sddp_soc \ + --config config.toml \ + --protocol screening.toml \ + --output run/seg001 \ + --resume-from none +``` + +Those six flags are the whole contract; `--run-id`, `--segment`, `--attempt`, +`--stop-file` and `--max-seconds` exist for an automated caller and all default. +The protocol descriptor is written once per case with + +```julia +include("portfolio_runner.jl") +write_protocol_descriptor("case/pglib_opf_case118_ieee", "screening.toml") +``` + +and a descriptor naming the FINAL protocol is refused, both when writing one and +when a run is launched against one — before any scenario is solved. + +**Continuation is the only path.** The runner trains in chunks of +`checkpoint_every` iterations, and every chunk rebuilds both graphs from the +frozen case and restores the previous chunk's cuts through +`SDDP.read_cuts_from_file` — whether or not the process was ever interrupted. A +resumed run therefore does not merely resemble the uninterrupted one, it takes +the identical path. `train_battery_sddp` gained one keyword for this, +`resume_cuts`, which reads a tagged cut file into the backward graph AND the ACP +forward graph before the first iteration; a forward graph resumed without the +cuts would decide against an empty cost-to-go. No solver object is serialized +and no SDDP internal is parsed by hand. + +Sampling is made a function of the global iteration index the same way: each +chunk's seed is derived from `(seed, iterations already completed)`. + +**What a segment writes.** `checkpoints/ck_XXXXXXXX..json`, which embeds +the cut set exactly as `SDDP.write_cuts_to_file` produced it together with the +iteration count, the sampling position, the convergence and evaluation histories +and the best admissible true-ACP forward result — plus a `.meta.toml` sidecar +naming its digest, written second so no sidecar can vouch for an unfinished +file. Then `history.csv`, `trajectory.csv`, `evaluation.csv`, `result.toml` and +`identity.toml`. The arm's scalar travels with `bound_name` and +`bound_bounds_acp` in every one of them, so the DC arm's number never acquires +the word "bound" from a column heading. + +--- + +# The user workflow + +The whole workflow is nine calls. Everything below is a single Julia session +started with `julia --project=.`. + +```julia +include("build_battery_case.jl") # → battery_case.jl, battery_demand.jl +include("battery_diagnostics.jl") # → battery_powermodels.jl +include("battery_analysis.jl") +``` + +## 1. Obtain any PGLib case + +```julia +src = acquire_pglib_case("pglib_opf_case30_ieee") +src.report # counts, totals, connected component, generation headroom +src.versions # the PGLib / PowerModels / Julia versions the bytes came from +``` + +`acquire_pglib_case` preserves the benchmark's own component identifiers — PGLib +cases contain nonconsecutive ones — and validates that every in-service component +sits on an existing bus, that every in-service bus is connected to the reference +bus, and that the case has a positively priced generator and positive load. + +PGLib ships three libraries per benchmark and they are **different systems, not +different settings of one**. The suffix selects it: + +```julia +acquire_pglib_case("pglib_opf_case30_ieee") # typical operating condition +acquire_pglib_case("pglib_opf_case30_ieee__api") # congested (active power increase) +acquire_pglib_case("pglib_opf_case30_ieee__sad") # small angle difference +``` + +## 2. Specify or randomly sample battery locations + +```julia +# explicit +buses, prec = select_battery_buses(src.network, ExplicitPlacement([5, 12, 30])) + +# a reproducible uniform draw from the load buses +buses, prec = select_battery_buses(src.network, SampledPlacement(3; seed = 7)) + +# weighted by nominal demand +load_at = nominal_load_at_bus(src.network) +buses, prec = select_battery_buses(src.network, + SampledPlacement(3; seed = 7, weight = b -> max(0.0, get(load_at, b, 0.0)))) + +# a custom eligibility rule, and a custom placement rule +select_battery_buses(src.network, SampledPlacement(2; seed = 7); + eligible = bus -> Float64(bus["vmax"]) >= 1.06) +select_battery_buses(src.network, CallablePlacement( + (candidates, meta) -> candidates[1:2]; name = "two-lowest-ids")) +``` + +Eligibility rejects nonexistent, out-of-service and disconnected buses, and a +sampled draw is without replacement. `prec` is the manifest record: the full +eligible pool, the strategy, the seed, the weights and the selection. + +## 3. Configure capacities and initial energy + +```julia +total = sum(max(0.0, v) for v in values(nominal_load_at_bus(src.network))) +fleet, crec = battery_fleet(src.network, buses; + power = 0.04 * total, # pu, per battery + energy_hours = 2.0, # duration at full power + charge_efficiency = 0.95, discharge_efficiency = 0.95, + self_discharge = 0.999, throughput_cost = 5.0, + initial_fraction = 0.5) +``` + +Every rating accepts a scalar, a `Dict` keyed by bus, or a callable — which is +how a **sampled** capacity is expressed without the case contract depending on +Distributions.jl: + +```julia +rng = StableRNG(4) +fleet, _ = battery_fleet(src.network, buses; + power = _ -> rand(rng, Uniform(0.2, 0.4)), energy_hours = 2.0) +``` + +## 4. Define a demand sampler + +Demand is the only uncertainty. For the original PGLib load values, the realized +demand of load `i` at stage `t` is + +``` +pd[i,t] = h[i,t] * m[i,t] * pd0[i] +qd[i,t] = h[i,t] * m[i,t] * qd0[i] +``` + +with `h` a deterministic profile and `m` the uncertain multiplier. The **same** +multiplier scales active and reactive demand, so every realization preserves each +load's own power factor. + +A sampler's fundamental output is a **joint multiplier vector** over the case's +loads — not a scalar, and not an implicit collection of independent draws. + +```julia +meta = demand_meta(src.network) # load_ids, load_bus, nominal_pd/qd, num_loads +``` + +**A `Distribution`, system-wide:** + +```julia +sampler = SystemMultiplier(Uniform(0.9, 1.1)) +sampler = SystemMultiplier(DiscreteNonParametric([0.95, 1.0, 1.05], fill(1/3, 3))) +``` + +**Independent per load (a convenience, not the general interface):** + +```julia +sampler = IndependentMultiplier(LogNormal(0.0, 0.05)) +sampler = IndependentMultiplier(Dict(3 => Uniform(0.8, 1.2))) # others deterministic +``` + +**Regional finite atoms — a genuinely correlated joint vector:** + +```julia +pocket = [j for j in 1:meta.num_loads if meta.load_bus[j] in [1,3,5,6,7,8,9,14]] +sampler = GroupMultiplier( + [meta.load_ids[pocket], setdiff(meta.load_ids, meta.load_ids[pocket])], + [DiscreteNonParametric([0.96, 1.04], [0.5, 0.5]), + DiscreteNonParametric([0.99, 1.01], [0.5, 0.5])]) +``` + +**A custom callable** — the general interface every other sampler is a +convenience over: + +```julia +sampler = CallableMultiplier( + (rng, t, meta) -> 1.0 .+ 0.05 .* randn(rng, meta.num_loads) .* (t > 12); + name = "late-stage jitter") +``` + +**Explicit atoms, and stage dependence:** + +```julia +sampler = FiniteMultiplier([0.9, 1.0, 1.1], [0.25, 0.5, 0.25]) +sampler = StageMultiplier([t <= 12 ? DeterministicMultiplier(1.0) : sampler + for t in 1:24]) +``` + +**Composition** multiplies element-wise, so a system-wide level, a regional +effect and a per-load term compose without knowing about each other: + +```julia +sampler = ProductMultiplier(SystemMultiplier(Uniform(0.98, 1.02)), + GroupMultiplier(...), IndependentMultiplier(...)) +``` + +Check it before freezing: + +```julia +validate_sampler(sampler, src.network, 24) +``` + +which asserts the output dimension and load order, finiteness and +nonnegativity, exact seeded reproduction, that the sampler does not reach for the +global RNG, that every load keeps its power factor, and that any declared finite +support normalizes. + +## 5. Freeze the sampler into finite support + +**Both SDDP and TS-DDR train from finite support.** A general sampler or a +continuous `Distribution` is an *authoring mechanism*; before either method +trains, it is materialized into a frozen, hashed, stage-major support + +``` +W_t = { (w_{t,1}, p_{t,1}), …, (w_{t,K_t}, p_{t,K_t}) } +``` + +and both engines read the **same bytes**. Neither method may resample or +rediscretize the authoring law on its own — that is the boundary that makes "the +two methods faced the same stochastic program" a checkable statement. + +```julia +support = freeze_demand_support(sampler, src.network, 30; + seed = 20260805, + method = :auto, # :exact | :empirical | :auto + atoms_per_stage = 8, # for an empirical freeze + profile = diurnal_profile(30), + profile_period = 24, + stage_hours = 1.0, + protocol_seed = 20260805) +support_digest(support) +``` + +- an **explicitly discrete** sampler keeps its support and probabilities exactly + — no reweighting, no resampling, only exact-duplicate merging; +- a **continuous or general** sampler is discretized transparently: draw + `atoms_per_stage` joint vectors per stage from `StableRNG(seed)` and weight them + equally. It is reproducible from `(seed, atoms_per_stage)` alone and claims no + moment matching. A different discretizer is declared by giving the sampler a + `support` callable; there is no hidden quadrature rule. + +The frozen support records the authoring sampler's description, the seed, the +atom count, the method and the resulting atoms in `demand.json`. + +**Four distinct objects, and the README will not conflate them:** + +| object | what it is | where it lives | +|---|---|---| +| authoring sampler | an arbitrary joint law, possibly continuous | your code, never a case | +| frozen training support | finite atoms + probabilities per stage, hashed | `demand.json`, mirrored to both engines | +| screening protocol | a small set of global scenario IDs drawn from that support, used while choosing a case and selecting checkpoints | regenerated from `protocol_seed` | +| final unseen protocol | a fresh, larger paired protocol no policy was ever selected on | regenerated from the manifest, evaluated once | + +## 6. Build and verify the manifest + +```julia +case = build_case(src; dir = "case/my_case", batteries = fleet, support = support, + placement = Dict("buses" => prec, "capacity" => crec), + protocol_stages = 30, protocol_scenarios = 500) +verify("case/my_case") +``` + +`build_case` writes the four artifacts, records every hash, and reads the case +back **through the verifier** — a case that cannot be re-read is a build failure +rather than a later mystery. Two properties are deliberately fail-closed, because +both have historically corrupted a study silently: the stage duration must be +present, positive and agree with the manifest, and every artifact must hash to +what the manifest records. + +## 7. Sample incoming battery energies + +```julia +states = sample_incoming_energy(case; kind = :uniform, seed = 3, batch = 8) +sample_incoming_energy(case; kind = :fixed, level = 0.5) +sample_incoming_energy(case; kind = :distribution, dist = Beta(2, 2), seed = 3, batch = 4) +sample_incoming_energy(case; kind = :callable, callable = (rng, b, m) -> 0.25, seed = 1) +``` + +The state is sampled in NORMALIZED terms and mapped into each battery's own +bounds, so a sampler transfers unchanged to a case with different ratings. Every +resulting energy is validated. This samples **diagnostic initial states only** — +it is not the demand process and plays no part in training or evaluation. + +## 8. Run targetless single-stage probes + +```julia +p = targetless_probe(case, PowerModels.ACPPowerModel; + energy_in = states[1], stage = 19, atom = 2) +p.cost_stage, p.cost_generation, p.cost_deficit, p.cost_surplus +p.energy_out, p.p_ch, p.p_dis, p.p_bat, p.simultaneous +p.vm, p.va, p.p_fr, p.q_to, p.price_active, p.price_reactive +p.energy_in_dual # ∂Q/∂e_in — what the INHERITED energy was worth +p.residuals # recomputed independently of the engine +binding_table(p) # what stopped the network +``` + +The same call with `PowerModels.SOCWRConicPowerModel` runs the relaxation, and +`targetless_batch` sweeps incoming-energy vectors × stages × atoms, keeping every +combination including the failures. + +> **A targetless one-stage solve is myopic.** Nothing in it prices the energy +> left in a battery at the end of the stage, so it will rationally discharge as +> much as is useful and leave the battery empty. That is the definition of a +> one-stage problem, not a finding. Read it for the physics — what the network +> could deliver, what limit stopped it, what the energy the stage *inherited* was +> worth — and read the next section for the value of what is left behind. + +## 9. Inspect fixed-target value curves and nodal prices + +```julia +b = first(case.batteries) +lo, hi = reachable_interval(b, states[1][b.index], stage_hours(case)) +cmp = compare_value_curves(case; battery = b.index, energy_in = states[1], + stage = 19, atom = 2, + grid = collect(range(lo, hi; length = 9))) +value_curve_table(cmp) +plot_value_curves(cmp, "value_curves.png") +``` + +The outgoing energy is fixed by the **same hard equality the strict formulation +uses** — there is no target slack anywhere — so the reported multiplier is +`∂Q/∂e_out`, the price the model puts on carrying one more unit of energy out of +the stage. + +Three properties of the check matter: + +- **finite differences confirm the interior multipliers.** The value curve is + convex and *piecewise* smooth — a binding limit puts a kink in it — so each + interior point is classified first, and `λ` is compared against a central + difference only where the curve is locally smooth. At a kink the correct + statement is that `λ` lies *between* the one-sided slopes, and `bracketed` + records it. +- **endpoints are not evidence.** At an endpoint of the reachable interval the + target sits on a bound, `λ` is a subgradient, and two solvers may report two + valid values orders of magnitude apart. +- **points that used physical recourse are excluded.** There, part of the + marginal value is the recourse *price* — chosen to be far above any generator — + rather than the network's valuation of stored energy. + +The quantity that carries a finding is `dlambda = λ_SOC − λ_ACP`, and its +locational *ranking*: a uniform level shift changes nothing about where a policy +puts energy, a change in the order of buses changes everything. + +```julia +marginal_value_table([cmp1, cmp2, ...]) # rank_acp, rank_soc, rank_changed +``` + +## 10. Solve one multiperiod deterministic equivalent + +```julia +proto = scenario_index_matrix(case.demand, 30, 500) +de = deterministic_equivalent(case, proto[:, 1]) # true ACP +de.total_cost, de.stage_cost, de.cumulative_cost +de.energy_terminal, de.throughput, de.simultaneous +de.worst_deficit, de.worst_surplus, de.residuals +stage_cost_table(de); battery_table(de, case); price_table(de, case) +plot_energy_trajectory(de, case, "energy.png") +``` + +The whole horizon is solved at once, knowing the entire demand path: no policy, +no target, no target slack, no future-cost approximation. It is assembled from +the same per-stage builder the study trains on — one PowerModels model per stage, +all in one JuMP model, coupled only by `e_in[t+1] == e_out[t]`. + +The same call with `model_type = PowerModels.SOCWRConicPowerModel` gives the +**relaxed** perfect-foresight solve. That is a diagnostic, not a physical +reference: its cost is not attainable. + +## 11. Calculate a small perfect-foresight panel + +```julia +panel = perfect_foresight_panel(case, proto; ids = 1:8) +panel.complete, panel.mean, panel.std, panel.sem +panel_table(panel) +``` + +Every requested identifier is retained: none is replaced, dropped or renumbered, +a first-attempt failure is re-solved once with a completely fresh model, and both +statuses are recorded. If any identifier is unsolved the panel is marked +incomplete — a mean over the scenarios that happened to succeed is a mean over a +different problem. + +> **What the mean is, and is not.** The true-ACP perfect-foresight mean is a +> **wait-and-see lower bound** on the nonanticipative stochastic problem: a +> clairvoyant operator cannot be beaten by one who must decide before seeing the +> future. The gap between a policy and this bound is diagnostic **headroom**, and +> it *contains the value of future information*, which no nonanticipative policy +> — TS-DDR or SDDP — can recover. It is not a target and it is not attainable. +> The bound may be computed while screening candidates; its relationship to a +> trained policy is assessed only on **paired** paths, after that policy exists. + +## 12. Train and evaluate SDDP + +```julia +trained = train_battery_sddp(case; num_stages = 30, iteration_limit = 300) +trained.bound # over the SOC-WR relaxation, over 30 stages +assert_graph_provenance(trained.backward_graph, PowerModels.SOCWRConicPowerModel) +assert_graph_provenance(trained.forward, PowerModels.ACPPowerModel) + +sims = simulate_battery_sddp_on(trained, proto; + ids = [b.index for b in case.batteries], columns = 1:8) +costs = [sum(s[t][:stage_objective] for t in 1:30) for s in sims] +paired_difference(costs, [panel.results[c].total_cost for c in 1:8]) +``` + +The backward pass is an actual convex PowerModels formulation, the forward pass +an actual `PowerModels.ACPPowerModel`, and the two are joined by SDDP.jl's own +`AlternativeForwardPass` / `AlternativePostIterationCallback`. No +`duality_handler` is overridden and no cut is filtered, retried or reweighted. + +`simulate_battery_sddp_on` replays a fixed protocol through `SDDP.Historical`, +which is what makes an SDDP cost **paired** with a perfect-foresight cost and, +later, with a TS-DDR cost. + +### Two backward formulations, and what their scalars mean + +`backward = :soc` (the default) builds the backward nodes as +`PowerModels.SOCWRConicPowerModel`; `backward = :dc` builds them as +`PowerModels.DCPPowerModel`. That is the only thing the keyword changes: the +forward pass, the demand support and its probabilities, the storage state, the +battery layer, the generator data and the generator cost polynomials, the +uncapped physical recourse and the SDDP machinery are shared, not duplicated. + +```julia +dc = train_battery_sddp(case; backward = :dc, num_stages = 24, iteration_limit = 300, + cut_path = sddp_cut_path("out", case, :dc)) +dc.backward # :dc +dc.backward_formulation # PowerModels.DCPPowerModel +dc.bound_name # "DC-approximation training bound" +dc.bound_bounds_acp # false +sddp_method_id(dc) # :sddp_dc +``` + +The two SCALARS are not the same kind of object and this example never lets them +be printed as if they were: + +| arm | scalar | is it a lower bound on the true ACP problem? | +|---|---|---| +| `:soc` | **SOC-WR relaxation bound** | **yes** — the relaxation lower-bounds ACP, so its bound does too | +| `:dc` | **DC-approximation training bound** | **no** — the DC approximation drops the reactive balance and fixes voltage magnitudes, so it is neither a relaxation nor a restriction of ACP and its value bounds nothing in either direction | + +The DC scalar is the internal convergence scalar of the approximation the cuts +came from, reported to say whether that training converged. It changes arithmetic +in exactly one place: `cost_report`'s recoverable ceiling `f` drops the bound +term from `max(bound, pf_mean)` on the DC arm, because a number that does not +bound the best nonanticipative cost has no right to tighten a cap on it. + +Neither scalar is comparable to a forward objective accumulated over a different +number of stages, and nothing here invites that. + +**The arms cannot be confused.** `backward`, `backward_formulation`, `bound_name` +and `bound_bounds_acp` travel with every result; `cost_report` prints the method +identifier and labels row `b` from `bound_name`; and a cut file must carry the +arm's tag in its name — `sddp_cut_path` builds one and `train_battery_sddp` +refuses a path that does not. + +### The two arms no longer share a solver + +`socwr_optimizer` (Clarabel, frozen at `tol = 1e-8` with equilibration on) solves +the SOC-WR backward nodes. `dc_optimizer` is **HiGHS at its own defaults**, with +output suppressed and nothing else set, for every case alike. + +They were the same factory until the portfolio measured what that cost. Under +`DCPPowerModel` the backward subproblem is a convex QP over a *linear* feasible +set — PowerModels' DC equations, the linear battery transition, finite state and +control bounds, and uncapped positive-price active recourse. Handing that LP to +an interior-point conic solver returned `INFEASIBLE`, `DUAL_INFEASIBLE`, +`LOCALLY_INFEASIBLE` and `SLOW_PROGRESS` on nine of ten portfolio cases, at stage +nodes between 3 and 22, on subproblems whose primal feasibility is not in +question. The provocation is conditioning, not modelling: the right-hand side +spans roughly ten orders of magnitude, from reactive-demand deviations near +`5e-4` to recourse prices near `7e5`. Presolve and a simplex/QP basis absorb that +range; an unpreconditioned conic IPM does not. + +Sharing one solver was meant to make the two arms differ only by FORMULATION. +That is still what they differ by — a DC baseline that cannot finish an iteration +measures the solver, not the DC approximation, and reporting it as the latter +would be false. Each formulation is now solved by something that can solve it, +with no per-case tuning on either side and no fallback anywhere. + +The DC arm is validated against a **direct PowerModels DC oracle**: an ordinary +`solve_opf(net, DCPPowerModel, …)` on a network with no storage table at all, +whose per-bus load is the realized demand minus the battery's net injection. It +must agree on generation, generation cost and every branch flow, and the DC nodal +balance and line flows are recomputed independently from the reported angles. + +## 13. Where strict TS-DDR enters + +Nothing above trains a policy. Strict TS-DDR is the GPU engine's business: +`DecisionRulesExa.jl/examples/BatteryStorageOPF` reads the **same frozen case +bytes and the same frozen support**, builds the multistage true-ACP +deterministic equivalent in ExaModels, evaluates the reachable policy before each +stage, and differentiates the stage value through the strict target multiplier. +This engine's role afterwards is to replay and verify what that engine produced, +through the shared solution schema. + +--- + +# The preregistered PGLib portfolio + +Sections 1–13 are the toolkit. The **study** is not one case built by hand with +it: it is a panel of canonical PGLib systems whose every construction choice is +a documented function of the benchmark's own bytes. `battery_portfolio.jl` is +that function, and `battery_portfolio.json` is what it produced. + +## Regenerating a case + +```bash +julia --project=. battery_portfolio.jl --list # the panel +julia --project=. battery_portfolio.jl --verify # check the manifest +julia --project=. battery_portfolio.jl --case pglib_opf_case118_ieee --out /tmp/panel +``` + +The third command acquires the canonical PGLib case, recomputes the regions and +the placement, rebuilds the frozen support at the recorded demand level, writes +the four case artifacts into `/tmp/panel/pglib_opf_case118_ieee/`, and **fails +closed** if the acquired network, the recomputed regions, the recomputed +placement, the frozen support or any written artifact does not hash to what the +manifest records. Add `--verify-all` to re-run the full `24 × 6` headroom gate +as well. No private repository, no cluster scheduler and no pre-generated JSON +is involved. + +## What is frozen, and how + +| choice | rule | +|---|---| +| horizon | `T = 24`, one hour per stage | +| profile | one common normalized 24-value daily profile, multiplying `pd` AND `qd`, so every realization keeps each load's own power factor | +| eligible buses | in service, positive nominal active demand, and reachable from the reference bus over in-service branches | +| battery count | `min(#eligible, clamp(round(0.20 n_bus), 24, 240))` | +| placement | weighted sampling without replacement, proportional to nominal active demand, with each bus's key drawn from `SHA-256(schema, "placement", seed, network digest, bus)` — no RNG, so the panel survives a reimplementation | +| ratings | 10 % of the calibrated peak active demand, split by nominal demand capped at 3× the selected-bus median; 8 h duration, 5 % reserve, 50 % initial, 0.95/0.95 efficiency, 0.999 self-discharge, throughput cost 5.0 | +| regions | six, **demand-balanced** assignment over unit-norm PTDF sensitivity signatures on the highest-reach rated corridors, relabelled by descending demand; each region carries 8–28 % of nominal demand | +| uncertainty | six equiprobable joint atoms; atom `r` gives region `r` a multiplier of `1.15` and every other region `0.97`, so each region's support mean is exactly `(1.15 + 5×0.97)/6 = 1` and the regions are **negatively** correlated | +| demand level | `κ_case = 0.95 κ_max`, where `κ_max` is the largest level in `[0.50, 1.25]` at which every atom of the peak-profile stage solves in base ACP, unmodified and battery-free, with residual ≤ 1e-7 and no meaningful recourse | +| protocols | a 500-column final panel from one seed, and a 32-column screening panel from an independent seed, **repaired against the final one so the two share no scenario by construction** | + +The demand level is the only quantity the manifest carries that a reader cannot +cheaply recompute — it costs a bisection plus a `24 × 6` verification of true-ACP +solves per case — so it is recorded and `--verify-all` re-derives it on demand. +Everything else in the manifest is a digest of something the reader regenerates. + +## Why the regions are balanced + +Six regions are six LEVERS only if they carry comparable demand. The atoms move +demand by region, so a region holding 0.2 % of the load is an atom that moves +nothing — and an unconstrained clustering does exactly that: on the first freeze +it put 80.8 % of `case1951_rte`'s demand in one region and 61.5 % of +`case300_ieee`'s, collapsing six atoms toward two directions. + +The assignment step is therefore an integer program (HiGHS, through JuMP) that +keeps the same PTDF objective and adds the demand bounds: + +```math +\min_x \sum_{i,r} w_i \lVert s_i - c_r \rVert^2 x_{ir} +\quad\text{s.t.}\quad +\sum_r x_{ir} = 1,\; +0.08\,W \le \sum_i w_i x_{ir} \le 0.28\,W,\; +x_{ir} \in \{0,1\}, +``` + +refined over Lloyd sweeps so the bounds hold at every sweep rather than only at +the end. A bus is indivisible, so if ONE bus alone exceeds 28 % the cap rises to +exactly that bus's share and a `Σy ≤ 1` constraint lets a single region use it — +the minimum necessary exception, recorded in the manifest. + +Balancing did not cost PTDF coherence. Measured over the panel, weighted +within-region signature dispersion went to 0.79–1.10× of the unconstrained +value — better on seven cases — because both are local searches and solving each +assignment step to global optimality lands in a better basin. Both numbers are +recorded per case so the tradeoff is visible rather than assumed. + +## The reported cost is not the solver's objective + +An interior-point method does not leave a nonnegative variable at zero; it +leaves it a barrier tolerance away, and the sign depends on the solver. The +recourse price is 1e5–1e6 per pu, so 1e-8 pu on a couple of thousand buses is +tens of cost units of pure numerical residue — on a stage where neither engine +used any recourse at all. + +`physical_stage_cost`, in the byte-identical `battery_solution_schema.jl`, is the +only function either engine may use to produce a headline cost: + +```julia +c = physical_stage_cost(sol, case.recourse) +c.raw # the solver's own objective, preserved for diagnostics +c.corrected # generation + throughput + recourse actually charged +c.correction # the barrier artifact, reported rather than discovered +c.admissible # false if any element exceeded the physical tolerance +``` + +Every recourse element within `PHYSICAL_RECOURSE_TOL = 1e-6` pu of zero is +projected to exactly zero, element by element. An element OUTSIDE it is not +projected: the solve is marked inadmissible and the caller rejects it. The +projection changes what is reported, never what was solved — the stage problem +still carries the recourse at full price. + +> **The margin is not a knob.** `0.95` is a constant of `battery_portfolio.jl`, +> fixed before any method was run, identical for every case. So is the bracket, +> so is the tolerance, and so is the solver configuration: a case is never +> "helped" to a higher level. A case whose unmodified base ACP fails at `0.50` is +> **replaced** from a preregistered reserve list, and the replacement is recorded +> in the manifest. A SOC, DC or method failure never causes a replacement. + +## Validating a frozen case + +```julia +include("battery_portfolio.jl") +case = materialize_portfolio_case("pglib_opf_case118_ieee"; dir = "/tmp/panel/c118") +validate_portfolio_case(case; stages = 1:24, atoms = 1:1) # strict-stage gate +aggressive_charge_probe(case) # the diagnostic +``` + +`validate_portfolio_case` checks that every initial state is inside its bounds, +that every reachable interval is nonempty, that the idle/hold target is feasible, +that every strict ACP solve completes, that the independently recomputed physical +residual is at most 1e-7 and that no solve uses meaningful recourse. + +`aggressive_charge_probe` is the opposite: it aims the whole fleet at the top of +its reachable interval on one stage, which is an admissible target the network +may not be able to serve. When it draws recourse, the right outcome is that +admissibility **rejects** the solution — not that the case is changed. + +--- + +## Commands + +```bash +# rebuild (and mirror) the correctness case from PGLib +DR_BAT_MIRROR=/path/to/DecisionRulesExa.jl/examples/BatteryStorageOPF \ + julia --project=. build_battery_case.jl + +# the portfolio panel +julia --project=. battery_portfolio.jl --list +julia --project=. battery_portfolio.jl --verify +julia --project=. battery_portfolio.jl --case --out + +# re-verify a frozen case in place: hashes, schemas, stage duration, support, protocol +julia --project=. build_battery_case.jl --verify + +# stock SDDP: SOC-WR backward, true-ACP forward, construction smoke +julia --project=. battery_sddp.jl + +# the same smoke with DC backward nodes +DR_BAT_SDDP_BACKWARD=dc julia --project=. battery_sddp.jl + +# the consolidated regression suite +julia --project=. test/runtests.jl +``` + +## The study's four method identifiers + +`BATTERY_METHODS` carries the four the study compares, with the same rows and the +same invariant fields in **both** public engines: + +| identifier | engine | what varies | +|---|---|---| +| `tsddr_nonlinear` | the Exa engine | LSTM encoder, nonlinear head | +| `tsldr_recurrent_linear` | the Exa engine | affine recurrence, affine head | +| `sddp_soc` | this one | `SOCWRConicPowerModel` backward cuts | +| `sddp_dc` | this one | `DCPPowerModel` backward cuts | + +```julia +battery_method(:sddp_dc) # the descriptor and the shared invariants +run_battery_method(:sddp_dc, case; num_stages = 24, iteration_limit = 300) +run_battery_method(:tsddr_nonlinear, case) # refused here: it is the Exa engine's +``` + +Every row declares the same horizon (24), protocol (screening), strict target +semantics, recourse and admissibility rule, cost contract +(`physical_stage_cost`) and comparison path (true ACP on paired protocol +columns), and each suite asserts it. `run_battery_method` is a dispatch layer, +not a campaign runner: it selects an implementation and forwards keyword +arguments, and it schedules nothing. + +## Environment variables + +Case construction (`build_battery_case.jl`): + +| variable | meaning | +|---|---| +| `DR_BAT_CASE` | PGLib case name (default `pglib_opf_case14_ieee`) | +| `DR_BAT_DIR` | output directory for the artifacts | +| `DR_BAT_NUM` | number of batteries to place | +| `DR_BAT_BUSES` | comma-separated explicit bus override; bypasses the seeded draw | +| `DR_BAT_SEED` | seed of the deterministic placement draw | +| `DR_BAT_HORIZON` | frozen horizon | +| `DR_BAT_PROTOCOL_SEED`, `DR_BAT_PROTOCOL_STAGES`, `DR_BAT_PROTOCOL_SCENARIOS` | the evaluation protocol the manifest records a digest for | +| `DR_BAT_MIRROR` | Exa example directory to mirror the case and shared sources into | + +SDDP smoke (`battery_sddp.jl`): `DR_BAT_SDDP_BACKWARD` (`soc`), +`DR_BAT_SDDP_STAGES` (3), `DR_BAT_SDDP_ITERATIONS` (10), `DR_BAT_SDDP_SIMS` (6). + +## What the recourse variables are, and are not + +Every bus carries two nonnegative, UNCAPPED variables — a deficit injection `d` +and a surplus sink `s` — which enter the ACTIVE nodal balance with opposite +signs and are priced far above any generator. They are physical operating +recourse: they are what makes a dynamically reachable battery target attainable +under the true network, and they are charged identically in the ACP model, the +SOC-WR model, both SDDP passes and the Exa engine. + +They are NOT target slack. There is no target-slack variable, no target penalty +and no soft-target formulation anywhere in the supported workflow; the strict +target is a hard equality whose multiplier is the actor signal. A policy that +uses either recourse variable materially is rejected, not priced — and a +diagnostic point that used one is excluded from the evidence rather than +reported. + +## Expected outputs + +`build_battery_case.jl` prints the case summary and the artifact, support and +protocol digests. `battery_sddp.jl` prints the method identifier of the arm it +ran, its backward scalar under that arm's own name together with whether that +scalar bounds the true ACP problem, the number of stock cuts created, the +true-ACP forward cost over the simulated paths, the worst recourse on any +simulated stage, and one battery's energy trajectory. The scalar and the forward +cost are quoted over the SAME horizon; a bound never bounds a metric accumulated +over a different number of stages. diff --git a/examples/BatteryStorageOPF/battery_analysis.jl b/examples/BatteryStorageOPF/battery_analysis.jl new file mode 100644 index 0000000..0dc32e4 --- /dev/null +++ b/examples/BatteryStorageOPF/battery_analysis.jl @@ -0,0 +1,467 @@ +# battery_analysis.jl +# +# Tables and figures for the battery-storage study. +# +# Everything here consumes the SHARED SOLUTION SCHEMA — the long +# `(scenario, stage, class, index, value)` format defined in +# `battery_solution_schema.jl` — or the named tuples the diagnostics return. +# Nothing reads a solver-specific internal layout, which is what lets one plot +# overlay a JuMP trajectory and an ExaModels trajectory without either engine +# knowing about the other. +# +# Figures are written to files rather than displayed: this study runs on cluster +# nodes without a display, and a figure that only exists in a REPL cannot be +# regenerated from the evidence. + +using DataFrames +using Plots +using Printf +using Statistics + +@isdefined(SolutionRecorder) || include(joinpath(@__DIR__, "battery_solution_schema.jl")) + +# Headless rendering: GR opens no window and writes straight to the file. +ENV["GKSwstype"] = get(ENV, "GKSwstype", "100") + +# ───────────────────────────────────────────────────────────────────────────── +# The schema as a DataFrame +# ───────────────────────────────────────────────────────────────────────────── + +""" + solution_frame(source) -> DataFrame + +Read the shared solution schema into a long `DataFrame` with columns +`scenario, stage, class, index, value`. + +# Arguments +- `source`: a path to a solution CSV, a [`SolutionRecorder`](@ref), or the + dictionary [`read_solution`](@ref) returns. + +# Notes +Long format is kept all the way to the plotting layer. Widening happens per +figure, through [`pivot_class`](@ref), because different figures widen on +different keys and a single wide table would have to guess which. +""" +function solution_frame(source) + rows = if source isa SolutionRecorder + source.rows + elseif source isa AbstractString + [(k[1], k[2], k[3], k[4], v) for (k, v) in read_solution(source)] + elseif source isa AbstractDict + [(k[1], k[2], k[3], k[4], v) for (k, v) in source] + else + error("solution_frame: unsupported source of type $(typeof(source))") + end + df = DataFrame(scenario = [r[1] for r in rows], stage = [r[2] for r in rows], + class = [r[3] for r in rows], index = [r[4] for r in rows], + value = [r[5] for r in rows]) + return sort!(df, [:scenario, :stage, :class, :index]) +end + +""" + pivot_class(df, class; scenario=nothing) -> DataFrame + +One row per stage, one column per component identifier, for a single class. + +# Notes +Column names are the component IDENTIFIERS as strings, never positions, so a +table built from a case with nonconsecutive identifiers still names the right +component. +""" +function pivot_class(df::DataFrame, class::AbstractString; scenario = nothing) + sub = scenario === nothing ? df[df.class .== class, :] : + df[(df.class .== class) .& (df.scenario .== scenario), :] + isempty(sub) && return DataFrame(stage = Int[]) + return unstack(sub, :stage, :index, :value; renamecols = i -> Symbol(string(i))) +end + +# ───────────────────────────────────────────────────────────────────────────── +# Tables +# ───────────────────────────────────────────────────────────────────────────── + +""" + stage_cost_table(result) -> DataFrame + +Per-stage cost decomposition of a [`deterministic_equivalent`](@ref) result: the +generation, throughput, deficit and surplus components, the stage total and the +running total. + +# Notes +The cumulative column is what makes a stagewise comparison readable: two policies +that differ by a fraction of a percent per stage are indistinguishable stage by +stage and obvious cumulatively. +""" +function stage_cost_table(result) + T = result.horizon + return DataFrame( + stage = 1:T, + generation = [result.stages[t].cost_generation for t in 1:T], + throughput = [result.stages[t].cost_throughput for t in 1:T], + deficit = [result.stages[t].cost_deficit for t in 1:T], + surplus = [result.stages[t].cost_surplus for t in 1:T], + total = [result.stages[t].cost_stage for t in 1:T], + cumulative = cumsum([result.stages[t].cost_stage for t in 1:T]), + ) +end + +""" + battery_table(result, case) -> DataFrame + +Per-stage, per-battery energy, charging, discharging, throughput and the +simultaneous-operation measure. + +# Notes +`simultaneous = min(p_ch, p_dis)` is reported because a continuous relaxation of +the charge/discharge complementarity CAN return a solution that does both at +once, and a positive throughput cost is the only thing making that suboptimal. A +column of zeros here is what licenses reading `p_bat` as a physical injection. +""" +function battery_table(result, case::BatteryCase) + rows = DataFrame(stage = Int[], battery = Int[], bus = Int[], + energy_in = Float64[], energy_out = Float64[], + p_ch = Float64[], p_dis = Float64[], p_bat = Float64[], + throughput = Float64[], simultaneous = Float64[]) + Δt = stage_hours(case) + for t in 1:result.horizon, b in case.batteries + s = result.stages[t] + push!(rows, (t, b.index, b.bus, s.energy_in[b.index], s.energy_out[b.index], + s.p_ch[b.index], s.p_dis[b.index], s.p_bat[b.index], + Δt * (s.p_ch[b.index] + s.p_dis[b.index]), + min(s.p_ch[b.index], s.p_dis[b.index]))) + end + return rows +end + +""" + price_table(result, case; buses=nothing) -> DataFrame + +Per-stage active and reactive nodal prices. + +# Notes +Reactive prices are reported alongside active ones because on a case where +storage matters for VOLTAGE support rather than for energy arbitrage, the +reactive price is where the mechanism shows — and a study that only tabulates +active prices would report that nothing is happening. +""" +function price_table(result, case::BatteryCase; buses = nothing) + ids = buses === nothing ? sort!([Int(b["index"]) for (_, b) in case.network["bus"]]) : + collect(Int.(buses)) + rows = DataFrame(stage = Int[], bus = Int[], price_active = Float64[], + price_reactive = Float64[], vm = Float64[], + deficit = Float64[], surplus = Float64[]) + for t in 1:result.horizon, i in ids + s = result.stages[t] + push!(rows, (t, i, get(s.price_active, i, NaN), get(s.price_reactive, i, NaN), + s.vm[i], s.deficit[i], s.surplus[i])) + end + return rows +end + +""" + value_curve_table(cmp) -> DataFrame + +The ACP-versus-SOC-WR value curve comparison of +[`compare_value_curves`](@ref), one row per grid point. + +# Notes +The column that carries the finding is `dlambda`: the difference in the MARGINAL +value of stored energy. `dcost` — the difference in the value curves themselves — +is reported beside it and is expected to be nonzero everywhere, because a +relaxation is below the true model by construction. Reading the level difference +as the mechanism is the mistake this table is laid out to prevent. +""" +function value_curve_table(cmp) + n = length(cmp.energy) + return DataFrame( + energy = cmp.energy, + reachable = cmp.acp.reachable, + endpoint = cmp.acp.endpoint, + nonsmooth = cmp.acp.nonsmooth, + value_acp = cmp.acp.value, + value_soc = cmp.soc.value, + dcost = cmp.acp.value .- cmp.soc.value, + lambda_acp = cmp.acp.lambda, + lambda_soc = cmp.soc.lambda, + dlambda = cmp.dlambda, + fd_acp = cmp.acp.fd, + fd_error_acp = cmp.acp.fd_error, + recourse_acp = cmp.acp.worst_recourse, + ) +end + +""" + marginal_value_table(comparisons) -> DataFrame + +One row per probed battery: the interior ACP and SOC-WR marginal stored-energy +values and their disagreement, with the LOCATIONAL RANK each formulation assigns. + +# Arguments +- `comparisons`: an iterable of [`compare_value_curves`](@ref) results. + +# Notes +Ranking is the point. A uniform level shift in the relaxation's marginal values +changes nothing about where a policy puts energy; a change in the ORDER of buses +does. `rank_acp != rank_soc` is the decision-level evidence that a cut built on +the relaxation would steer storage to a different place from the truth. + +Interior points only — see [`energy_value_curve`](@ref) on why a multiplier at a +reachable-interval endpoint is not comparable across solvers. +""" +function marginal_value_table(comparisons) + rows = DataFrame(battery = Int[], bus = Int[], stage = Int[], atom = Int[], + lambda_acp = Float64[], lambda_soc = Float64[], + dlambda = Float64[], rel_dlambda = Float64[], + reversal = Bool[], num_interior = Int[], + num_recourse_excluded = Int[]) + for c in comparisons + idx = c.interior + isempty(idx) && continue + λa = mean(c.acp.lambda[idx]) + λs = mean(c.soc.lambda[idx]) + push!(rows, (c.acp.battery, c.acp.bus, c.acp.stage, c.acp.atom, + λa, λs, λs - λa, (λs - λa) / max(abs(λa), 1e-8), + c.reversal, length(idx), c.num_recourse_excluded)) + end + isempty(rows) && return rows + rows.rank_acp = competerank_desc(rows.lambda_acp) + rows.rank_soc = competerank_desc(rows.lambda_soc) + rows.rank_changed = rows.rank_acp .!= rows.rank_soc + return rows +end + +""" + competerank_desc(v) -> Vector{Int} + +Descending competition rank of `v`: the largest value gets rank 1, ties share the +smaller rank. + +# Notes +Written out rather than pulled from StatsBase, which this example does not +otherwise need, and ties are handled explicitly because two buses with equal +marginal value are not "differently ranked" in any meaningful sense. +""" +function competerank_desc(v::AbstractVector) + order = sortperm(v; rev = true) + r = Vector{Int}(undef, length(v)) + prev = NaN + prev_rank = 0 + for (pos, i) in enumerate(order) + if !(v[i] ≈ prev) + prev_rank = pos + prev = v[i] + end + r[i] = prev_rank + end + return r +end + +""" + binding_table(probe; limit=20) -> DataFrame + +The binding and nearly binding limits of a probe, most binding first. +""" +function binding_table(probe; limit::Integer = 20) + rows = first(probe.binding, limit) + return DataFrame(kind = [r.kind for r in rows], index = [r.index for r in rows], + side = [r.side for r in rows], value = [r.value for r in rows], + limit = [r.limit for r in rows], slack = [r.slack for r in rows]) +end + +""" + panel_table(panel) -> DataFrame + +One row per scenario of a [`perfect_foresight_panel`](@ref), with the statuses +kept as strings so a failed scenario survives a CSV round-trip. +""" +function panel_table(panel) + return DataFrame(scenario = [r.scenario for r in panel.rows], + solved = [r.solved for r in panel.rows], + first_status = [string(r.first_status) for r in panel.rows], + status = [string(r.status) for r in panel.rows], + cost = [r.cost for r in panel.rows], + worst_deficit = [r.worst_deficit for r in panel.rows], + worst_surplus = [r.worst_surplus for r in panel.rows], + simultaneous = [r.simultaneous for r in panel.rows], + terminal_energy = [r.terminal_energy for r in panel.rows], + throughput = [r.throughput for r in panel.rows]) +end + +# ───────────────────────────────────────────────────────────────────────────── +# Figures +# ───────────────────────────────────────────────────────────────────────────── + +""" + plot_value_curves(cmp, path) -> String + +Two stacked panels: the ACP and SOC-WR value curves, and the marginal value of +stored energy under each. + +# Notes +The marginal values go on their OWN panel rather than a twin axis. The two levels +routinely differ by less than a percent while the quantity being studied is that +difference, and overlaying them on one axis makes a difference of interest look +like a coincident pair of lines. + +Reachable-interval endpoints are drawn as open markers, because a multiplier +there is a subgradient and comparing two solvers' subgradients is not evidence. +""" +function plot_value_curves(cmp, path::AbstractString) + e = cmp.energy + ok = cmp.acp.solved .& cmp.soc.solved + interior = ok .& .!cmp.acp.endpoint + + top = plot(e[ok], cmp.acp.value[ok]; label = "ACP (true)", lw = 2, + ylabel = "stage cost", legend = :topleft) + plot!(top, e[ok], cmp.soc.value[ok]; label = "SOC-WR (relaxation)", lw = 2, ls = :dash) + + bot = plot(e[interior], cmp.acp.lambda[interior]; label = "∂Q/∂e ACP", lw = 2, + xlabel = "outgoing energy of battery $(cmp.acp.battery) at bus $(cmp.acp.bus) (pu·h)", + ylabel = "marginal value", legend = :topleft) + plot!(bot, e[interior], cmp.soc.lambda[interior]; label = "∂Q/∂e SOC-WR", lw = 2, ls = :dash) + endpoints = ok .& cmp.acp.endpoint + any(endpoints) && scatter!(bot, e[endpoints], cmp.acp.lambda[endpoints]; + label = "endpoint (subgradient)", markershape = :circle, + markercolor = :white) + + fig = plot(top, bot; layout = (2, 1), size = (860, 620), + title = ["stage $(cmp.acp.stage), atom $(cmp.acp.atom)" ""]) + savefig(fig, path) + return path +end + +""" + plot_energy_trajectory(result, case, path) -> String + +Battery energy, charging and discharging over the horizon of one deterministic +equivalent. +""" +function plot_energy_trajectory(result, case::BatteryCase, path::AbstractString) + T = result.horizon + top = plot(; ylabel = "stored energy (pu·h)", legend = :topright) + bot = plot(; ylabel = "power (pu)", xlabel = "stage", legend = :topright) + for b in case.batteries + plot!(top, 1:T, [result.stages[t].energy_out[b.index] for t in 1:T]; + label = "battery $(b.index) @ bus $(b.bus)", lw = 2) + plot!(bot, 1:T, [result.stages[t].p_bat[b.index] for t in 1:T]; + label = "net injection $(b.index)", lw = 2) + end + hline!(bot, [0.0]; label = "", lc = :black, ls = :dot) + fig = plot(top, bot; layout = (2, 1), size = (860, 620)) + savefig(fig, path) + return path +end + +""" + plot_stage_costs(results, labels, path) -> String + +Stage and cumulative cost of one or more deterministic equivalents. + +# Notes +Cumulative cost is plotted as a DIFFERENCE from the first series when more than +one is given: two trajectories whose cumulative costs differ by a fraction of a +percent are one line at any readable scale, and the difference is the quantity. +""" +function plot_stage_costs(results, labels, path::AbstractString) + T = results[1].horizon + top = plot(; ylabel = "stage cost", legend = :topleft) + for (r, l) in zip(results, labels) + plot!(top, 1:T, r.stage_cost; label = l, lw = 2) + end + bot = if length(results) == 1 + plot(1:T, results[1].cumulative_cost; label = labels[1], lw = 2, + ylabel = "cumulative cost", xlabel = "stage", legend = :topleft) + else + p = plot(; ylabel = "cumulative Δcost vs $(labels[1])", xlabel = "stage", + legend = :topleft) + for (r, l) in zip(results[2:end], labels[2:end]) + plot!(p, 1:T, r.cumulative_cost .- results[1].cumulative_cost; label = l, lw = 2) + end + hline!(p, [0.0]; label = "", lc = :black, ls = :dot) + p + end + fig = plot(top, bot; layout = (2, 1), size = (860, 620)) + savefig(fig, path) + return path +end + +""" + plot_prices(result, case, path; buses=nothing) -> String + +Active and reactive nodal prices over the horizon, plus the voltage magnitude at +the same buses. +""" +function plot_prices(result, case::BatteryCase, path::AbstractString; buses = nothing) + ids = buses === nothing ? sort!(unique(b.bus for b in case.batteries)) : collect(Int.(buses)) + T = result.horizon + pa = plot(; ylabel = "active price", legend = :topleft) + pr = plot(; ylabel = "reactive price", legend = :topleft) + pv = plot(; ylabel = "|V| (pu)", xlabel = "stage", legend = :topleft) + for i in ids + plot!(pa, 1:T, [get(result.stages[t].price_active, i, NaN) for t in 1:T]; + label = "bus $i", lw = 2) + plot!(pr, 1:T, [get(result.stages[t].price_reactive, i, NaN) for t in 1:T]; + label = "bus $i", lw = 2) + plot!(pv, 1:T, [result.stages[t].vm[i] for t in 1:T]; label = "bus $i", lw = 2) + end + fig = plot(pa, pr, pv; layout = (3, 1), size = (860, 860)) + savefig(fig, path) + return path +end + +""" + plot_paired_costs(a, b, labels, path) -> String + +A cost-distribution overlay and the paired-difference histogram for two policies +evaluated on the SAME scenarios. + +# Notes +Both panels are drawn because they answer different questions and are routinely +confused. The overlay shows that the two distributions are nearly identical — a +property of the problem, not of the comparison. The paired differences show +whether one policy is systematically cheaper, which the overlay cannot resolve +when the between-scenario spread dwarfs the between-policy difference. +""" +function plot_paired_costs(a::AbstractVector, b::AbstractVector, labels, path::AbstractString) + d = Float64.(a) .- Float64.(b) + top = histogram(Float64.(a); label = labels[1], alpha = 0.5, bins = 30, + ylabel = "scenarios", legend = :topright) + histogram!(top, Float64.(b); label = labels[2], alpha = 0.5, bins = 30) + bot = histogram(d; label = "$(labels[1]) − $(labels[2])", bins = 30, + xlabel = "paired cost difference", ylabel = "scenarios", + legend = :topright) + vline!(bot, [0.0]; label = "", lc = :black, ls = :dot) + vline!(bot, [mean(d)]; label = @sprintf("mean %+.2f", mean(d)), lc = :red, lw = 2) + fig = plot(top, bot; layout = (2, 1), size = (860, 620)) + savefig(fig, path) + return path +end + +""" + plot_demand_support(case, path; stages=nothing) -> String + +The frozen demand support: the total system demand of every atom at every stage. + +# Notes +This is the figure that shows what the uncertainty actually is — how wide the +support is, whether it widens in the stressed window, and whether the +deterministic profile or the multiplier is doing the work. A study that never +plots its own uncertainty tends to discover late that it has almost none. +""" +function plot_demand_support(case::BatteryCase, path::AbstractString; stages = nothing) + ts = stages === nothing ? (1:horizon(case.demand)) : collect(Int.(stages)) + fig = plot(; xlabel = "stage", ylabel = "total active demand (pu)", legend = :topleft) + maxK = maximum(num_atoms(case.demand, t) for t in ts) + for k in 1:maxK + xs = Int[] + ys = Float64[] + for t in ts + k <= num_atoms(case.demand, t) || continue + pd, _ = realized_bus_demand(case, t, k) + push!(xs, t) + push!(ys, sum(values(pd))) + end + plot!(fig, xs, ys; label = "atom $k", lw = 2, seriestype = :steppost) + end + savefig(fig, path) + return path +end diff --git a/examples/BatteryStorageOPF/battery_case.jl b/examples/BatteryStorageOPF/battery_case.jl new file mode 100644 index 0000000..3d3d487 --- /dev/null +++ b/examples/BatteryStorageOPF/battery_case.jl @@ -0,0 +1,1701 @@ +# battery_case.jl +# +# Frozen case contract for the multistage battery-storage AC-OPF study. +# +# This file is the SINGLE source of truth for what "the case" is, and it is +# shipped BYTE-IDENTICALLY in both public engines (DecisionRules.jl, the +# JuMP/PowerModels/SDDP engine, and DecisionRulesExa.jl, the ExaModels/GPU +# engine). Neither engine may re-derive a battery parameter, a demand +# realization, or a scenario index on its own: both read them from here, so a +# disagreement between the engines can never be a disagreement about the case. +# +# It therefore depends only on JSON, SHA, StableRNGs and the Julia standard +# library. In particular it does NOT depend on PowerModels, PGLib or +# Distributions: those are needed to BUILD the frozen artifacts (see +# `battery_demand.jl` and `build_battery_case.jl`, which +# exist only in the JuMP engine), never to READ them. +# +# Artifacts of one case live in one directory: +# +# /network.json the parsed PGLib network, per-unit, verbatim +# /batteries.json battery placement and parameters +# /demand.json the FROZEN finite demand support +# /case_manifest.json units, counts, stage duration and SHA-256s +# +# Every number that both engines must agree on is in one of those four files. +# +# THE DEMAND CONTRACT, stated once. +# The authoring sampler (a `Distribution`, a callable, a regional group model — +# see `battery_demand.jl`) is NOT part of the case. What is frozen, hashed and +# mirrored is its FINITE SUPPORT: for every stage t, a list of joint multiplier +# vectors over the case's loads together with their probabilities. Both SDDP and +# TS-DDR train from those bytes and neither is permitted to resample or +# rediscretize the authoring sampler. That is what makes "the two methods faced +# the same stochastic program" a checkable statement rather than an intention. + +using JSON +using SHA +using StableRNGs +using Printf + +# ───────────────────────────────────────────────────────────────────────────── +# Schema tags +# +# Every artifact carries a schema string. A loader that meets an unknown schema +# FAILS rather than guessing, because a silently-shifted field is exactly the +# class of defect that makes two engines solve two different problems. +# +# The demand artifact is at schema 2: schema 1 carried a single scalar +# multiplier per stage, which cannot express a joint per-load realization. +# ───────────────────────────────────────────────────────────────────────────── + +const BATTERY_NETWORK_SCHEMA = "battery_storage_opf/network/1" +const BATTERY_BATTERY_SCHEMA = "battery_storage_opf/batteries/2" +const BATTERY_DEMAND_SCHEMA = "battery_storage_opf/demand/2" +const BATTERY_MANIFEST_SCHEMA = "battery_storage_opf/manifest/3" + +""" + STAGE_AVAILABILITY_KEY + +Name of the optional generator field that carries a per-stage availability +schedule: `gen[STAGE_AVAILABILITY_KEY][t]` is a nonnegative multiplier applied to +that generator's active and reactive limits at stage `t` by +[`apply_stage_availability!`](@ref). + +# Notes +The schedule lives INSIDE the network table rather than beside it in an artifact +of its own, for one reason: the frozen case hashes `network.json`, so a schedule +carried there is covered by the case digest, travels with the case to every +consumer, and cannot drift out of step with the network it describes. A separate +artifact would have needed its own hash, its own read-back check and its own +statement of which generator each row refers to. + +The network schema is unchanged because the field is OPTIONAL and additive: a +generator that does not carry it is available in every stage, so every case built +before this convention existed still means exactly what it meant then. +""" +const STAGE_AVAILABILITY_KEY = "stage_availability" + +# ───────────────────────────────────────────────────────────────────────────── +# Canonical JSON +# +# `JSON.print` iterates a `Dict` in hash order, so writing the same object twice +# from two processes can produce two different byte strings and destroy the +# point of hashing an artifact. The emitter below sorts object keys and prints +# every scalar through a round-tripping representation, which makes the bytes a +# pure function of the value. +# ───────────────────────────────────────────────────────────────────────────── + +""" + canonical_json(value) -> String + +Serialize `value` to JSON whose bytes depend only on the value, not on +dictionary iteration order or on floating-point printing defaults. + +# Arguments +- `value`: any nesting of `AbstractDict{<:AbstractString}`, `AbstractVector`, + `AbstractString`, `Bool`, `Integer`, `AbstractFloat` and `nothing`. + +# Returns +- A `String` holding the canonical JSON text (2-space indentation, object keys + sorted lexicographically by `isless` on the key strings). + +# Notes +Floats are printed with `Base.print`, which emits the shortest decimal literal +that round-trips through `parse(Float64, ·)`. Reading the emitted text back +with `JSON.parsefile` therefore reproduces the original `Float64` bit pattern, +which is what lets the two engines hash and compare the same artifact. + +Non-finite floats are rejected: JSON has no representation for them, and a +silently emitted `NaN` token would be unparseable by a conforming reader. +""" +function canonical_json(value) + io = IOBuffer() + _canonical_json!(io, value, 0) + return String(take!(io)) +end + +# Recursive canonical writer. `depth` is the current indentation level; the +# emitter never depends on the container's iteration order. +function _canonical_json!(io::IO, value, depth::Int) + pad = " "^depth + pad_in = " "^(depth + 1) + if value === nothing + print(io, "null") + elseif value isa Bool + # Checked before Integer: `Bool <: Integer` in Julia. + print(io, value ? "true" : "false") + elseif value isa Integer + print(io, string(value)) + elseif value isa AbstractFloat + isfinite(value) || error("canonical_json: non-finite float $value has no JSON representation") + # Shortest round-tripping decimal; `1.0` stays `1.0` (never `1`), which + # keeps the emitted type distinguishable from an integer on re-read. + print(io, string(Float64(value))) + elseif value isa AbstractString + _canonical_json_string!(io, value) + elseif value isa AbstractDict + isempty(value) && return print(io, "{}") + # Stringify the keys once, then sort: sorting is what makes the bytes + # order-independent, and going through the pair list avoids re-indexing + # the source dictionary with a converted key. + pairs = sort!([(string(k), v) for (k, v) in value]; by = first) + allunique(first.(pairs)) || + error("canonical_json: dictionary has keys that collide once stringified") + print(io, "{\n") + for (i, (k, v)) in enumerate(pairs) + print(io, pad_in) + _canonical_json_string!(io, k) + print(io, ": ") + _canonical_json!(io, v, depth + 1) + print(io, i == length(pairs) ? "\n" : ",\n") + end + print(io, pad, "}") + elseif value isa AbstractVector + isempty(value) && return print(io, "[]") + print(io, "[\n") + for (i, v) in enumerate(value) + print(io, pad_in) + _canonical_json!(io, v, depth + 1) + print(io, i == length(value) ? "\n" : ",\n") + end + print(io, pad, "]") + else + error("canonical_json: unsupported value of type $(typeof(value))") + end + return nothing +end + +# Minimal RFC 8259 string escaping. +function _canonical_json_string!(io::IO, s::AbstractString) + print(io, '"') + for c in s + if c == '"' + print(io, "\\\"") + elseif c == '\\' + print(io, "\\\\") + elseif c == '\n' + print(io, "\\n") + elseif c == '\r' + print(io, "\\r") + elseif c == '\t' + print(io, "\\t") + elseif c < ' ' + print(io, "\\u", lpad(string(UInt16(c); base = 16), 4, '0')) + else + print(io, c) + end + end + print(io, '"') + return nothing +end + +""" + write_canonical_json(path, value) -> String + +Write `canonical_json(value)` to `path` and return the SHA-256 of the bytes +actually written. + +# Arguments +- `path::AbstractString`: destination file. +- `value`: object accepted by [`canonical_json`](@ref). + +# Returns +- Lowercase hexadecimal SHA-256 digest of the file contents. +""" +function write_canonical_json(path::AbstractString, value) + text = canonical_json(value) + mkpath(dirname(path)) + write(path, text) + return bytes2hex(sha256(text)) +end + +""" + sha256_file(path) -> String + +Lowercase hexadecimal SHA-256 digest of the bytes of `path`. +""" +sha256_file(path::AbstractString) = bytes2hex(sha256(read(path))) + +""" + plain(value) + +Recursively rebuild a parsed-JSON tree out of plain `Dict{String,Any}`, +`Vector{Any}` and scalars. + +# Notes +JSON parsers return their own container types (`JSON.Object`, lazily-typed +arrays). Those satisfy the `AbstractDict`/`AbstractVector` interfaces but not the +CONCRETE types that downstream modelling packages assume when they build typed +lookup tables from a network dictionary, which surfaces as a `convert` error +deep inside a library rather than as a data problem. Normalizing once, at the +boundary where the case is read, keeps that class of failure out of every +consumer. +""" +function plain(value) + if value isa AbstractDict + return Dict{String,Any}(string(k) => plain(v) for (k, v) in value) + elseif value isa AbstractString + return String(value) + elseif value isa AbstractVector + return Any[plain(v) for v in value] + else + return value + end +end + +# ───────────────────────────────────────────────────────────────────────────── +# Battery parameters +# ───────────────────────────────────────────────────────────────────────────── + +""" + BatterySpec + +Parameters of one battery, in the per-unit system of the host network. + +# Fields +- `index::Int`: battery identifier. Identifiers are arbitrary positive integers + and need not be consecutive; every engine keys on this value. +- `bus::Int`: identifier of the bus the battery injects into. Again an + arbitrary network identifier, never a positional index. +- `energy_min::Float64`, `energy_max::Float64`: energy bounds ``\\underline e_b`` + and ``\\overline e_b`` in per-unit-hours (pu·h), i.e. per-unit power sustained + for one hour. +- `energy_initial::Float64`: ``e_{b,0}``, the energy carried into stage 1 (pu·h). +- `charge_max::Float64`, `discharge_max::Float64`: ``\\overline p^{ch}_b`` and + ``\\overline p^{dis}_b`` in per-unit power (pu). +- `charge_efficiency::Float64`, `discharge_efficiency::Float64`: + ``\\eta^{ch}_b, \\eta^{dis}_b \\in (0,1]``. +- `self_discharge::Float64`: ``\\alpha_b \\in (0,1]``, the fraction of stored + energy retained across one stage. +- `throughput_cost::Float64`: ``c^{deg}_b``, the degradation price charged on + ``\\Delta t\\,(p^{ch}+p^{dis})``, in objective units per pu·h. + +# Notes +The state transition these fields parameterize is + +```math +e_{b,t} = \\alpha_b e_{b,t-1} + + \\eta^{ch}_b \\Delta t\\, p^{ch}_{b,t} + - \\frac{\\Delta t}{\\eta^{dis}_b} p^{dis}_{b,t}, +``` + +with ``e_{b,t}`` the END-of-stage energy. That convention is fixed here and is +never re-stated with a different meaning anywhere in either engine. +""" +struct BatterySpec + index::Int + bus::Int + energy_min::Float64 + energy_max::Float64 + energy_initial::Float64 + charge_max::Float64 + discharge_max::Float64 + charge_efficiency::Float64 + discharge_efficiency::Float64 + self_discharge::Float64 + throughput_cost::Float64 +end + +""" + reachable_interval(b::BatterySpec, e_prev, Δt) -> (lower, upper) + +One-stage battery-dynamic reachable interval for the outgoing energy. + +# Arguments +- `b::BatterySpec`: battery parameters. +- `e_prev::Real`: incoming energy ``e_{b,t-1}`` (pu·h). +- `Δt::Real`: stage duration in hours. + +# Returns +- `(lower, upper)`: the closed interval + +```math +\\underline r = \\max\\{\\underline e_b,\\; + \\alpha_b e_{t-1} - \\tfrac{\\Delta t}{\\eta^{dis}_b}\\overline p^{dis}_b\\}, +\\qquad +\\overline r = \\min\\{\\overline e_b,\\; + \\alpha_b e_{t-1} + \\eta^{ch}_b \\Delta t\\, \\overline p^{ch}_b\\}. +``` + +# Notes +Every value in `[lower, upper]` is attained by an admissible +``(p^{ch}, p^{dis})`` pair, because the transition is affine and monotone in +each control and the controls' own boxes are intervals containing 0. This is +the map the strict policy squashes its normalized output into; note that BOTH +endpoints depend on `e_prev` with slope ``\\alpha_b`` wherever the energy bound +is not the binding term, which is why a policy that differentiates through this +map must not treat the endpoints as constants. + +The returned interval is nonempty whenever ``\\underline e_b \\le \\alpha_b +e_{t-1} + \\eta^{ch}\\Delta t \\overline p^{ch}`` and ``\\alpha_b e_{t-1} - +\\Delta t \\overline p^{dis}/\\eta^{dis} \\le \\overline e_b``; with +``\\alpha_b = 1`` and ``e_{t-1} \\in [\\underline e_b, \\overline e_b]`` both +hold, so the interval is nonempty by induction along any trajectory the policy +itself generates. +""" +function reachable_interval(b::BatterySpec, e_prev::Real, Δt::Real) + decayed = b.self_discharge * e_prev + lower = max(b.energy_min, decayed - (Δt / b.discharge_efficiency) * b.discharge_max) + upper = min(b.energy_max, decayed + b.charge_efficiency * Δt * b.charge_max) + return lower, upper +end + +""" + dispatch_for_target(b::BatterySpec, e_prev, e_target, Δt) -> (p_ch, p_dis) + +The charge/discharge pair that realizes `e_target` from `e_prev` in one stage. + +# Arguments +- `b::BatterySpec`, `e_prev::Real`, `e_target::Real`, `Δt::Real`. + +# Returns +- `(p_ch, p_dis)`: nonnegative powers (pu) satisfying the state transition + exactly, with at most one of them nonzero. + +# Notes +Writing ``\\delta := e_{target} - \\alpha_b e_{prev}``, the transition +``\\delta = \\eta^{ch}\\Delta t\\,p^{ch} - (\\Delta t/\\eta^{dis})p^{dis}`` +is solved by + +```math +p^{ch} = \\frac{\\max(\\delta, 0)}{\\eta^{ch}\\Delta t}, +\\qquad +p^{dis} = \\frac{\\eta^{dis}\\max(-\\delta, 0)}{\\Delta t}. +``` + +This is the unique solution with `p_ch * p_dis == 0`; it is admissible exactly +when `e_target` lies in [`reachable_interval`](@ref). It is used to CERTIFY +recourse (given a reachable target, exhibit the controls that hit it) and never +to replace an optimizer's choice inside a stage problem. +""" +function dispatch_for_target(b::BatterySpec, e_prev::Real, e_target::Real, Δt::Real) + δ = e_target - b.self_discharge * e_prev + p_ch = max(δ, 0.0) / (b.charge_efficiency * Δt) + p_dis = b.discharge_efficiency * max(-δ, 0.0) / Δt + return p_ch, p_dis +end + +""" + battery_injection(b::BatterySpec, p_ch, p_dis) -> Float64 + +Active power injected into the network, ``p^{bat} = p^{dis} - p^{ch}`` (pu). + +# Notes +The battery operates at unity power factor: ``q^{bat} \\equiv 0``. Discharging +is a positive injection; charging is a negative one. +""" +battery_injection(::BatterySpec, p_ch::Real, p_dis::Real) = float(p_dis - p_ch) + +""" + RecourseCosts + +Prices of the two-sided physical active-power recourse, in objective units per +pu per stage. + +# Fields +- `deficit::Float64`: ``C^{def}``, price of the nonnegative uncapped injection + ``d_{i,t}`` (unserved load, or the active power a charging target needs and + the grid cannot deliver). +- `surplus::Float64`: ``C^{sur}``, price of the nonnegative uncapped sink + ``s_{i,t}`` (active power a discharging target produces and the grid cannot + absorb). + +# Notes +Both prices must sit far above the most expensive generator so that recourse is +never an economic substitute for dispatch; both must be IDENTICAL in the +PowerModels ACP model, the PowerModels SOC-WR model, the Exa ACP model, and in +both SDDP passes. They travel in the frozen case artifact for exactly that +reason: neither engine gets to choose them. + +`d` and `s` are physical operating recourse, not target slack. They appear only +in the nodal ACTIVE balance; they appear in no battery state equation and in no +strict target equality, and there is no target-slack variable anywhere in the +supported formulation. +""" +struct RecourseCosts + deficit::Float64 + surplus::Float64 +end + +# ───────────────────────────────────────────────────────────────────────────── +# The frozen finite demand support +# +# Demand is the study's ONLY uncertainty. For the original PGLib load values +# p^{d,0}_i and q^{d,0}_i, the realized demand of load i at stage t under atom k +# is +# +# p^d_{i,t,k} = h_{i,t} m^{(k)}_{i,t} p^{d,0}_i +# q^d_{i,t,k} = h_{i,t} m^{(k)}_{i,t} q^{d,0}_i +# +# with h a DETERMINISTIC temporal profile and m the uncertain multiplier. The +# SAME multiplier scales the active and the reactive demand, so every +# realization has the case's own power factor: the uncertainty moves how much +# power is consumed, never what kind. +# +# The multiplier is a JOINT VECTOR over loads, not a scalar and not a collection +# of independent draws — an independent-per-load sampler is one way to produce +# such a vector, never the representation itself. +# ───────────────────────────────────────────────────────────────────────────── + +""" + DemandSupport + +The frozen, stage-major finite support of the demand process. + +# Fields +- `stage_hours::Float64`: ``\\Delta t``, the duration of one stage in hours. +- `horizon::Int`: number of stages the support is frozen for. Stage indices + outside `1:horizon` are an error, never silently wrapped. +- `load_ids::Vector{Int}`: LOAD identifiers, sorted ascending. This vector fixes + the order of every multiplier and profile row and is the only definition of + "component `j`" in this artifact. +- `profile::Matrix{Float64}`: ``h_{i,t}``, size `(length(load_ids), horizon)`, + the deterministic temporal profile. +- `atoms::Vector{Matrix{Float64}}`: `atoms[t]` has size + `(length(load_ids), K_t)`; column `k` is the joint multiplier vector + ``m^{(k)}_{\\cdot,t}``. +- `probabilities::Vector{Vector{Float64}}`: `probabilities[t]` has length + `K_t` and sums to 1. +- `protocol_seed::Int`: seed of the `StableRNG` that generates evaluation + scenario index matrices. +- `source::Dict{String,Any}`: a description of the AUTHORING sampler and of the + discretization that produced these atoms. Documentation, not data: nothing + reads it to build a model. It exists so a frozen support can be traced back to + the sampler it came from. + +# Notes +The support is stage-dependent by construction (`K_t` may differ across stages) +and stagewise independent: an atom index at stage `t` carries no information +about stage `t+1`. That is what both SDDP's backward enumeration and TS-DDR's +trajectory sampling assume, and it is enforced by the representation rather than +by a comment. +""" +struct DemandSupport + stage_hours::Float64 + horizon::Int + load_ids::Vector{Int} + profile::Matrix{Float64} + atoms::Vector{Matrix{Float64}} + probabilities::Vector{Vector{Float64}} + protocol_seed::Int + source::Dict{String,Any} +end + +"Number of loads the support is defined over." +num_loads(s::DemandSupport) = length(s.load_ids) + +"Number of stages the support is frozen for." +horizon(s::DemandSupport) = s.horizon + +""" + profile_period(s::DemandSupport) -> Int + +The cycle length the deterministic profile repeats on, in stages. + +# Notes +Recorded by the freezing operation and used for exactly one purpose: as the +period of the ``(\\sin, \\cos)`` clock feature a policy is given, so that the +position in the daily cycle is encoded without the discontinuity a raw stage +index would introduce at midnight. It is DETERMINISTIC information — knowing the +clock is not knowing the future demand — and it never enters a stage problem. + +Falls back to the frozen horizon when a support was written without one, which +degrades the feature to "position in the horizon" rather than silently claiming +a 24-stage cycle a case may not have. +""" +profile_period(s::DemandSupport) = Int(get(s.source, "profile_period", s.horizon)) + +""" + num_atoms(s::DemandSupport, t) -> Int + +Size ``K_t`` of the finite support at stage `t`. +""" +function num_atoms(s::DemandSupport, t::Integer) + _check_stage(s, t) + return length(s.probabilities[t]) +end + +""" + atom_probabilities(s::DemandSupport, t) -> Vector{Float64} + +The probabilities ``(p_{t,1},\\ldots,p_{t,K_t})`` of stage `t`'s atoms. +""" +function atom_probabilities(s::DemandSupport, t::Integer) + _check_stage(s, t) + return s.probabilities[t] +end + +""" + demand_multipliers(s::DemandSupport, t, atom) -> Vector{Float64} + +The TOTAL per-load multiplier ``h_{i,t}\\,m^{(atom)}_{i,t}`` at stage `t`, in +`load_ids` order. + +# Notes +This is the only place the deterministic profile and the uncertain multiplier are +combined, so the two can never be applied twice or in the wrong order anywhere +downstream. +""" +function demand_multipliers(s::DemandSupport, t::Integer, atom::Integer) + _check_stage(s, t) + K = num_atoms(s, t) + 1 <= atom <= K || + throw(ArgumentError("atom index $atom outside 1:$K at stage $t")) + return @views s.profile[:, t] .* s.atoms[t][:, atom] +end + +# Fail closed on a stage index outside the frozen window. Silently wrapping (as +# a cyclic profile would) is how a horizon change becomes an undetected change +# of problem. +function _check_stage(s::DemandSupport, t::Integer) + 1 <= t <= s.horizon || + throw(ArgumentError("stage $t outside the frozen horizon 1:$(s.horizon)")) + return nothing +end + +""" + support_digest(s::DemandSupport) -> String + +SHA-256 of the frozen support, in a fixed textual encoding. + +# Notes +This digest is what makes "SDDP and TS-DDR consumed the same demand support" a +verifiable claim: both engines recompute it from the bytes they loaded and it is +recorded in the manifest. It covers the stage duration, the horizon, the load +order, the profile, every atom and every probability — that is, everything a +stage problem's demand depends on — and deliberately NOT the `source` +description, which is prose about how the atoms were authored and must not be +able to change the identity of a support. +""" +function support_digest(s::DemandSupport) + io = IOBuffer() + println(io, "battery_storage_opf/support/2") + println(io, s.stage_hours, " ", s.horizon, " ", num_loads(s), " ", s.protocol_seed) + println(io, join(s.load_ids, ",")) + for t in 1:s.horizon + println(io, "t", t, " ", num_atoms(s, t)) + println(io, join((string(x) for x in @views s.profile[:, t]), ",")) + for k in 1:num_atoms(s, t) + println(io, string(s.probabilities[t][k]), " ", + join((string(x) for x in @views s.atoms[t][:, k]), ",")) + end + end + return bytes2hex(sha256(take!(io))) +end + +""" + scenario_index_matrix(s::DemandSupport, num_stages, num_scenarios; + seed=nothing, exclude=nothing) -> Matrix{Int} + +A paired evaluation protocol, reproduced by construction rather than stored. + +# Arguments +- `s::DemandSupport`: supplies the default seed and the per-stage support sizes. +- `num_stages::Integer`, `num_scenarios::Integer`: shape of the protocol. + +# Keywords +- `seed`: `nothing` for the support's own `protocol_seed` — which is the FINAL + protocol's seed — or another integer for an INDEPENDENT protocol. A study needs + at least two: a small screening protocol it may look at while choosing a case + and selecting checkpoints, and a final one no policy was ever selected on. + Taking the screening set as a prefix of the final one would make the final + protocol not fresh, which is the whole property it exists to have. +- `exclude`: an iterable of length-`num_stages` integer columns this protocol may + not contain. Passing the FINAL protocol's columns here is what makes a + screening protocol disjoint from it BY CONSTRUCTION rather than by the + probabilistic argument that a collision is unlikely — see the notes. + +# Returns +- `Matrix{Int}` of size `(num_stages, num_scenarios)`; entry `[t, s]` is the + atom index realized at stage `t` of paired column `s`. + +# Notes +Drawn from `StableRNG(protocol_seed)`, whose stream is fixed across Julia +versions and platforms, so both engines regenerate the identical matrix and only +its SHA-256 needs to be recorded in the manifest. Scenario columns are global +and immutable: column `s` means the same demand path to every policy and to +every shard of an evaluation. + +Draw ORDER is stage-major (all scenarios of stage 1, then all of stage 2, …). +Because each stage's support size ``K_t`` may differ, a scenario-major order +would make the stream position depend on the horizon; stage-major keeps a +protocol of `num_scenarios` columns a prefix of a protocol of more columns only +within a stage, which is the property shard boundaries rely on. + +The exclusion is applied as a REPAIR after that stage-major draw, never as a +per-draw filter, precisely so the stage-major property survives it: the matrix is +drawn exactly as it would have been without `exclude`, then any column that is +banned or that repeats an earlier column of this same matrix is redrawn — in +ascending column order, from the continuation of the same stream, retrying until +the column is admissible. On a support with more paths than columns nothing is +ever redrawn and the matrix is bit-identical to the unexcluded one; the repair +exists so that "screening and final share no scenario" is a structural fact on +a small support too, where a collision is not merely unlikely but certain. +""" +function scenario_index_matrix(s::DemandSupport, num_stages::Integer, num_scenarios::Integer; + seed = nothing, exclude = nothing) + num_stages >= 1 || throw(ArgumentError("num_stages must be positive")) + num_scenarios >= 1 || throw(ArgumentError("num_scenarios must be positive")) + num_stages <= s.horizon || + throw(ArgumentError("protocol asks for $num_stages stages but the support is frozen for $(s.horizon)")) + rng = StableRNG(seed === nothing ? s.protocol_seed : Int(seed)) + m = Matrix{Int}(undef, num_stages, num_scenarios) + for t in 1:num_stages + K = num_atoms(s, t) + for c in 1:num_scenarios + m[t, c] = rand(rng, 1:K) + end + end + exclude === nothing && return m + + # The banned set: the columns the caller forbids, plus — as they are + # accepted — the columns of this protocol itself, so a repaired protocol + # never contains the same scenario twice either. + banned = Set{Vector{Int}}() + for col in exclude + v = Int.(collect(col)) + length(v) == num_stages || + throw(ArgumentError("excluded column has $(length(v)) stages, expected $num_stages")) + push!(banned, v) + end + # The support has ∏_t K_t distinct paths; asking for more admissible columns + # than exist is a specification error, not something to discover by looping. + capacity = prod(BigInt(num_atoms(s, t)) for t in 1:num_stages) + capacity >= length(banned) + num_scenarios || + throw(ArgumentError("the support has $capacity distinct $num_stages-stage paths, " * + "which cannot supply $num_scenarios columns disjoint from " * + "$(length(banned)) excluded ones")) + for c in 1:num_scenarios + col = Int[m[t, c] for t in 1:num_stages] + while col in banned + for t in 1:num_stages + col[t] = rand(rng, 1:num_atoms(s, t)) + end + end + for t in 1:num_stages + m[t, c] = col[t] + end + push!(banned, col) + end + return m +end + +""" + protocol_columns(m::AbstractMatrix{<:Integer}) -> Vector{Vector{Int}} + +The columns of a protocol index matrix, in the form +[`scenario_index_matrix`](@ref) accepts as `exclude`. +""" +protocol_columns(m::AbstractMatrix{<:Integer}) = + [Int[m[t, c] for t in 1:size(m, 1)] for c in 1:size(m, 2)] + +""" + protocol_digest(s::DemandSupport, num_stages, num_scenarios; + seed=nothing, exclude=nothing) -> String + +SHA-256 of the protocol index matrix, in a fixed textual encoding. + +# Notes +The digest, not the matrix, is what the manifest stores. Both engines recompute +the matrix from the seed and must obtain this digest; a mismatch means the two +engines are not evaluating the same scenarios and no comparison between them is +meaningful. + +The digest is of the MATRIX, so it says nothing about how the matrix was +repaired: two calls that produce the same columns hash the same whether or not an +exclusion set was in force. What records the exclusion is the manifest field that +names it, which is also what a reader needs in order to regenerate the matrix. +""" +function protocol_digest(s::DemandSupport, num_stages::Integer, num_scenarios::Integer; + seed = nothing, exclude = nothing) + m = scenario_index_matrix(s, num_stages, num_scenarios; seed = seed, exclude = exclude) + io = IOBuffer() + println(io, "battery_storage_opf/protocol/2") + println(io, num_stages, " ", num_scenarios, " ", + seed === nothing ? s.protocol_seed : Int(seed)) + println(io, join((num_atoms(s, t) for t in 1:num_stages), ",")) + for t in 1:num_stages + println(io, join(view(m, t, :), ",")) + end + return bytes2hex(sha256(take!(io))) +end + +""" + validate_support(s::DemandSupport) -> Nothing + +Fail closed on every property the rest of the study assumes of a frozen support. + +# Notes +Each check corresponds to a way two engines could end up solving different +problems while both reporting success: a probability vector that does not +normalize silently reweights an SDDP backward pass; a negative or non-finite +multiplier produces a load the network was never meant to serve; a load order +that is not sorted-unique makes "component `j`" mean two different things in two +engines. +""" +function validate_support(s::DemandSupport) + s.stage_hours > 0 || error("stage_hours must be positive, got $(s.stage_hours)") + s.horizon >= 1 || error("horizon must be at least 1, got $(s.horizon)") + n = num_loads(s) + n >= 1 || error("a demand support must cover at least one load") + issorted(s.load_ids) && allunique(s.load_ids) || + error("load_ids must be sorted and unique; got $(s.load_ids)") + size(s.profile) == (n, s.horizon) || + error("profile must be $(n)×$(s.horizon), got $(size(s.profile))") + all(isfinite, s.profile) || error("profile has a non-finite entry") + all(>=(0), s.profile) || error("profile has a negative entry") + length(s.atoms) == s.horizon || + error("support has $(length(s.atoms)) stages of atoms but horizon $(s.horizon)") + length(s.probabilities) == s.horizon || + error("support has $(length(s.probabilities)) stages of probabilities but horizon $(s.horizon)") + for t in 1:s.horizon + A = s.atoms[t] + p = s.probabilities[t] + size(A, 1) == n || + error("stage $t atoms have $(size(A, 1)) rows but the support covers $n loads") + size(A, 2) == length(p) || + error("stage $t has $(size(A, 2)) atoms but $(length(p)) probabilities") + length(p) >= 1 || error("stage $t has an empty support") + all(isfinite, A) || error("stage $t has a non-finite multiplier") + all(>=(0), A) || error("stage $t has a negative multiplier") + all(>(0), p) || error("stage $t has a non-positive probability") + isapprox(sum(p), 1.0; atol = 1e-12) || + error("stage $t probabilities sum to $(sum(p)), not 1") + end + return nothing +end + +# ───────────────────────────────────────────────────────────────────────────── +# Battery placement +# +# Placement is authoring-side, but it lives here because the manifest must be +# able to record exactly how a fleet was chosen, and because the eligibility and +# validity rules are properties of the case contract rather than of a script. +# ───────────────────────────────────────────────────────────────────────────── + +""" + PlacementStrategy + +How the buses hosting batteries are chosen. Concrete strategies: +[`ExplicitPlacement`](@ref), [`SampledPlacement`](@ref), +[`CallablePlacement`](@ref). + +# Notes +Every strategy returns bus IDENTIFIERS, never positions, and every strategy is +reproducible from the data recorded in the manifest alone. +""" +abstract type PlacementStrategy end + +""" + ExplicitPlacement(buses) + +Place batteries at the given bus identifiers, in ascending order. + +# Notes +`count` is not consulted: the list IS the fleet size. Duplicates are rejected +rather than deduplicated, because a repeated identifier is far more likely to be +a typo than a request for two batteries at one bus (which is expressed by giving +two `BatterySpec`s at the same bus instead). +""" +struct ExplicitPlacement <: PlacementStrategy + buses::Vector{Int} +end + +ExplicitPlacement(buses) = ExplicitPlacement(sort!(collect(Int.(buses)))) + +""" + SampledPlacement(count; seed, weight=nothing) + +Draw `count` distinct eligible buses without replacement. + +# Fields +- `count::Int`: fleet size. +- `seed::Int`: seed of the `StableRNG` driving the draw. +- `weight`: `nothing` for a uniform draw, or a callable `bus -> Float64` + returning a nonnegative sampling weight (e.g. nominal demand at the bus). + +# Notes +Without replacement means a bus drawn once is removed from the pool, so a +weighted draw is a successive-sampling scheme rather than `count` independent +draws. The candidate pool is SORTED before the first draw, which is what makes +the result independent of dictionary iteration order. +""" +struct SampledPlacement <: PlacementStrategy + count::Int + seed::Int + weight::Any +end + +SampledPlacement(count::Integer; seed::Integer, weight = nothing) = + SampledPlacement(Int(count), Int(seed), weight) + +""" + CallablePlacement(f; name="callable") + +Place batteries at `f(candidates, meta)`, where `candidates` is the sorted vector +of eligible bus identifiers and `meta` is the placement metadata named tuple. + +# Notes +The escape hatch for a placement rule the study does not anticipate — a +graph-theoretic centrality, an optimization, a hand-drawn map. Whatever it +returns is validated exactly as any other strategy's output, and `name` is what +the manifest records in place of a rule it cannot serialize. +""" +struct CallablePlacement <: PlacementStrategy + f::Any + name::String +end + +CallablePlacement(f; name::AbstractString = "callable") = CallablePlacement(f, String(name)) + +""" + load_buses(network) -> Vector{Int} + +Sorted identifiers of buses hosting at least one in-service load. + +# Notes +The default eligible set. A battery at a bus that neither consumes nor generates +is a pure network-support device, which is a different study; restricting to load +buses keeps a randomly placed fleet physically interpretable on any PGLib case. +""" +function load_buses(network::AbstractDict) + out = Set{Int}() + for (_, load) in network["load"] + Int(get(load, "status", 1)) == 0 && continue + push!(out, Int(load["load_bus"])) + end + return sort!(collect(out)) +end + +""" + nominal_load_at_bus(network) -> Dict{Int,Float64} + +Nominal in-service active demand aggregated per bus identifier (pu). +""" +function nominal_load_at_bus(network::AbstractDict) + out = Dict{Int,Float64}() + for (_, load) in network["load"] + Int(get(load, "status", 1)) == 0 && continue + bus = Int(load["load_bus"]) + out[bus] = get(out, bus, 0.0) + Float64(load["pd"]) + end + return out +end + +""" + eligible_buses(network; eligible=nothing) -> Vector{Int} + +The sorted candidate set a sampling placement draws from. + +# Keywords +- `eligible`: `nothing` for the default (in-service load buses), an iterable of + bus identifiers, or a predicate `bus_dict -> Bool` applied to each bus entry of + the network. + +# Notes +Whatever the source, the returned buses are checked to exist, to be in service +and to be connected — a bus with no incident in-service branch cannot host a +battery that participates in the study, and PGLib cases do contain isolated +buses. +""" +function eligible_buses(network::AbstractDict; eligible = nothing) + ids = Set(Int(b["index"]) for (_, b) in network["bus"]) + candidates = if eligible === nothing + load_buses(network) + elseif eligible isa Function + sort!([Int(b["index"]) for (_, b) in network["bus"] if eligible(b)]) + else + sort!(collect(Int.(eligible))) + end + allunique(candidates) || error("eligible bus set contains duplicates") + in_service = Set(Int(b["index"]) for (_, b) in network["bus"] + if Int(get(b, "bus_type", 1)) != 4) + connected = Set{Int}() + for (_, br) in network["branch"] + Int(get(br, "br_status", 1)) == 0 && continue + push!(connected, Int(br["f_bus"])) + push!(connected, Int(br["t_bus"])) + end + for b in candidates + b in ids || error("bus $b is not in the network") + b in in_service || error("bus $b is out of service (bus_type 4)") + b in connected || error("bus $b has no in-service branch and is disconnected") + end + isempty(candidates) && error("no eligible bus remains after filtering") + return candidates +end + +""" + select_battery_buses(network, strategy; eligible=nothing) -> (buses, record) + +Apply a [`PlacementStrategy`](@ref) and return both the chosen buses and the +manifest record describing how they were chosen. + +# Returns +- `buses::Vector{Int}`: sorted, distinct, validated bus identifiers. +- `record::Dict{String,Any}`: strategy name, seed, eligible set, weights and the + selection, in a form the manifest can serialize verbatim. + +# Notes +The eligible set is recorded in FULL, not summarized. "Three buses were drawn +from the load buses" is not reproducible if a later revision of the case adds a +load; the actual pool that was drawn from is. +""" +function select_battery_buses(network::AbstractDict, strategy::PlacementStrategy; + eligible = nothing) + candidates = eligible_buses(network; eligible = eligible) + record = Dict{String,Any}("eligible" => candidates) + + buses = if strategy isa ExplicitPlacement + allunique(strategy.buses) || error("explicit battery buses must be distinct") + for b in strategy.buses + b in candidates || + error("explicit battery bus $b is not in the eligible set") + end + record["strategy"] = "explicit" + copy(strategy.buses) + + elseif strategy isa SampledPlacement + strategy.count >= 1 || throw(ArgumentError("count must be at least 1")) + strategy.count <= length(candidates) || + throw(ArgumentError("cannot place $(strategy.count) batteries on $(length(candidates)) eligible buses")) + rng = StableRNG(strategy.seed) + weights = strategy.weight === nothing ? + fill(1.0, length(candidates)) : + [Float64(strategy.weight(b)) for b in candidates] + all(isfinite, weights) || error("placement weights must be finite") + all(>=(0), weights) || error("placement weights must be nonnegative") + record["strategy"] = strategy.weight === nothing ? "uniform" : "weighted" + record["seed"] = strategy.seed + record["weights"] = weights + sort!(_sample_without_replacement(rng, candidates, weights, strategy.count)) + + elseif strategy isa CallablePlacement + chosen = sort!(collect(Int.(strategy.f(candidates, (network = network, + candidates = candidates))))) + allunique(chosen) || error("callable placement returned duplicate buses") + for b in chosen + b in candidates || error("callable placement returned ineligible bus $b") + end + record["strategy"] = "callable:" * strategy.name + chosen + + else + error("unsupported placement strategy $(typeof(strategy))") + end + + isempty(buses) && error("placement selected no bus") + record["selected"] = buses + return buses, record +end + +""" + _sample_without_replacement(rng, items, weights, count) -> Vector + +Successive weighted sampling without replacement. + +# Notes +At each of `count` rounds the remaining items are sampled with probability +proportional to their weight and the chosen item is removed. With all weights +equal this reduces to a uniform draw without replacement. The implementation +consumes the stream through `rand(rng)` only, so the result depends on the seed +and not on any `Random` API whose behaviour is free to change between Julia +versions. +""" +function _sample_without_replacement(rng, items::AbstractVector, weights::AbstractVector, + count::Integer) + pool = collect(items) + w = collect(Float64.(weights)) + out = eltype(items)[] + for _ in 1:count + total = sum(w) + total > 0 || error("placement weights of the remaining pool sum to zero") + u = rand(rng) * total + acc = 0.0 + j = length(w) + for i in eachindex(w) + acc += w[i] + if u <= acc + j = i + break + end + end + push!(out, pool[j]) + deleteat!(pool, j) + deleteat!(w, j) + end + return out +end + +""" + battery_fleet(network, buses; power, energy_hours, charge_efficiency, + discharge_efficiency, self_discharge, throughput_cost, + initial_fraction) -> (Vector{BatterySpec}, record) + +Give the selected buses their ratings. + +# Arguments +- `buses::AbstractVector{Int}`: the output of [`select_battery_buses`](@ref). + +# Keywords +- `power`: the power rating rule. Either a `Real` in pu applied to every + battery, a `Dict{Int,<:Real}` keyed by bus, or a callable `bus -> Real`. A + callable closing over a `Distribution` and an RNG is how a SAMPLED capacity is + expressed without this file depending on Distributions.jl. +- `energy_hours`: energy rating as hours at full discharge power; same three + forms as `power`. +- `charge_efficiency`, `discharge_efficiency`, `self_discharge`, + `throughput_cost`, `initial_fraction`: same three forms; scalars in practice. +- `reserve_fraction`: the OPERATING BAND. `energy_min = reserve_fraction * + energy_max` and `energy_max` is unchanged, so a nonzero value keeps the battery + off the exact bottom of its box. Physically it is the reserve a real battery is + not allowed to discharge below; numerically it matters more than it sounds, + because at an exact box corner the one-stage reachable interval collapses + against a bound, the transition equality and the energy bound become parallel, + and the resulting near-degenerate face is what defeats a conic interior-point + method on the SOC-WR relaxation. + +# Returns +- The fleet sorted by battery index, and the manifest record of the capacity rule. + +# Notes +Batteries are indexed `1:n` in the order of the (sorted) bus identifiers. Battery +INDEX is an identity, not a position — every engine keys on it — but assigning +them consecutively at construction keeps the frozen artifact readable. + +Every parameter is validated here rather than at read time as well, so a case +that cannot be built is rejected where the rule that produced it is still in +scope. +""" +function battery_fleet(network::AbstractDict, buses::AbstractVector{<:Integer}; + power, + energy_hours, + charge_efficiency = 0.95, + discharge_efficiency = 0.95, + self_discharge = 1.0, + throughput_cost = 0.0, + initial_fraction = 0.5, + reserve_fraction = 0.0) + resolve(rule, bus) = rule isa Function ? Float64(rule(bus)) : + rule isa AbstractDict ? Float64(rule[bus]) : Float64(rule) + + specs = BatterySpec[] + record = Dict{String,Any}("power_pu" => Dict{String,Any}(), + "energy_hours" => Dict{String,Any}()) + for (i, bus) in enumerate(buses) + p = resolve(power, bus) + h = resolve(energy_hours, bus) + ηc = resolve(charge_efficiency, bus) + ηd = resolve(discharge_efficiency, bus) + α = resolve(self_discharge, bus) + c = resolve(throughput_cost, bus) + f0 = resolve(initial_fraction, bus) + rf = resolve(reserve_fraction, bus) + + p > 0 || error("battery at bus $bus: power rating must be positive, got $p") + h > 0 || error("battery at bus $bus: energy duration must be positive, got $h") + 0 < ηc <= 1 || error("battery at bus $bus: charge_efficiency out of (0,1]") + 0 < ηd <= 1 || error("battery at bus $bus: discharge_efficiency out of (0,1]") + 0 < α <= 1 || error("battery at bus $bus: self_discharge out of (0,1]") + c >= 0 || error("battery at bus $bus: throughput_cost must be nonnegative") + 0 <= f0 <= 1 || error("battery at bus $bus: initial_fraction out of [0,1]") + 0 <= rf < 1 || error("battery at bus $bus: reserve_fraction out of [0,1)") + rf <= f0 || error("battery at bus $bus: initial_fraction $f0 is below the reserve $rf") + + e_max = h * p + push!(specs, BatterySpec(i, Int(bus), rf * e_max, e_max, f0 * e_max, p, p, + ηc, ηd, α, c)) + record["power_pu"][string(bus)] = p + record["reserve_fraction"] = rf + record["energy_hours"][string(bus)] = h + end + return specs, record +end + +# ───────────────────────────────────────────────────────────────────────────── +# Case container and I/O +# ───────────────────────────────────────────────────────────────────────────── + +""" + BatteryCase + +Everything both engines need in order to build the same stage problem. + +# Fields +- `dir::String`: directory the artifacts were read from. +- `name::String`: PGLib case name, e.g. `"pglib_opf_case14_ieee"`. +- `network::Dict{String,Any}`: the parsed PGLib network, per-unit, verbatim. +- `batteries::Vector{BatterySpec}`: sorted by battery index. +- `recourse::RecourseCosts`: prices of the two-sided nodal active recourse. +- `demand::DemandSupport`: the frozen finite demand support. +- `manifest::Dict{String,Any}`: the manifest as read from disk. +""" +struct BatteryCase + dir::String + name::String + network::Dict{String,Any} + batteries::Vector{BatterySpec} + recourse::RecourseCosts + demand::DemandSupport + manifest::Dict{String,Any} +end + +"Stage duration ``\\Delta t`` in hours." +stage_hours(c::BatteryCase) = c.demand.stage_hours + +""" + nominal_load_demand(case) -> (pd::Vector{Float64}, qd::Vector{Float64}) + +Nominal active and reactive demand of every load in `case.demand.load_ids` +order (pu). + +# Notes +The support's load order — not the network dictionary's iteration order — is what +indexes every multiplier vector, so it is what indexes the nominal values too. +""" +function nominal_load_demand(case::BatteryCase) + by_id = Dict{Int,Any}(Int(l["index"]) => l for (_, l) in case.network["load"]) + pd = Vector{Float64}(undef, num_loads(case.demand)) + qd = Vector{Float64}(undef, num_loads(case.demand)) + for (j, id) in enumerate(case.demand.load_ids) + load = by_id[id] + pd[j] = Float64(load["pd"]) + qd[j] = Float64(load["qd"]) + end + return pd, qd +end + +""" + nominal_bus_demand(case) -> (pd::Dict{Int,Float64}, qd::Dict{Int,Float64}) + +Nominal active and reactive demand aggregated per BUS identifier (pu). + +# Notes +A bus may host several loads; the network's nodal balance constrains only their +sum, so both engines aggregate to the bus before anything else happens. Buses +with no load appear with an explicit `0.0` so downstream code can index every +bus without a `get` default and its attendant typo risk. +""" +function nominal_bus_demand(case::BatteryCase) + pd = Dict{Int,Float64}(Int(b["index"]) => 0.0 for (_, b) in case.network["bus"]) + qd = Dict{Int,Float64}(Int(b["index"]) => 0.0 for (_, b) in case.network["bus"]) + for (_, load) in case.network["load"] + Int(get(load, "status", 1)) == 0 && continue + bus = Int(load["load_bus"]) + pd[bus] += Float64(load["pd"]) + qd[bus] += Float64(load["qd"]) + end + return pd, qd +end + +""" + terminal_energy_floor(case) -> Union{Nothing,Dict{Int,Float64}} + +The per-battery lower bound on TERMINAL energy the case declares, or `nothing` +when it declares none. + +# Returns +`nothing` for every case whose manifest carries no `"boundary"` section — which +is every case frozen before the boundary convention existed — and otherwise a +map from battery identifier to the floor ``\\underline e^T_b`` in pu·h. + +# Notes +The requirement lives in the CASE, not in the caller. A horizon solve, a policy +layer and a comparator that each rebuilt the floor from a remembered fraction +would be three chances to disagree about the boundary condition, and a boundary +disagreement is indistinguishable in the reported cost from a difference in the +policy. Reading it from one place is what makes "the same terminal condition" +checkable rather than asserted. + +A case that declares the convention has already had its `energy_initial` set to +the same value, so the usual contract is ``e^T_b \\ge e^0_b``; the floor is stored +explicitly rather than derived from `energy_initial` so that a case may declare +a different terminal level than its initial one without a new field. +""" +function terminal_energy_floor(case::BatteryCase) + b = get(case.manifest, "boundary", nothing) + b === nothing && return nothing + raw = get(b, "terminal_energy_min", nothing) + raw === nothing && return nothing + return Dict{Int,Float64}(parse(Int, string(k)) => Float64(v) for (k, v) in raw) +end + +""" + realized_bus_demand(case, t, atom) -> (pd::Dict{Int,Float64}, qd::Dict{Int,Float64}) + +Per-bus demand realized at stage `t` under atom index `atom` (pu). + +# Notes +Each load is scaled by its own total multiplier +``h_{i,t} m^{(atom)}_{i,t}`` and the scaled loads are then aggregated to their +bus. The same multiplier scales active and reactive demand, so the power factor +of every individual load is preserved exactly — which is a stronger statement +than preserving the aggregate power factor at the bus, and is the one the study +claims. + +Loads that are out of service contribute nothing, and buses with no load appear +with `0.0`, so the returned dictionaries cover every bus of the network. +""" +function realized_bus_demand(case::BatteryCase, t::Integer, atom::Integer) + pd = Dict{Int,Float64}(Int(b["index"]) => 0.0 for (_, b) in case.network["bus"]) + qd = Dict{Int,Float64}(Int(b["index"]) => 0.0 for (_, b) in case.network["bus"]) + mult = demand_multipliers(case.demand, t, atom) + by_id = Dict{Int,Any}(Int(l["index"]) => l for (_, l) in case.network["load"]) + for (j, id) in enumerate(case.demand.load_ids) + load = by_id[id] + Int(get(load, "status", 1)) == 0 && continue + bus = Int(load["load_bus"]) + pd[bus] += Float64(load["pd"]) * mult[j] + qd[bus] += Float64(load["qd"]) * mult[j] + end + return pd, qd +end + +""" + demand_path(case, atoms) -> Vector{Tuple{Dict{Int,Float64},Dict{Int,Float64}}} + +Materialize a COMPLETE demand path: the per-bus `(pd, qd)` of every stage of the +atom-index vector `atoms`. + +# Notes +A "demand path" is the object a deterministic-equivalent solve and a +perfect-foresight panel consume; giving it a name here keeps every caller from +re-deriving the stage-to-atom mapping and getting the stage offset wrong. +""" +function demand_path(case::BatteryCase, atoms::AbstractVector{<:Integer}) + return [realized_bus_demand(case, t, atoms[t]) for t in eachindex(atoms)] +end + +""" + write_battery_case(dir; name, network, batteries, recourse, demand, + source_version, placement, protocol_stages, + protocol_scenarios) -> Dict{String,Any} + +Write the four frozen artifacts and return the manifest that was written. + +# Notes +The manifest is written LAST and records the SHA-256 of the three artifacts as +they landed on disk, so a manifest can never describe bytes that were never +written. Every value the two engines must agree on — stage duration, unit +conventions, component counts, the support digest, the protocol digest — is +recorded here rather than recomputed independently on each side. + +Stage duration is recorded once, in hours, and read back by +[`read_battery_case`](@ref) with a fail-closed check. This is the battery +analogue of the hydro `stage_hours`, whose omission once rescaled a whole +study's dynamics by a factor of 168. +""" +function write_battery_case(dir::AbstractString; + name::AbstractString, + network::AbstractDict, + batteries::AbstractVector{BatterySpec}, + recourse::RecourseCosts, + demand::DemandSupport, + source_version::AbstractString, + placement::AbstractDict = Dict{String,Any}(), + protocol_stages::Integer, + protocol_scenarios::Integer, + screening_seed::Union{Nothing,Integer} = nothing, + screening_scenarios::Integer = 0, + boundary::Union{Nothing,AbstractDict} = nothing, + variant::Union{Nothing,AbstractDict} = nothing) + validate_support(demand) + mkpath(dir) + + network_obj = Dict{String,Any}( + "schema" => BATTERY_NETWORK_SCHEMA, + "name" => name, + "source_version" => source_version, + "data" => network, + ) + network_sha = write_canonical_json(joinpath(dir, "network.json"), network_obj) + + batteries_obj = Dict{String,Any}( + "schema" => BATTERY_BATTERY_SCHEMA, + "case" => name, + # The two-sided nodal active recourse is part of the same extension + # layer as the batteries: it is what makes a strict, dynamically + # reachable target admissible under the true network. Freezing its + # prices here is what guarantees both engines and both SDDP passes + # charge for it identically. + "recourse" => Dict{String,Any}( + "deficit_cost" => recourse.deficit, + "surplus_cost" => recourse.surplus, + ), + # How the fleet was chosen, in enough detail to redraw it. + "placement" => Dict{String,Any}(placement), + "batteries" => [Dict{String,Any}( + "index" => b.index, + "bus" => b.bus, + "energy_min" => b.energy_min, + "energy_max" => b.energy_max, + "energy_initial" => b.energy_initial, + "charge_max" => b.charge_max, + "discharge_max" => b.discharge_max, + "charge_efficiency" => b.charge_efficiency, + "discharge_efficiency" => b.discharge_efficiency, + "self_discharge" => b.self_discharge, + "throughput_cost" => b.throughput_cost, + ) for b in batteries], + ) + batteries_sha = write_canonical_json(joinpath(dir, "batteries.json"), batteries_obj) + + demand_obj = Dict{String,Any}( + "schema" => BATTERY_DEMAND_SCHEMA, + "case" => name, + "stage_hours" => demand.stage_hours, + "horizon" => demand.horizon, + "load_ids" => demand.load_ids, + # Stage-major, load-minor: `profile[t][j]` is load `load_ids[j]` at stage + # `t`. Writing it stage-major matches how a stage problem reads it. + "profile" => [Float64[demand.profile[j, t] for j in 1:num_loads(demand)] + for t in 1:demand.horizon], + "atoms" => [[Float64[demand.atoms[t][j, k] for j in 1:num_loads(demand)] + for k in 1:num_atoms(demand, t)] for t in 1:demand.horizon], + "probabilities" => [copy(demand.probabilities[t]) for t in 1:demand.horizon], + "protocol_seed" => demand.protocol_seed, + "source" => Dict{String,Any}(demand.source), + ) + demand_sha = write_canonical_json(joinpath(dir, "demand.json"), demand_obj) + + manifest = Dict{String,Any}( + "schema" => BATTERY_MANIFEST_SCHEMA, + "case" => name, + "source" => Dict{String,Any}("package" => "PGLib.jl", "version" => source_version), + "stage_hours" => demand.stage_hours, + "units" => Dict{String,Any}( + "power" => "per-unit on network baseMVA", + "energy" => "per-unit-hours (pu power sustained for one hour)", + "time" => "hours", + # Every number this study builds, solves and reports is in this one + # unit. No stage objective is rescaled on its way into a solver and + # none is converted on its way out: the model a solver sees carries + # the physical stage cost, so a reported cost, a cut and a multiplier + # are all comparable without any conversion step. + "cost" => "objective units of the PGLib case per hour", + ), + "counts" => Dict{String,Any}( + "bus" => length(network["bus"]), + "gen" => length(network["gen"]), + "branch" => length(network["branch"]), + "load" => length(network["load"]), + "shunt" => length(get(network, "shunt", Dict())), + "battery" => length(batteries), + "horizon" => demand.horizon, + "demand_atom_min" => minimum(num_atoms(demand, t) for t in 1:demand.horizon), + "demand_atom_max" => maximum(num_atoms(demand, t) for t in 1:demand.horizon), + ), + "baseMVA" => Float64(network["baseMVA"]), + "recourse" => Dict{String,Any}( + "deficit_cost" => recourse.deficit, + "surplus_cost" => recourse.surplus, + ), + "support" => Dict{String,Any}( + "sha256" => support_digest(demand), + "source" => Dict{String,Any}(demand.source), + ), + # The FINAL paired protocol. Generated from the support's own seed and + # hashed here; a phase that must not evaluate on it can still record what + # it will be. It is generated FIRST and depends on nothing else, which is + # what keeps it independent of every screening decision. + "protocol" => Dict{String,Any}( + "seed" => demand.protocol_seed, + "num_stages" => protocol_stages, + "num_scenarios" => protocol_scenarios, + "sha256" => protocol_digest(demand, protocol_stages, protocol_scenarios), + ), + # The SCREENING protocol, drawn from an INDEPENDENT seed and repaired + # against the final protocol's columns, so the two panels share no + # scenario BY CONSTRUCTION. Everything a case-selection or + # checkpoint-selection decision may look at comes from here, which is what + # leaves the final protocol fresh. + "screening" => screening_seed === nothing ? nothing : Dict{String,Any}( + "seed" => Int(screening_seed), + "num_stages" => protocol_stages, + "num_scenarios" => Int(screening_scenarios), + "excludes" => "protocol", + "sha256" => protocol_digest(demand, protocol_stages, Int(screening_scenarios); + seed = Int(screening_seed), + exclude = protocol_columns( + scenario_index_matrix(demand, protocol_stages, + protocol_scenarios))), + ), + # OPT-IN boundary convention. A case written without it carries no + # "boundary" key at all, which is what `terminal_energy_floor` reads to + # decide that the case imposes no terminal requirement — so every case + # frozen before this field existed keeps its free-terminal behaviour + # without a version check anywhere. + "boundary" => boundary === nothing ? nothing : Dict{String,Any}(boundary), + # OPT-IN provenance of a derived case: which recipe produced it, from + # which base case, and what it changed. Documentation, not data. + "variant" => variant === nothing ? nothing : Dict{String,Any}(variant), + "artifacts" => Dict{String,Any}( + "network.json" => network_sha, + "batteries.json" => batteries_sha, + "demand.json" => demand_sha, + ), + ) + write_canonical_json(joinpath(dir, "case_manifest.json"), manifest) + return manifest +end + +""" + read_battery_case(dir; verify=true) -> BatteryCase + +Read a frozen case from `dir`. + +# Keywords +- `verify::Bool`: when `true` (the default) every artifact hash, schema tag, + stage duration, support digest and protocol digest recorded in the manifest is + re-checked against the bytes on disk before anything is returned. + +# Notes +Verification is on by default and failures are ERRORS, never warnings: an +engine that proceeds on a case it could not verify is producing numbers that +cannot be compared with the other engine's. +""" +function read_battery_case(dir::AbstractString; verify::Bool = true) + manifest_path = joinpath(dir, "case_manifest.json") + isfile(manifest_path) || error("no case_manifest.json in $dir") + manifest = JSON.parsefile(manifest_path) + manifest["schema"] == BATTERY_MANIFEST_SCHEMA || + error("unexpected manifest schema $(manifest["schema"]); expected $BATTERY_MANIFEST_SCHEMA") + + if verify + for (file, want) in manifest["artifacts"] + path = joinpath(dir, file) + isfile(path) || error("case artifact $file missing from $dir") + got = sha256_file(path) + got == want || error("case artifact $file has SHA-256 $got but the manifest records $want") + end + end + + network_obj = JSON.parsefile(joinpath(dir, "network.json")) + network_obj["schema"] == BATTERY_NETWORK_SCHEMA || + error("unexpected network schema $(network_obj["schema"])") + network = plain(network_obj["data"])::Dict{String,Any} + + batteries_obj = JSON.parsefile(joinpath(dir, "batteries.json")) + batteries_obj["schema"] == BATTERY_BATTERY_SCHEMA || + error("unexpected batteries schema $(batteries_obj["schema"])") + batteries = BatterySpec[ + BatterySpec(Int(b["index"]), Int(b["bus"]), + Float64(b["energy_min"]), Float64(b["energy_max"]), + Float64(b["energy_initial"]), + Float64(b["charge_max"]), Float64(b["discharge_max"]), + Float64(b["charge_efficiency"]), Float64(b["discharge_efficiency"]), + Float64(b["self_discharge"]), Float64(b["throughput_cost"])) + for b in batteries_obj["batteries"]] + sort!(batteries; by = b -> b.index) + recourse = RecourseCosts(Float64(batteries_obj["recourse"]["deficit_cost"]), + Float64(batteries_obj["recourse"]["surplus_cost"])) + + demand_obj = JSON.parsefile(joinpath(dir, "demand.json")) + demand_obj["schema"] == BATTERY_DEMAND_SCHEMA || + error("unexpected demand schema $(demand_obj["schema"])") + load_ids = Int.(demand_obj["load_ids"]) + T = Int(demand_obj["horizon"]) + n = length(load_ids) + profile = Matrix{Float64}(undef, n, T) + for t in 1:T + col = Float64.(demand_obj["profile"][t]) + length(col) == n || + error("demand.json profile row $t has $(length(col)) entries, expected $n") + profile[:, t] .= col + end + atoms = Vector{Matrix{Float64}}(undef, T) + probs = Vector{Vector{Float64}}(undef, T) + for t in 1:T + raw = demand_obj["atoms"][t] + K = length(raw) + A = Matrix{Float64}(undef, n, K) + for k in 1:K + col = Float64.(raw[k]) + length(col) == n || + error("demand.json stage $t atom $k has $(length(col)) entries, expected $n") + A[:, k] .= col + end + atoms[t] = A + probs[t] = Float64.(demand_obj["probabilities"][t]) + end + demand = DemandSupport(Float64(demand_obj["stage_hours"]), T, load_ids, + profile, atoms, probs, + Int(demand_obj["protocol_seed"]), + plain(get(demand_obj, "source", Dict{String,Any}()))) + + # ── Fail-closed contract checks ───────────────────────────────────────── + # Each of these has a documented failure mode behind it; none is cosmetic. + validate_support(demand) + demand.stage_hours == Float64(manifest["stage_hours"]) || + error("demand.json stage_hours $(demand.stage_hours) disagrees with manifest $(manifest["stage_hours"])") + network_load_ids = sort!([Int(l["index"]) for (_, l) in network["load"]]) + demand.load_ids == network_load_ids || + error("demand support covers loads $(demand.load_ids) but the network has $(network_load_ids)") + bus_ids = Set(Int(b["index"]) for (_, b) in network["bus"]) + for b in batteries + b.bus in bus_ids || error("battery $(b.index) sits at bus $(b.bus), which is not in the network") + 0 < b.charge_efficiency <= 1 || error("battery $(b.index): charge_efficiency out of (0,1]") + 0 < b.discharge_efficiency <= 1 || error("battery $(b.index): discharge_efficiency out of (0,1]") + 0 < b.self_discharge <= 1 || error("battery $(b.index): self_discharge out of (0,1]") + b.energy_min <= b.energy_initial <= b.energy_max || + error("battery $(b.index): initial energy $(b.energy_initial) outside [$(b.energy_min), $(b.energy_max)]") + b.charge_max >= 0 && b.discharge_max >= 0 || + error("battery $(b.index): negative power rating") + b.throughput_cost >= 0 || error("battery $(b.index): negative throughput cost") + end + allunique(b.index for b in batteries) || error("battery indices are not unique") + recourse.deficit > 0 && recourse.surplus > 0 || + error("recourse prices must be strictly positive; got $(recourse)") + recourse.deficit == Float64(manifest["recourse"]["deficit_cost"]) && + recourse.surplus == Float64(manifest["recourse"]["surplus_cost"]) || + error("batteries.json recourse prices disagree with the manifest") + if verify + want_support = manifest["support"]["sha256"] + got_support = support_digest(demand) + got_support == want_support || + error("regenerated support digest $got_support does not match the manifest's $want_support") + want = manifest["protocol"]["sha256"] + got = protocol_digest(demand, Int(manifest["protocol"]["num_stages"]), + Int(manifest["protocol"]["num_scenarios"])) + got == want || + error("regenerated protocol digest $got does not match the manifest's $want") + scr = get(manifest, "screening", nothing) + if scr !== nothing + wants = scr["sha256"] + # The screening protocol is regenerated exactly as it was written: + # from its own seed, excluding the final protocol's columns when the + # record says it does. Regenerating it without the exclusion would + # silently pass on every case where no repair was needed and fail + # only on the small supports where the property actually bites. + ex = get(scr, "excludes", nothing) == "protocol" ? + protocol_columns(scenario_index_matrix(demand, + Int(manifest["protocol"]["num_stages"]), + Int(manifest["protocol"]["num_scenarios"]))) : + nothing + gots = protocol_digest(demand, Int(scr["num_stages"]), + Int(scr["num_scenarios"]); seed = Int(scr["seed"]), + exclude = ex) + gots == wants || + error("regenerated screening digest $gots does not match the manifest's $wants") + end + end + + return BatteryCase(String(dir), String(manifest["case"]), network, batteries, + recourse, demand, manifest) +end + +""" + _sampler_kind(d) -> String + +A one-line name for a recorded authoring sampler. + +# Notes +The full description is a nested dictionary that can run to thousands of +characters on a stage-dependent regional sampler. It stays in the artifact, where +it belongs; a case summary that scrolled it off the screen would be worse than +useless. +""" +function _sampler_kind(d) + d isa AbstractDict || return "(unrecorded)" + kind = String(get(d, "sampler", "?")) + kind == "product" && return "product(" * + join([_sampler_kind(c) for c in get(d, "components", [])], " × ") * ")" + if kind == "stage" + inner = sort!(unique([_sampler_kind(v) for (_, v) in get(d, "stages", Dict())])) + return "stage[" * join(inner, "|") * "]" + end + kind == "group" && return "group(" * string(length(get(d, "groups", []))) * " regions)" + return kind +end + +""" + describe(case::BatteryCase) -> String + +One-screen human summary of a frozen case: counts, stage duration, battery +ratings and the demand support. +""" +function describe(case::BatteryCase) + s = case.demand + io = IOBuffer() + println(io, "battery case: ", case.name, " (", case.dir, ")") + @printf(io, " buses %d gens %d branches %d loads %d baseMVA %.1f\n", + length(case.network["bus"]), length(case.network["gen"]), + length(case.network["branch"]), length(case.network["load"]), + Float64(case.network["baseMVA"])) + @printf(io, " stage duration %.4f h horizon %d loads in support %d\n", + s.stage_hours, s.horizon, num_loads(s)) + ks = [num_atoms(s, t) for t in 1:s.horizon] + @printf(io, " atoms per stage: min %d max %d support sha %s\n", + minimum(ks), maximum(ks), support_digest(s)[1:16]) + @printf(io, " profile range over stages: [%.4f, %.4f]\n", + minimum(s.profile), maximum(s.profile)) + @printf(io, " authoring sampler: %s (freeze %s, seed %s)\n", + _sampler_kind(get(s.source, "sampler", nothing)), + get(s.source, "method", "?"), string(get(s.source, "seed", "?"))) + @printf(io, " recourse prices: deficit %.1f surplus %.1f (per pu per stage)\n", + case.recourse.deficit, case.recourse.surplus) + for b in case.batteries + @printf(io, " battery %d @ bus %-4d e∈[%.4f, %.4f] e0=%.4f pch≤%.4f pdis≤%.4f η=(%.3f,%.3f) α=%.4f c_deg=%.4f\n", + b.index, b.bus, b.energy_min, b.energy_max, b.energy_initial, + b.charge_max, b.discharge_max, + b.charge_efficiency, b.discharge_efficiency, + b.self_discharge, b.throughput_cost) + end + return String(take!(io)) +end + +""" + evaluation_protocol(case) -> (matrix, kind) + +The protocol a policy may be SELECTED on, and which one it is. + +# Returns +- `matrix::Matrix{Int}`: the `(stages × scenarios)` atom-index matrix. +- `kind::Symbol`: `:screening` when the case declares a screening protocol, + `:sole` when it declares only one protocol and therefore has no final/screening + split at all. + +# Notes +**Selection may never touch the final protocol.** A case of the study's panel +declares two: a large final one, drawn from the support's own `protocol_seed` +and evaluated ONCE after every selection is made, and a small screening one from +an independent seed, repaired against the final one so the two share no scenario +by construction. Regenerating the screening protocol therefore has to regenerate +the final one's COLUMNS as the exclusion set — which is index arithmetic on the +frozen support, exactly what `read_battery_case` already does on every load, and +not an evaluation of anything. + +An earlier revision of this file read `manifest["protocol"]` here. That is the +FINAL protocol, so checkpoint selection was scoring policies on the very panel +that exists to be fresh. The defect was silent — the columns solve, the costs are +finite and the numbers look like a panel — which is why the protocol's kind is +returned beside the matrix, recorded in the checkpoint and printed by the +trainer, rather than left as something a reader has to re-derive. + +`:sole` is reachable only on a case built without a screening protocol at all — +the small correctness fixture this package's regression suite runs on. Every +panel case has the split, so a study run cannot land there. The digest is +re-verified against the manifest either way. +""" +function evaluation_protocol(case::BatteryCase) + scr = get(case.manifest, "screening", nothing) + if scr === nothing + stages = Int(case.manifest["protocol"]["num_stages"]) + scen = Int(case.manifest["protocol"]["num_scenarios"]) + m = scenario_index_matrix(case.demand, stages, scen) + protocol_digest(case.demand, stages, scen) == case.manifest["protocol"]["sha256"] || + error("regenerated protocol digest does not match the manifest's") + return m, :sole + end + ex = get(scr, "excludes", nothing) == "protocol" ? + protocol_columns(scenario_index_matrix(case.demand, + Int(case.manifest["protocol"]["num_stages"]), + Int(case.manifest["protocol"]["num_scenarios"]))) : + nothing + stages, scen, seed = Int(scr["num_stages"]), Int(scr["num_scenarios"]), Int(scr["seed"]) + m = scenario_index_matrix(case.demand, stages, scen; seed = seed, exclude = ex) + protocol_digest(case.demand, stages, scen; seed = seed, exclude = ex) == scr["sha256"] || + error("regenerated screening digest does not match the manifest's") + return m, :screening +end diff --git a/examples/BatteryStorageOPF/battery_demand.jl b/examples/BatteryStorageOPF/battery_demand.jl new file mode 100644 index 0000000..f1e1569 --- /dev/null +++ b/examples/BatteryStorageOPF/battery_demand.jl @@ -0,0 +1,1098 @@ +# battery_demand.jl +# +# The AUTHORING side of the demand process: a composable sampler abstraction, +# the deterministic temporal profile, and the operation that freezes either of +# them into the finite support both methods train from. +# +# THE TWO OBJECTS, AND WHY THEY ARE NOT THE SAME OBJECT. +# +# authoring sampler an arbitrary joint law over per-load multipliers. May be +# continuous, may be a user callable, may be built by +# composing several pieces. Lives here, never in a case. +# frozen support a finite, stage-major list of joint multiplier vectors +# with explicit probabilities. Lives in `demand.json`, is +# hashed, and is mirrored byte-identically into both +# engines (see `battery_case.jl`). +# +# SDDP enumerates a finite support in its backward pass; TS-DDR samples atom +# indices from a finite support in its trajectories. If each were allowed to +# discretize a continuous authoring law on its own, the two would train on two +# different stochastic programs while every report still said "the same demand +# process". `freeze_demand_support` is therefore not a convenience — it is the +# boundary that makes the comparison well posed, and it is the ONLY supported +# path from a sampler to a trainable case. +# +# This file is authoring-only and is NOT mirrored into the Exa engine: that +# engine reads frozen bytes and never touches a sampler. + +using Distributions +using StableRNGs +using Statistics + +@isdefined(BatterySpec) || include(joinpath(@__DIR__, "battery_case.jl")) + +# ───────────────────────────────────────────────────────────────────────────── +# Sampler metadata +# ───────────────────────────────────────────────────────────────────────────── + +""" + demand_meta(network) -> NamedTuple + +The case description every sampler and every user callable receives. + +# Returns +A `NamedTuple` with + +- `load_ids::Vector{Int}` — LOAD identifiers, sorted ascending. This vector fixes + the order of every multiplier vector in this file and in the frozen support. +- `load_bus::Vector{Int}` — the bus of each load, in `load_ids` order. +- `nominal_pd::Vector{Float64}`, `nominal_qd::Vector{Float64}` — the original + PGLib load values ``p^{d,0}_i`` and ``q^{d,0}_i`` (pu). +- `status::Vector{Int}` — the in-service flag of each load. +- `num_loads::Int`, `baseMVA::Float64`. + +# Notes +Sorting the identifiers once, here, is what guarantees that "component `j`" +means the same load to a sampler, to the frozen support and to both engines. +Nothing downstream is permitted to re-sort or re-index; a sampler that wants a +per-bus view builds it from `load_bus`. +""" +function demand_meta(network::AbstractDict) + ids = sort!([Int(l["index"]) for (_, l) in network["load"]]) + by_id = Dict{Int,Any}(Int(l["index"]) => l for (_, l) in network["load"]) + return (load_ids = ids, + load_bus = [Int(by_id[i]["load_bus"]) for i in ids], + nominal_pd = [Float64(by_id[i]["pd"]) for i in ids], + nominal_qd = [Float64(by_id[i]["qd"]) for i in ids], + status = [Int(get(by_id[i], "status", 1)) for i in ids], + num_loads = length(ids), + baseMVA = Float64(network["baseMVA"])) +end + +# ───────────────────────────────────────────────────────────────────────────── +# The sampler interface +# ───────────────────────────────────────────────────────────────────────────── + +""" + DemandSampler + +An authoring law for the uncertain demand multiplier. + +# Interface +A concrete sampler implements + +- `sample_multiplier(s, rng, t, meta) -> Vector{Float64}` — ONE draw of the JOINT + multiplier vector ``m_{\\cdot,t}``, of length `meta.num_loads`, in + `meta.load_ids` order; +- `finite_support(s, t, meta) -> Union{Nothing,Tuple{Matrix{Float64},Vector{Float64}}}` + — the sampler's EXACT finite support at stage `t`, or `nothing` when the law is + continuous or otherwise not finitely supported; +- `describe_sampler(s) -> Dict{String,Any}` — a serializable description. + +# Notes +The fundamental output is a JOINT VECTOR, never a scalar and never an implicit +collection of independent draws. A system-wide scalar law and a set of +independent per-load laws are both expressed as samplers that happen to produce +a particular kind of vector — [`SystemMultiplier`](@ref) and +[`IndependentMultiplier`](@ref) — so correlated regional structure +([`GroupMultiplier`](@ref)) and arbitrary user laws +([`CallableMultiplier`](@ref)) are first-class rather than special cases. + +Samplers draw ONLY through the `rng` they are handed. Nothing here touches the +global random state, which is what makes a seeded freeze bit-reproducible in a +process that has done other random work. +""" +abstract type DemandSampler end + +"Exact finite support of a sampler at stage `t`, or `nothing` when it has none." +finite_support(::DemandSampler, ::Integer, ::NamedTuple) = nothing + +""" + DeterministicMultiplier(value) + +A degenerate sampler: the same multiplier vector at every stage, with +probability 1. + +# Arguments +- `value`: a `Real` applied to every load, a `Vector{<:Real}` in `load_ids` + order, or a `Dict{Int,<:Real}` keyed by LOAD identifier (missing loads take + `1.0`). + +# Notes +This is what makes "run the study with no demand uncertainty" expressible in the +same API rather than as a special code path, and it is the identity element of +[`ProductMultiplier`](@ref). +""" +struct DeterministicMultiplier <: DemandSampler + value::Any +end + +sample_multiplier(s::DeterministicMultiplier, ::Any, ::Integer, meta::NamedTuple) = + _expand_per_load(s.value, meta) + +finite_support(s::DeterministicMultiplier, ::Integer, meta::NamedTuple) = + (reshape(_expand_per_load(s.value, meta), meta.num_loads, 1), [1.0]) + +describe_sampler(s::DeterministicMultiplier) = Dict{String,Any}( + "sampler" => "deterministic", + "value" => s.value isa Real ? Float64(s.value) : + s.value isa AbstractDict ? Dict{String,Any}(string(k) => Float64(v) for (k, v) in s.value) : + Float64.(collect(s.value)), +) + +""" + SystemMultiplier(dist) + +One scalar draw per stage, applied to EVERY load. + +# Arguments +- `dist`: a `Distributions.Distribution` used at every stage, or a + `Vector{<:Distribution}` indexed by stage (stage-dependent law), or a + `Dict{Int,<:Distribution}` keyed by stage. + +# Notes +The system-wide case: demand moves up and down together. This is maximal +correlation across loads and is the sampler under which storage value is purely +temporal, with no locational component from the uncertainty itself. +""" +struct SystemMultiplier <: DemandSampler + dist::Any +end + +function sample_multiplier(s::SystemMultiplier, rng, t::Integer, meta::NamedTuple) + ξ = rand(rng, _stage_dist(s.dist, t)) + return fill(Float64(ξ), meta.num_loads) +end + +function finite_support(s::SystemMultiplier, t::Integer, meta::NamedTuple) + d = _stage_dist(s.dist, t) + d isa DiscreteNonParametric || return nothing + vals = support(d) + probs = Distributions.probs(d) + A = Matrix{Float64}(undef, meta.num_loads, length(vals)) + for (k, v) in enumerate(vals) + A[:, k] .= Float64(v) + end + return A, Float64.(collect(probs)) +end + +describe_sampler(s::SystemMultiplier) = Dict{String,Any}( + "sampler" => "system", + "distribution" => _describe_dist(s.dist), +) + +""" + IndependentMultiplier(dist) + +Independent draws, one per load. + +# Arguments +- `dist`: a `Distribution` used for every load, or a `Dict{Int,<:Distribution}` + keyed by LOAD identifier (missing loads are deterministic 1). + +# Notes +A convenience, not the general interface: independence is a particular joint law, +and it is the one under which aggregate demand concentrates as the case grows, +so on a large system it produces LESS system-level uncertainty than a +system-wide sampler with the same marginal. Choose it when locational demand +diversity is the object of study, not as a default. +""" +struct IndependentMultiplier <: DemandSampler + dist::Any +end + +function sample_multiplier(s::IndependentMultiplier, rng, ::Integer, meta::NamedTuple) + out = Vector{Float64}(undef, meta.num_loads) + for (j, id) in enumerate(meta.load_ids) + d = s.dist isa AbstractDict ? get(s.dist, id, nothing) : s.dist + out[j] = d === nothing ? 1.0 : Float64(rand(rng, d)) + end + return out +end + +describe_sampler(s::IndependentMultiplier) = Dict{String,Any}( + "sampler" => "independent", + "distribution" => _describe_dist(s.dist), +) + +""" + GroupMultiplier(groups, dists; by=:load) + +One draw per GROUP, applied jointly to every load of that group. + +# Arguments +- `groups::Vector{<:AbstractVector{Int}}`: the groups, given as LOAD identifiers + (`by = :load`) or BUS identifiers (`by = :bus`). +- `dists`: one `Distribution` per group, or a single `Distribution` used by every + group. + +# Notes +The regional sampler, and the smallest construction that produces a genuinely +CORRELATED joint vector: loads inside a region move together, regions move +independently of one another. It is what makes a demand process able to stress +one part of a network while leaving another slack — the structure that gives a +battery a locational reason to exist. + +Loads in no group take multiplier 1. Groups must be disjoint: overlapping groups +would make a load's multiplier depend on group order, which is exactly the kind +of silent ordering dependence this file exists to prevent. +""" +struct GroupMultiplier <: DemandSampler + groups::Vector{Vector{Int}} + dists::Any + by::Symbol +end + +function GroupMultiplier(groups, dists; by::Symbol = :load) + by in (:load, :bus) || throw(ArgumentError("by must be :load or :bus, got :$by")) + g = [sort!(collect(Int.(x))) for x in groups] + seen = Set{Int}() + for grp in g, id in grp + id in seen && throw(ArgumentError("group member $id appears in more than one group")) + push!(seen, id) + end + return GroupMultiplier(g, dists, by) +end + +function sample_multiplier(s::GroupMultiplier, rng, ::Integer, meta::NamedTuple) + out = fill(1.0, meta.num_loads) + key = s.by === :load ? meta.load_ids : meta.load_bus + for (gi, grp) in enumerate(s.groups) + d = s.dists isa AbstractVector ? s.dists[gi] : s.dists + ξ = Float64(rand(rng, d)) + members = Set(grp) + for j in 1:meta.num_loads + key[j] in members && (out[j] = ξ) + end + end + return out +end + +function finite_support(s::GroupMultiplier, ::Integer, meta::NamedTuple) + # A product of finitely supported group laws has a finite support: the + # Cartesian product of the groups' atoms with the product probabilities. + dists = [s.dists isa AbstractVector ? s.dists[gi] : s.dists + for gi in eachindex(s.groups)] + all(d -> d isa DiscreteNonParametric, dists) || return nothing + key = s.by === :load ? meta.load_ids : meta.load_bus + members = [Set(grp) for grp in s.groups] + vals = [Float64.(collect(support(d))) for d in dists] + ps = [Float64.(collect(Distributions.probs(d))) for d in dists] + combos = Iterators.product(map(v -> 1:length(v), vals)...) + cols = Vector{Vector{Float64}}() + probs = Float64[] + for c in combos + m = fill(1.0, meta.num_loads) + p = 1.0 + for (gi, ki) in enumerate(c) + p *= ps[gi][ki] + for j in 1:meta.num_loads + key[j] in members[gi] && (m[j] = vals[gi][ki]) + end + end + push!(cols, m) + push!(probs, p) + end + A = Matrix{Float64}(undef, meta.num_loads, length(cols)) + for (k, col) in enumerate(cols) + A[:, k] .= col + end + return A, probs +end + +describe_sampler(s::GroupMultiplier) = Dict{String,Any}( + "sampler" => "group", + "by" => String(s.by), + "groups" => [copy(g) for g in s.groups], + "distribution" => _describe_dist(s.dists), +) + +""" + JointRegionMultiplier(groups, modes, probabilities; by=:load) + +Regions that move JOINTLY: a finite set of joint outcomes over regions, each +with its own probability. + +# Arguments +- `groups`: `Vector{Vector{Int}}`, the regions, as load ids (`by=:load`) or bus + ids (`by=:bus`). Must be disjoint; a load in no region takes multiplier 1. +- `modes`: `(num_regions, K)` matrix, or a `Vector` of `K` length-`num_regions` + vectors. Column `k` is one joint outcome: what EVERY region does together. +- `probabilities`: length `K`, positive, summing to 1. + +# Notes +[`GroupMultiplier`](@ref) draws each region independently, so its joint law is +forced to be a product and its regional correlation is always zero. That is the +wrong shape for a locational storage study: the interesting demand processes are +the ones where regions are ANTI-correlated in some outcomes (one region is +stressed while another is slack — the case for putting storage in a specific +place) and correlated in others (system-wide stress, which no amount of +locational cleverness can hedge). + +Enumerating joint outcomes rather than per-region laws expresses both, exactly, +with no copula and no approximation. It is also CHEAPER: a product of `R` +regions with two atoms each has `2^R` atoms, while the joint form spends atoms +only on the outcomes that carry meaning. Since the number of atoms per stage +multiplies the cost of every SDDP backward pass, that is the difference between +a converged baseline and an unaffordable one. + +Combined with [`StageMultiplier`](@ref) — a different `JointRegionMultiplier` +per stage — the mean, the tail and the inter-region correlation all become +functions of time, which is what forces a policy to hold a genuinely +multi-dimensional strategy rather than one storage schedule replicated +everywhere. +""" +struct JointRegionMultiplier <: DemandSampler + groups::Vector{Vector{Int}} + modes::Matrix{Float64} + probabilities::Vector{Float64} + by::Symbol +end + +function JointRegionMultiplier(groups, modes, probabilities; by::Symbol = :load) + by in (:load, :bus) || throw(ArgumentError("by must be :load or :bus, got :$by")) + g = [sort!(collect(Int.(x))) for x in groups] + seen = Set{Int}() + for grp in g, id in grp + id in seen && throw(ArgumentError("region member $id appears in more than one region")) + push!(seen, id) + end + M = modes isa AbstractMatrix ? Float64.(Matrix(modes)) : + reduce(hcat, [Float64.(collect(m)) for m in modes]) + size(M, 1) == length(g) || + throw(ArgumentError("modes have $(size(M, 1)) rows for $(length(g)) regions")) + p = Float64.(collect(probabilities)) + size(M, 2) == length(p) || + throw(ArgumentError("$(size(M, 2)) modes but $(length(p)) probabilities")) + isapprox(sum(p), 1.0; atol = 1e-12) || + throw(ArgumentError("mode probabilities sum to $(sum(p)), not 1")) + all(>(0), p) || throw(ArgumentError("mode probabilities must be positive")) + all(>(0), M) || throw(ArgumentError("mode multipliers must be positive")) + return JointRegionMultiplier(g, M, p, by) +end + +"The `(num_loads, K)` multiplier matrix this sampler's modes induce." +function _region_atoms(s::JointRegionMultiplier, meta::NamedTuple) + key = s.by === :load ? meta.load_ids : meta.load_bus + members = [Set(grp) for grp in s.groups] + A = fill(1.0, meta.num_loads, size(s.modes, 2)) + for k in 1:size(s.modes, 2), (gi, mem) in enumerate(members) + for j in 1:meta.num_loads + key[j] in mem && (A[j, k] = s.modes[gi, k]) + end + end + return A +end + +finite_support(s::JointRegionMultiplier, ::Integer, meta::NamedTuple) = + (_region_atoms(s, meta), copy(s.probabilities)) + +sample_multiplier(s::JointRegionMultiplier, rng, ::Integer, meta::NamedTuple) = + _region_atoms(s, meta)[:, _sample_index(rng, s.probabilities)] + +describe_sampler(s::JointRegionMultiplier) = Dict{String,Any}( + "sampler" => "joint_region", + "by" => String(s.by), + "groups" => [copy(g) for g in s.groups], + "modes" => [Float64.(s.modes[:, k]) for k in 1:size(s.modes, 2)], + "probabilities" => copy(s.probabilities), +) + +""" + CallableMultiplier(f; name="callable", support=nothing) + +An arbitrary user law: `f(rng, t, meta) -> Vector{Float64}`. + +# Arguments +- `f`: the callable. It receives the RNG (and must draw from nothing else), the + ABSOLUTE stage index, and the [`demand_meta`](@ref) named tuple. + +# Keywords +- `name::AbstractString`: what the manifest records in place of a rule it cannot + serialize. +- `support`: an optional `(t, meta) -> (atoms, probs)` callable declaring an + exact finite support, when the user law happens to have one. + +# Notes +This is the general interface, and every other sampler in this file is a +convenience over it. The returned vector is validated on every draw — length, +finiteness, nonnegativity — because a user callable is the one place a silently +wrong dimension can enter. +""" +struct CallableMultiplier <: DemandSampler + f::Any + name::String + support::Any +end + +CallableMultiplier(f; name::AbstractString = "callable", support = nothing) = + CallableMultiplier(f, String(name), support) + +function sample_multiplier(s::CallableMultiplier, rng, t::Integer, meta::NamedTuple) + v = Float64.(collect(s.f(rng, t, meta))) + length(v) == meta.num_loads || + error("callable sampler \"$(s.name)\" returned $(length(v)) multipliers at stage $t, expected $(meta.num_loads)") + return v +end + +finite_support(s::CallableMultiplier, t::Integer, meta::NamedTuple) = + s.support === nothing ? nothing : s.support(t, meta) + +describe_sampler(s::CallableMultiplier) = Dict{String,Any}( + "sampler" => "callable", "name" => s.name, +) + +""" + FiniteMultiplier(atoms, probabilities) + +An explicitly enumerated finite law, the same at every stage. + +# Arguments +- `atoms`: `Matrix{Float64}` of size `(num_loads, K)` whose columns are the joint + multiplier vectors, or a `Vector{<:Real}` of `K` system-wide scalars. +- `probabilities`: `Vector{Float64}` of length `K`, summing to 1. + +# Notes +Freezing this sampler PRESERVES its support and probabilities exactly: no +resampling, no reweighting, no merging beyond exact duplicates. That is the +contract for a user who has already decided what the scenarios are. +""" +struct FiniteMultiplier <: DemandSampler + atoms::Any + probabilities::Vector{Float64} +end + +function FiniteMultiplier(atoms, probabilities) + p = Float64.(collect(probabilities)) + isapprox(sum(p), 1.0; atol = 1e-12) || + throw(ArgumentError("finite-support probabilities sum to $(sum(p)), not 1")) + all(>(0), p) || throw(ArgumentError("finite-support probabilities must be positive")) + return FiniteMultiplier(atoms, p) +end + +function finite_support(s::FiniteMultiplier, ::Integer, meta::NamedTuple) + A = if s.atoms isa AbstractMatrix + Float64.(Matrix(s.atoms)) + else + # A vector of scalars is a system-wide support: broadcast each atom + # across every load. + v = Float64.(collect(s.atoms)) + [v[k] for _ in 1:meta.num_loads, k in 1:length(v)] + end + size(A, 1) == meta.num_loads || + error("FiniteMultiplier atoms have $(size(A, 1)) rows, expected $(meta.num_loads)") + size(A, 2) == length(s.probabilities) || + error("FiniteMultiplier has $(size(A, 2)) atoms but $(length(s.probabilities)) probabilities") + return A, copy(s.probabilities) +end + +function sample_multiplier(s::FiniteMultiplier, rng, t::Integer, meta::NamedTuple) + A, p = finite_support(s, t, meta) + return A[:, _sample_index(rng, p)] +end + +describe_sampler(s::FiniteMultiplier) = Dict{String,Any}( + "sampler" => "finite", + "num_atoms" => length(s.probabilities), + "probabilities" => copy(s.probabilities), + "atoms" => s.atoms isa AbstractMatrix ? + [Float64.(s.atoms[:, k]) for k in 1:size(s.atoms, 2)] : + Float64.(collect(s.atoms)), +) + +""" + StageMultiplier(by_stage; default=nothing) + +A stage-dependent law: `by_stage[t]` (or `by_stage` keyed by stage) is the +sampler used at stage `t`. + +# Arguments +- `by_stage`: `Vector{<:DemandSampler}` indexed by stage, or + `Dict{Int,<:DemandSampler}` keyed by stage. + +# Keywords +- `default`: the sampler used at stages `by_stage` does not cover. `nothing` + makes an uncovered stage an error. + +# Notes +This is how "quiet early stages, uncertain stressed late stages" is expressed — +the structure the study needs in order for stored energy to have a hedging value +rather than only an arbitrage value. Each stage keeps its own exact support when +its sampler has one, so a stage-dependent finite support survives freezing +unchanged. +""" +struct StageMultiplier <: DemandSampler + by_stage::Any + default::Any +end + +StageMultiplier(by_stage; default = nothing) = StageMultiplier(by_stage, default) + +function _stage_sampler(s::StageMultiplier, t::Integer) + inner = s.by_stage isa AbstractDict ? get(s.by_stage, Int(t), nothing) : + (1 <= t <= length(s.by_stage) ? s.by_stage[t] : nothing) + inner === nothing && (inner = s.default) + inner === nothing && error("StageMultiplier has no sampler for stage $t and no default") + return inner +end + +sample_multiplier(s::StageMultiplier, rng, t::Integer, meta::NamedTuple) = + sample_multiplier(_stage_sampler(s, t), rng, t, meta) + +finite_support(s::StageMultiplier, t::Integer, meta::NamedTuple) = + finite_support(_stage_sampler(s, t), t, meta) + +function describe_sampler(s::StageMultiplier) + entries = if s.by_stage isa AbstractDict + Dict{String,Any}(string(k) => describe_sampler(v) for (k, v) in s.by_stage) + else + Dict{String,Any}(string(t) => describe_sampler(s.by_stage[t]) + for t in eachindex(s.by_stage)) + end + return Dict{String,Any}("sampler" => "stage", "stages" => entries, + "default" => s.default === nothing ? nothing : + describe_sampler(s.default)) +end + +""" + ProductMultiplier(components...) + +The composition operator: the element-wise PRODUCT of several samplers' joint +vectors. + +# Notes +Composition is multiplicative because the multiplier is multiplicative: a +system-wide factor times a regional factor is a demand that is `(system × +region)` times nominal. This is what lets a case be authored as "a common +economy-wide level, plus a regional weather effect, plus an idiosyncratic +per-load term" without any of the three needing to know about the others. + +Components are drawn from the SAME rng in order, so the composite is +reproducible from one seed. When EVERY component has an exact finite support the +product's support is their Cartesian product with product probabilities; +otherwise the product is treated as continuous and freezing discretizes it. +""" +struct ProductMultiplier <: DemandSampler + components::Vector{DemandSampler} +end + +ProductMultiplier(components::DemandSampler...) = ProductMultiplier(collect(components)) + +function sample_multiplier(s::ProductMultiplier, rng, t::Integer, meta::NamedTuple) + out = fill(1.0, meta.num_loads) + for c in s.components + out .*= sample_multiplier(c, rng, t, meta) + end + return out +end + +function finite_support(s::ProductMultiplier, t::Integer, meta::NamedTuple) + parts = [finite_support(c, t, meta) for c in s.components] + any(isnothing, parts) && return nothing + A = parts[1][1] + p = copy(parts[1][2]) + for q in parts[2:end] + B, pb = q + A2 = Matrix{Float64}(undef, meta.num_loads, size(A, 2) * size(B, 2)) + p2 = Vector{Float64}(undef, size(A, 2) * size(B, 2)) + col = 0 + for ka in 1:size(A, 2), kb in 1:size(B, 2) + col += 1 + @views A2[:, col] .= A[:, ka] .* B[:, kb] + p2[col] = p[ka] * pb[kb] + end + A = A2 + p = p2 + end + return A, p +end + +describe_sampler(s::ProductMultiplier) = Dict{String,Any}( + "sampler" => "product", + "components" => [describe_sampler(c) for c in s.components], +) + +# ── Small shared helpers ───────────────────────────────────────────────────── + +"Expand a scalar / vector / load-keyed dictionary into a per-load vector." +function _expand_per_load(value, meta::NamedTuple) + if value isa Real + return fill(Float64(value), meta.num_loads) + elseif value isa AbstractDict + return [Float64(get(value, id, 1.0)) for id in meta.load_ids] + else + v = Float64.(collect(value)) + length(v) == meta.num_loads || + error("per-load value has $(length(v)) entries, expected $(meta.num_loads)") + return v + end +end + +"Select the distribution governing stage `t` from a scalar / vector / dictionary." +function _stage_dist(dist, t::Integer) + dist isa AbstractDict && return dist[Int(t)] + dist isa AbstractVector && return dist[t] + return dist +end + +"Draw an index from an explicit probability vector using only `rand(rng)`." +function _sample_index(rng, p::AbstractVector{Float64}) + u = rand(rng) + acc = 0.0 + for i in eachindex(p) + acc += p[i] + u <= acc && return i + end + return length(p) +end + +"A serializable description of a distribution, a vector of them or a dictionary." +function _describe_dist(d) + d isa AbstractDict && return Dict{String,Any}(string(k) => _describe_dist(v) for (k, v) in d) + d isa AbstractVector && return [_describe_dist(x) for x in d] + return string(d) +end + +# ───────────────────────────────────────────────────────────────────────────── +# Deterministic temporal profile +# ───────────────────────────────────────────────────────────────────────────── + +""" + profile_matrix(profile, meta, horizon) -> Matrix{Float64} + +Materialize the deterministic temporal profile ``h_{i,t}`` as a +`(num_loads × horizon)` matrix. + +# Arguments +- `profile`: one of + - a `Real` — flat, the same at every load and stage; + - a `Vector{<:Real}` of length `horizon` — one value per stage, shared by every + load; + - a `Vector{<:Real}` of a SHORTER length `P` — a cyclic profile of period `P`, + expanded as `profile[mod1(t, P)]`; + - a `Matrix{<:Real}` of size `(num_loads, horizon)` — the fully general case; + - a callable `(load_id, t) -> Real`. + +# Notes +The profile is materialized ONCE, here, and frozen with the case. A cyclic +profile is expanded rather than stored as a period, because a horizon change +must not be able to silently reinterpret which stage is the peak. +""" +function profile_matrix(profile, meta::NamedTuple, horizon::Integer) + n = meta.num_loads + H = Int(horizon) + out = Matrix{Float64}(undef, n, H) + if profile isa Real + fill!(out, Float64(profile)) + elseif profile isa AbstractMatrix + size(profile) == (n, H) || + error("profile matrix must be $(n)×$H, got $(size(profile))") + out .= Float64.(profile) + elseif profile isa AbstractVector + v = Float64.(collect(profile)) + P = length(v) + P >= 1 || error("profile vector must be non-empty") + for t in 1:H + out[:, t] .= v[mod1(t, P)] + end + elseif profile isa Function + for t in 1:H, j in 1:n + out[j, t] = Float64(profile(meta.load_ids[j], t)) + end + else + error("unsupported profile of type $(typeof(profile))") + end + all(isfinite, out) || error("profile has a non-finite entry") + all(>=(0), out) || error("profile has a negative entry") + return out +end + +""" + diurnal_profile(horizon; amplitude=0.12, peak_hour=19, period=24) -> Vector{Float64} + +A smooth single-peak daily profile of mean 1, +``h_t = 1 + a\\cos\\!\\left(2\\pi (t - t_{peak})/P\\right)``. + +# Notes +Returned as a length-`horizon` vector rather than one period, so the caller can +see exactly which stage is the peak. Mean 1 means the profile describes the +SHAPE of demand and the sampler describes its LEVEL; keeping the two +separable is what lets a stressed late window be authored by changing the +sampler alone. +""" +function diurnal_profile(horizon::Integer; amplitude::Real = 0.12, + peak_hour::Integer = 19, period::Integer = 24) + return [1.0 + amplitude * cos(2π * (t - peak_hour) / period) for t in 1:horizon] +end + +# ───────────────────────────────────────────────────────────────────────────── +# Sampling and validation +# ───────────────────────────────────────────────────────────────────────────── + +""" + sample_multiplier_path(sampler, meta, horizon; seed) -> Matrix{Float64} + +Draw one COMPLETE multiplier path: a `(num_loads × horizon)` matrix whose column +`t` is ``m_{\\cdot,t}``. + +# Notes +Uses a `StableRNG(seed)` and nothing else, so the same seed reproduces the same +path on any platform and any Julia version. Every drawn vector is validated +before it is returned. +""" +function sample_multiplier_path(sampler::DemandSampler, meta::NamedTuple, horizon::Integer; + seed::Integer) + rng = StableRNG(seed) + out = Matrix{Float64}(undef, meta.num_loads, Int(horizon)) + for t in 1:Int(horizon) + v = sample_multiplier(sampler, rng, t, meta) + _check_multiplier(v, meta, t) + out[:, t] .= v + end + return out +end + +""" + materialize_demand_path(network, profile, multipliers) -> (pd, qd) + +Turn a multiplier path into the realized per-LOAD demand it describes. + +# Arguments +- `profile::AbstractMatrix`, `multipliers::AbstractMatrix`: both + `(num_loads × horizon)`. + +# Returns +- `pd`, `qd`: `(num_loads × horizon)` matrices in pu. + +# Notes +The same total multiplier scales the active and the reactive value of each load, +so `qd[j,t]/pd[j,t]` equals the load's nominal ratio at every stage of every +path. That invariant is asserted by [`validate_sampler`](@ref); it is the +formal content of "the uncertainty moves how much power is consumed, never what +kind". +""" +function materialize_demand_path(meta::NamedTuple, profile::AbstractMatrix, + multipliers::AbstractMatrix) + size(profile) == size(multipliers) || + error("profile $(size(profile)) and multipliers $(size(multipliers)) disagree") + total = profile .* multipliers + return meta.nominal_pd .* total, meta.nominal_qd .* total +end + +"Fail closed on a multiplier vector a sampler produced." +function _check_multiplier(v::AbstractVector, meta::NamedTuple, t::Integer) + length(v) == meta.num_loads || + error("sampler returned $(length(v)) multipliers at stage $t, expected $(meta.num_loads)") + all(isfinite, v) || error("sampler returned a non-finite multiplier at stage $t") + all(>=(0), v) || error("sampler returned a negative multiplier at stage $t") + return nothing +end + +""" + validate_sampler(sampler, network, horizon; seed=1, draws=32) -> NamedTuple + +Exercise a sampler and check every property the study relies on. + +# Checks +1. **dimension and order** — every draw has one entry per load, in `load_ids` + order; +2. **finiteness and nonnegativity** — no `NaN`, no `Inf`, no negative demand; +3. **exact seeded reproduction** — two runs from the same seed produce + bit-identical paths; +4. **no hidden global RNG state** — perturbing the GLOBAL random stream between + two seeded runs does not change the result; +5. **power-factor preservation** — the realized `qd/pd` ratio of every load + equals its nominal ratio at every stage; +6. **finite support consistency** — where the sampler declares one, its + dimensions and probabilities are valid; +7. **load order preservation** — the metadata's identifiers are sorted and + unique, and no draw reorders them. + +# Returns +A `NamedTuple` of the observed multiplier range and, when declared, the per-stage +support sizes. Failures are errors, not warnings. +""" +function validate_sampler(sampler::DemandSampler, network::AbstractDict, horizon::Integer; + seed::Integer = 1, draws::Integer = 32) + meta = demand_meta(network) + issorted(meta.load_ids) && allunique(meta.load_ids) || + error("demand_meta produced load identifiers that are not sorted and unique") + + a = sample_multiplier_path(sampler, meta, horizon; seed = seed) + # Disturb the global stream: a sampler that reaches for it will now diverge. + rand(1000) + b = sample_multiplier_path(sampler, meta, horizon; seed = seed) + a == b || error("sampler is not reproducible from its seed, or reaches for the global RNG") + + lo, hi = Inf, -Inf + for d in 1:Int(draws) + p = sample_multiplier_path(sampler, meta, horizon; seed = seed + d) + lo = min(lo, minimum(p)) + hi = max(hi, maximum(p)) + pd, qd = materialize_demand_path(meta, profile_matrix(1.0, meta, horizon), p) + for j in 1:meta.num_loads + meta.nominal_pd[j] == 0 && continue + nominal_ratio = meta.nominal_qd[j] / meta.nominal_pd[j] + for t in 1:Int(horizon) + pd[j, t] == 0 && continue + isapprox(qd[j, t] / pd[j, t], nominal_ratio; rtol = 1e-12) || + error("load $(meta.load_ids[j]) lost its power factor at stage $t") + end + end + end + + sizes = Int[] + for t in 1:Int(horizon) + fs = finite_support(sampler, t, meta) + fs === nothing && continue + A, p = fs + size(A, 1) == meta.num_loads || + error("declared support at stage $t has $(size(A, 1)) rows, expected $(meta.num_loads)") + size(A, 2) == length(p) || + error("declared support at stage $t has $(size(A, 2)) atoms but $(length(p)) probabilities") + isapprox(sum(p), 1.0; atol = 1e-12) || + error("declared support at stage $t has probabilities summing to $(sum(p))") + all(>(0), p) || error("declared support at stage $t has a non-positive probability") + push!(sizes, size(A, 2)) + end + + return (multiplier_min = lo, multiplier_max = hi, + support_sizes = isempty(sizes) ? nothing : sizes, + num_loads = meta.num_loads) +end + +# ───────────────────────────────────────────────────────────────────────────── +# Freezing +# ───────────────────────────────────────────────────────────────────────────── + +""" + freeze_demand_support(sampler, network, horizon; seed, atoms_per_stage, + method=:auto, profile=1.0, protocol_seed=seed, + stage_hours=1.0, merge_duplicates=true) + -> DemandSupport + +Turn an authoring sampler into the FROZEN finite support both methods train from. + +# Arguments +- `sampler::DemandSampler`: the authoring law. +- `network::AbstractDict`: the parsed PGLib network (supplies the load order). +- `horizon::Integer`: number of stages to freeze. + +# Keywords +- `seed::Integer`: seed of the `StableRNG` used by an empirical discretization. +- `atoms_per_stage::Integer`: ``K_t`` for an empirical discretization. Ignored by + an exact one. +- `method::Symbol`: `:auto` (exact where the sampler declares a support, + empirical elsewhere), `:exact` (require a declared support at every stage) or + `:empirical` (discretize even a declared support — for a deliberate + sub-sampling of a large exact support). +- `profile`: anything [`profile_matrix`](@ref) accepts. +- `protocol_seed::Integer`: seed of the evaluation protocol. +- `stage_hours::Real`: ``\\Delta t``. +- `profile_period::Integer`: the cycle length the deterministic profile repeats + on, recorded for [`profile_period`](@ref). It is a POLICY FEATURE only and + enters no stage problem; the profile itself is materialized stage by stage and + is not reconstructed from it. +- `merge_duplicates::Bool`: merge EXACTLY equal atoms within a stage, summing + their probabilities. + +# Returns +- A validated [`DemandSupport`](@ref), ready to be written into a case. + +# Notes +**Exact preservation.** For a sampler that declares a finite support the atoms +and probabilities are carried through unchanged — no reweighting and no +resampling — because a user who enumerated their scenarios has already decided +what the stochastic program is. + +**Empirical discretization.** For a continuous or general sampler the default is +the most transparent estimator there is: draw `atoms_per_stage` independent joint +vectors per stage from `StableRNG(seed)` and weight them equally. It converges to +the authoring law, it is reproducible from `(seed, atoms_per_stage)` alone, and +it makes no claim about matching moments. A user who wants a different +discretizer declares one by giving their sampler a `support` callable; there is +no hidden quadrature rule. + +**Duplicate merging** is by EXACT equality of the multiplier vector, so it only +ever collapses atoms that are the same point of the support — typically a +discrete sampler drawn empirically. Two atoms that differ in the last bit are +two atoms. + +The `source` record written into the support carries the sampler description, +the seed, the atom count, the method and the profile summary, so a frozen support +can always be traced back to the law it approximates. +""" +function freeze_demand_support(sampler::DemandSampler, network::AbstractDict, + horizon::Integer; + seed::Integer, + atoms_per_stage::Integer = 1, + method::Symbol = :auto, + profile = 1.0, + protocol_seed::Integer = seed, + stage_hours::Real = 1.0, + profile_period::Integer = 24, + merge_duplicates::Bool = true) + method in (:auto, :exact, :empirical) || + throw(ArgumentError("method must be :auto, :exact or :empirical, got :$method")) + meta = demand_meta(network) + H = Int(horizon) + H >= 1 || throw(ArgumentError("horizon must be at least 1")) + prof = profile_matrix(profile, meta, H) + + rng = StableRNG(seed) + atoms = Vector{Matrix{Float64}}(undef, H) + probs = Vector{Vector{Float64}}(undef, H) + exact_stages = Int[] + + for t in 1:H + declared = method === :empirical ? nothing : finite_support(sampler, t, meta) + if declared === nothing + method === :exact && + error("method = :exact but the sampler declares no finite support at stage $t") + atoms_per_stage >= 1 || + throw(ArgumentError("atoms_per_stage must be at least 1 for an empirical freeze")) + A = Matrix{Float64}(undef, meta.num_loads, Int(atoms_per_stage)) + for k in 1:Int(atoms_per_stage) + v = sample_multiplier(sampler, rng, t, meta) + _check_multiplier(v, meta, t) + A[:, k] .= v + end + p = fill(1.0 / Int(atoms_per_stage), Int(atoms_per_stage)) + atoms[t], probs[t] = A, p + else + A, p = declared + A = Float64.(Matrix(A)) + p = Float64.(collect(p)) + size(A, 1) == meta.num_loads || + error("declared support at stage $t has $(size(A, 1)) rows, expected $(meta.num_loads)") + size(A, 2) == length(p) || + error("declared support at stage $t has $(size(A, 2)) atoms but $(length(p)) probabilities") + for k in 1:size(A, 2) + _check_multiplier(view(A, :, k), meta, t) + end + atoms[t], probs[t] = A, p + push!(exact_stages, t) + end + if merge_duplicates + atoms[t], probs[t] = _merge_duplicate_atoms(atoms[t], probs[t]) + end + # Renormalize only against accumulated floating-point error, never to + # repair a support that does not sum to 1 in the first place. + s = sum(probs[t]) + isapprox(s, 1.0; atol = 1e-9) || + error("stage $t probabilities sum to $s, not 1") + probs[t] ./= s + end + + source = Dict{String,Any}( + "sampler" => describe_sampler(sampler), + "method" => String(method), + "seed" => Int(seed), + "atoms_per_stage" => Int(atoms_per_stage), + "exact_stages" => exact_stages, + "merge_duplicates" => merge_duplicates, + "profile_period" => Int(profile_period), + "profile_min" => minimum(prof), + "profile_max" => maximum(prof), + "horizon" => H, + ) + + support = DemandSupport(Float64(stage_hours), H, copy(meta.load_ids), prof, + atoms, probs, Int(protocol_seed), source) + validate_support(support) + return support +end + +""" + _merge_duplicate_atoms(A, p) -> (A', p') + +Collapse columns of `A` that are EXACTLY equal, summing their probabilities. + +# Notes +First occurrence wins the position, so the order of the surviving atoms is the +order they were generated in — which keeps an empirically frozen support's bytes +a pure function of the seed. +""" +function _merge_duplicate_atoms(A::Matrix{Float64}, p::Vector{Float64}) + seen = Dict{Vector{Float64},Int}() + cols = Vector{Vector{Float64}}() + out = Float64[] + for k in 1:size(A, 2) + col = A[:, k] + j = get(seen, col, 0) + if j == 0 + push!(cols, col) + push!(out, p[k]) + seen[col] = length(cols) + else + out[j] += p[k] + end + end + B = Matrix{Float64}(undef, size(A, 1), length(cols)) + for (k, col) in enumerate(cols) + B[:, k] .= col + end + return B, out +end + +# ───────────────────────────────────────────────────────────────────────────── +# Serialization of an authoring sampler +# ───────────────────────────────────────────────────────────────────────────── + +""" + sampler_to_dict(sampler) -> Dict{String,Any} + +Serialize a sampler to a plain dictionary (the same description the frozen +support records). +""" +sampler_to_dict(s::DemandSampler) = describe_sampler(s) + +""" + sampler_from_dict(d) -> DemandSampler + +Reconstruct a sampler from [`sampler_to_dict`](@ref). + +# Notes +Every built-in sampler round-trips. A [`CallableMultiplier`](@ref) cannot: a +Julia closure has no serialization, and inventing one that reconstructs "some +callable with the right name" would be worse than failing, because the +reconstructed object would silently be a different law. The frozen SUPPORT is +what carries a callable-authored case forward, which is exactly why the support +rather than the sampler is the thing that is frozen. +""" +function sampler_from_dict(d::AbstractDict) + kind = String(d["sampler"]) + if kind == "deterministic" + v = d["value"] + return DeterministicMultiplier(v isa AbstractDict ? + Dict(parse(Int, k) => Float64(x) for (k, x) in v) : + v isa AbstractVector ? Float64.(v) : Float64(v)) + elseif kind == "finite" + atoms = d["atoms"] + A = atoms isa AbstractVector && !isempty(atoms) && atoms[1] isa AbstractVector ? + reduce(hcat, [Float64.(a) for a in atoms]) : Float64.(collect(atoms)) + return FiniteMultiplier(A, Float64.(d["probabilities"])) + elseif kind == "joint_region" + # Round-trips exactly: unlike the group/system samplers this one carries + # plain numbers, not a Distributions.jl object. + return JointRegionMultiplier([Int.(g) for g in d["groups"]], + [Float64.(m) for m in d["modes"]], + Float64.(d["probabilities"]); + by = Symbol(d["by"])) + elseif kind == "product" + return ProductMultiplier([sampler_from_dict(c) for c in d["components"]]) + elseif kind == "stage" + stages = Dict{Int,DemandSampler}(parse(Int, k) => sampler_from_dict(v) + for (k, v) in d["stages"]) + default = d["default"] === nothing ? nothing : sampler_from_dict(d["default"]) + return StageMultiplier(stages; default = default) + elseif kind in ("system", "independent", "group") + error("sampler_from_dict: \"$kind\" carries a Distributions.jl object, which is " * + "described but not serialized; rebuild it in code, or use the frozen support") + elseif kind == "callable" + error("sampler_from_dict: a callable sampler cannot be reconstructed; " * + "the frozen support is what carries such a case forward") + else + error("sampler_from_dict: unknown sampler kind \"$kind\"") + end +end diff --git a/examples/BatteryStorageOPF/battery_diagnostics.jl b/examples/BatteryStorageOPF/battery_diagnostics.jl new file mode 100644 index 0000000..2d946ed --- /dev/null +++ b/examples/BatteryStorageOPF/battery_diagnostics.jl @@ -0,0 +1,910 @@ +# battery_diagnostics.jl +# +# The diagnostic toolkit: what a user runs on a case BEFORE training anything. +# +# Everything here is built out of the production builders in +# `battery_powermodels.jl`. There is no second network model, no duplicated +# balance equation and no separate cost accounting: a diagnostic that measures a +# different model from the one the study trains on measures nothing about the +# study. The only thing these functions add is which variables are pinned and +# which are free. +# +# The five tools, and the question each answers: +# +# sample_incoming_energy what states should I probe from? +# targetless_probe what does the network do at this state, right now? +# energy_value_curve what is the stored energy WORTH here, and do the +# true model and the relaxation disagree about it? +# deterministic_equivalent what would a clairvoyant operator have done? +# perfect_foresight_panel how much headroom is there, over a protocol? +# +# The three are deliberately ordered from myopic to clairvoyant, because the +# first is the one that is easiest to over-read: a targetless ONE-STAGE solve is +# myopic and will rationally empty a battery, since nothing in it prices the +# energy it leaves behind. It diagnoses this stage's physics. It does not, by +# itself, measure the value of preserving energy — that is what +# `energy_value_curve` is for. + +using JuMP +using PowerModels +using StableRNGs +using Statistics +using Printf +import MathOptInterface as MOI + +@isdefined(BatterySpecification) || include(joinpath(@__DIR__, "battery_powermodels.jl")) + +# ───────────────────────────────────────────────────────────────────────────── +# B1 — incoming-energy sampling +# ───────────────────────────────────────────────────────────────────────────── + +""" + sample_incoming_energy(case; kind=:fixed, level=0.5, dist=nothing, + callable=nothing, explicit=nothing, seed=1, batch=1) + -> Vector{Dict{Int,Float64}} + +Reproducibly sample incoming battery-energy vectors inside their own bounds. + +# Keywords +- `kind::Symbol`: one of + - `:fixed` — every battery at the normalized state `level`; + - `:uniform` — independent uniform normalized states in `[0, 1]`; + - `:distribution` — independent draws of the NORMALIZED state from `dist` + (anything supporting `rand(rng, dist)`), clamped into `[0, 1]`; + - `:callable` — `callable(rng, battery, meta) -> normalized state`; + - `:explicit` — `explicit` is a vector of `Dict{Int,Float64}` energy vectors, + validated and returned. +- `level::Real`: the normalized state used by `:fixed`. +- `seed::Integer`: seed of the `StableRNG`. +- `batch::Integer`: how many vectors to draw. + +# Returns +- A vector of `batch` dictionaries keyed by BATTERY identifier, in pu·h. + +# Notes +The NORMALIZED state ``u_b \\in [0,1]`` maps to energy by +``e_b = \\underline e_b + u_b(\\overline e_b - \\underline e_b)``, so a sampler +expressed in normalized terms transfers unchanged to a case whose batteries have +different ratings. Every resulting energy is validated against its own bounds +before being returned. + +This samples DIAGNOSTIC INITIAL STATES only. It is not the demand process and it +plays no part in training or in evaluation; a state drawn here is a place to +stand while measuring the network, not a scenario. +""" +function sample_incoming_energy(case::BatteryCase; + kind::Symbol = :fixed, + level::Real = 0.5, + dist = nothing, + callable = nothing, + explicit = nothing, + seed::Integer = 1, + batch::Integer = 1) + kind in (:fixed, :uniform, :distribution, :callable, :explicit) || + throw(ArgumentError("kind must be :fixed, :uniform, :distribution, :callable or :explicit")) + rng = StableRNG(seed) + out = Dict{Int,Float64}[] + + if kind === :explicit + explicit === nothing && throw(ArgumentError(":explicit requires the energy vectors")) + for e in explicit + push!(out, Dict{Int,Float64}(Int(k) => Float64(v) for (k, v) in e)) + end + else + for _ in 1:Int(batch) + e = Dict{Int,Float64}() + for b in case.batteries + u = if kind === :fixed + Float64(level) + elseif kind === :uniform + rand(rng) + elseif kind === :distribution + dist === nothing && throw(ArgumentError(":distribution requires `dist`")) + clamp(Float64(rand(rng, dist)), 0.0, 1.0) + else + callable === nothing && throw(ArgumentError(":callable requires `callable`")) + Float64(callable(rng, b, (case = case,))) + end + e[b.index] = b.energy_min + u * (b.energy_max - b.energy_min) + end + push!(out, e) + end + end + + byid = Dict(b.index => b for b in case.batteries) + for e in out + Set(keys(e)) == Set(keys(byid)) || + error("incoming-energy vector covers batteries $(sort!(collect(keys(e)))), expected $(sort!(collect(keys(byid))))") + for (k, v) in e + b = byid[k] + isfinite(v) || error("battery $k: incoming energy is not finite") + b.energy_min - 1e-12 <= v <= b.energy_max + 1e-12 || + error("battery $k: incoming energy $v outside [$(b.energy_min), $(b.energy_max)]") + end + end + return out +end + +# ───────────────────────────────────────────────────────────────────────────── +# B2 — the single-stage targetless probe +# ───────────────────────────────────────────────────────────────────────────── + +""" + targetless_probe(case, model_type; energy_in, stage, atom, + optimizer=nothing, pins=nothing, silent=true) -> NamedTuple + +Solve ONE stage with the battery's charge, discharge and OUTGOING energy all +optimized, and return everything measurable about the result. + +# Arguments +- `case::BatteryCase`, `model_type::Type`: `PowerModels.ACPPowerModel` or + `PowerModels.SOCWRConicPowerModel`. + +# Keywords +- `energy_in::AbstractDict`: incoming energy per battery identifier (pu·h). +- `stage::Integer`, `atom::Integer`: which demand realization to impose. +- `optimizer`: defaults to [`acp_optimizer`](@ref) / [`socwr_optimizer`](@ref) + according to `model_type`. +- `pins::Union{Nothing,AbstractDict}`: outgoing energies to PIN by a hard + equality, per battery identifier. `nothing` leaves every battery free — the + targetless probe proper. Any battery not listed stays free. + +# Returns +A `NamedTuple` extending [`extract_stage_solution`](@ref) with + +- `residuals` — the engine-neutral physical residuals of the reported solution; +- `binding` — the binding and nearly binding limits; +- `simultaneous` — ``\\min(p^{ch}_b, p^{dis}_b)`` per battery, which is zero + exactly when the solution does not charge and discharge at once; +- `energy_in_dual` — ``\\partial Q/\\partial e_{b}^{in}``, the marginal value of + the energy the stage INHERITED; +- `pin_dual` — the multiplier of each pinned outgoing energy, i.e. + ``\\partial Q/\\partial e_b^{out}``. + +# Notes +Pinning is done with the same hard equality the strict formulation uses, so when +`pins` covers every battery the model built here is the strict stage problem. +The regression suite asserts exactly that, which is what lets one diagnostic +path serve both the free and the fixed probe without a second formulation. + +**A targetless one-stage solve is myopic.** Nothing in it prices the energy left +in a battery at the end of the stage, so it will rationally discharge as much as +is useful and leave the battery empty. That is not a defect and it is not a +finding: it is the definition of a one-stage problem. Read it for the physics — +what the network could deliver, what bound stopped it, what the energy the stage +INHERITED was worth — and read [`energy_value_curve`](@ref) for the value of what +is left behind. +""" +function targetless_probe(case::BatteryCase, model_type::Type; + energy_in::AbstractDict, + stage::Integer, + atom::Integer, + optimizer = nothing, + pins::Union{Nothing,AbstractDict} = nothing, + silent::Bool = true) + opt = optimizer === nothing ? _default_optimizer(model_type) : optimizer + pm = battery_stage_model(case, model_type; + optimizer = opt, + stage = stage, atom = atom, mode = :free, + state_in = Dict(Int(k) => Float64(v) for (k, v) in energy_in)) + assert_powermodels_provenance(pm, model_type) + + pin_con = Dict{Int,Any}() + if pins !== nothing + bat = pm.ext[:battery] + for (k, v) in pins + haskey(bat[:e_out], Int(k)) || error("pin refers to unknown battery $k") + pin_con[Int(k)] = JuMP.@constraint(pm.model, bat[:e_out][Int(k)] == Float64(v)) + end + end + + silent && JuMP.set_silent(pm.model) + JuMP.optimize!(pm.model) + sol = extract_stage_solution(pm) + + # The pin is written on the model whose objective is the physical stage cost, + # so its multiplier is already in the same units as the value curve a finite + # difference of `sol.objective` would produce — the units the strict target + # dual is reported in too. + pin_dual = Dict{Int,Float64}() + if JuMP.has_duals(pm.model) + for (k, con) in pin_con + pin_dual[k] = JuMP.dual(con) + end + end + + residuals = sol.solved ? + physical_residuals(case.network, case.batteries, stage_hours(case), sol) : + nothing + binding = sol.solved ? binding_constraints(pm, sol) : NamedTuple[] + + return merge(sol, (residuals = residuals, binding = binding, + pin_dual = pin_dual, stage = Int(stage), atom = Int(atom), + model_type = model_type)) +end + +""" + base_feasibility(case, model_type; stage, atom, optimizer=nothing, silent=true) + -> NamedTuple + +Solve the case's network with NO batteries installed, at one stage/atom demand +realization. + +# Returns +The stage solution extended with `residuals`, `binding`, `worst_recourse` and the +nodal-price spread. + +# Notes +This is the admissibility precondition of the entire strict-target construction: +the two-sided physical recourse guarantees that a dynamically reachable battery +target is attainable GIVEN that the base dispatch is feasible at the realized +demand. A case whose base problem already leans on the recourse has no such +guarantee, and every number measured on it would be measuring the recourse price +rather than the network. + +It is also the cheapest possible screen of a candidate benchmark: it answers +"does this system serve this demand at all, and what stops it" before any +battery, any horizon or any policy exists. + +`price_spread` is the ratio of the largest to the smallest active nodal price. It +is the direct locational signal: a system where every bus prices energy the same +cannot reward putting storage in one place rather than another, whatever its +topology looks like. +""" +function base_feasibility(case::BatteryCase, model_type::Type; + stage::Integer, atom::Integer, + optimizer = nothing, silent::Bool = true) + opt = optimizer === nothing ? _default_optimizer(model_type) : optimizer + t0 = time() + pm = battery_stage_model(case, model_type; + optimizer = opt, stage = stage, atom = atom, mode = :none) + assert_powermodels_provenance(pm, model_type) + silent && JuMP.set_silent(pm.model) + JuMP.optimize!(pm.model) + elapsed = time() - t0 + sol = extract_stage_solution(pm) + residuals = sol.solved ? + physical_residuals(case.network, BatterySpec[], stage_hours(case), sol) : + nothing + binding = sol.solved ? binding_constraints(pm, sol) : NamedTuple[] + prices = collect(values(sol.price_active)) + spread = isempty(prices) || minimum(prices) <= 0 ? NaN : + maximum(prices) / minimum(prices) + return merge(sol, (residuals = residuals, binding = binding, + worst_recourse = worst_recourse(sol), + price_spread = spread, elapsed = elapsed, + stage = Int(stage), atom = Int(atom), model_type = model_type)) +end + +"The solver each formulation is solved with unless the caller says otherwise." +function _default_optimizer(model_type::Type) + model_type === PowerModels.ACPPowerModel && return acp_optimizer() + model_type === PowerModels.SOCWRConicPowerModel && return socwr_optimizer() + error("no default optimizer for $model_type; pass one explicitly") +end + +""" + targetless_batch(case, model_type; states, stages, atoms, optimizer) + -> Vector{NamedTuple} + +Run [`targetless_probe`](@ref) over the Cartesian product of incoming-energy +vectors, stages and atoms. + +# Notes +Every combination is retained, including the ones that failed to solve: a batch +that silently drops its failures reports a success rate of 100 % by construction. +""" +function targetless_batch(case::BatteryCase, model_type::Type; + states::AbstractVector{<:AbstractDict}, + stages::AbstractVector{<:Integer}, + atoms::AbstractVector{<:Integer}, + optimizer = nothing) + out = NamedTuple[] + for (si, e) in enumerate(states), t in stages, k in atoms + p = targetless_probe(case, model_type; energy_in = e, stage = t, atom = k, + optimizer = optimizer) + push!(out, merge(p, (state_index = si,))) + end + return out +end + +# ───────────────────────────────────────────────────────────────────────────── +# B3 — the fixed-outgoing-energy value probe +# ───────────────────────────────────────────────────────────────────────────── + +""" + energy_value_curve(case, model_type; battery, energy_in, stage, atom, grid, + hold=nothing, optimizer=nothing, fd_tolerance=5e-3) + -> NamedTuple + +The stage value as a function of the outgoing energy of ONE battery, together +with its multiplier and a finite-difference check of that multiplier. + +# Keywords +- `battery::Integer`: which battery's outgoing energy is swept. +- `energy_in::AbstractDict`: the incoming state to sweep from. +- `grid::AbstractVector{<:Real}`: outgoing energies to solve at. Values outside + the battery's one-stage reachable interval are reported as UNREACHABLE rather + than solved, because a hard equality on an unreachable target is an infeasible + model, not an expensive one. +- `hold`: `nothing` to leave the other batteries free, or a + `Dict{Int,Float64}` pinning them too. +- `fd_tolerance::Real`: relative agreement required between an interior + multiplier and its central finite difference at a SMOOTH point. +- `kink_tolerance::Real`: relative disagreement between the one-sided slopes + above which an interior point is classified nonsmooth. + +# Returns +A `NamedTuple` with + +- `energy::Vector{Float64}`, `value::Vector{Float64}` — the value curve; +- `lambda::Vector{Float64}` — the strict target multiplier at each grid point, + i.e. ``\\partial Q/\\partial e^{out}``; +- `fd::Vector{Float64}` — the central finite difference of `value` at interior + points (`NaN` at the ends); +- `fd_error::Vector{Float64}` — relative disagreement between `lambda` and `fd` + at SMOOTH interior points; +- `fd_ok::Bool` — every smooth interior point agreed within `fd_tolerance`; +- `nonsmooth::Vector{Bool}` — the one-sided slopes disagree by more than + `kink_tolerance`, so the value curve has a kink at this point; +- `bracketed::Vector{Bool}` — at a nonsmooth point, whether the reported + multiplier lies between the two one-sided slopes, which is the correct + statement for a subgradient of a convex value function; +- `endpoint::Vector{Bool}` — the grid point sits at an endpoint of the reachable + interval, where the multiplier is a subgradient and finite differences bracket + rather than match it; +- `solved`, `status`, `residuals`, `worst_recourse`, `binding`, `prices` per + point. + +# Notes +This is the principal tool for locating buses where the SOC-WR relaxation +misprices stored energy. The interesting quantity is not the objective GAP +between the two formulations — a relaxation is below the true model everywhere +and that says nothing about decisions — but the difference in +``\\partial Q/\\partial e^{out}``: the price the two models put on carrying one +more unit of energy out of this stage. A bus where the two models disagree about +that price is a bus where a cut built on the relaxation will steer storage +differently from the truth. + +**Endpoints are not evidence.** At an endpoint of the reachable interval the +target sits on a bound, the multiplier is a subgradient, and two solvers may +report two valid values that differ by orders of magnitude. Interior points are +where a multiplier comparison means something, which is why they are the ones +finite differences are checked against. +""" +function energy_value_curve(case::BatteryCase, model_type::Type; + battery::Integer, + energy_in::AbstractDict, + stage::Integer, + atom::Integer, + grid::AbstractVector{<:Real}, + hold::Union{Nothing,AbstractDict} = nothing, + optimizer = nothing, + fd_tolerance::Real = 5e-3, + kink_tolerance::Real = 5e-2) + b = only(filter(x -> x.index == Int(battery), case.batteries)) + lo, hi = reachable_interval(b, Float64(energy_in[b.index]), stage_hours(case)) + n = length(grid) + + energy = Float64.(collect(grid)) + value = fill(NaN, n) + lambda = fill(NaN, n) + solved = falses(n) + status = Vector{Any}(undef, n) + reachable = falses(n) + endpoint = falses(n) + recourse_used = fill(NaN, n) + residuals = Vector{Any}(undef, n) + binding = Vector{Any}(undef, n) + prices = Vector{Any}(undef, n) + + for (i, e) in enumerate(energy) + # A tolerance on reachability, not on feasibility: the interval endpoints + # are computed in floating point and a grid built from them would + # otherwise fall a rounding error outside itself. + reachable[i] = lo - 1e-9 <= e <= hi + 1e-9 + endpoint[i] = reachable[i] && (abs(e - lo) <= 1e-9 || abs(e - hi) <= 1e-9) + reachable[i] || (status[i] = :unreachable; residuals[i] = nothing; + binding[i] = NamedTuple[]; prices[i] = nothing; continue) + + pins = Dict{Int,Float64}(b.index => clamp(e, lo, hi)) + hold === nothing || for (k, v) in hold + Int(k) == b.index && continue + pins[Int(k)] = Float64(v) + end + p = targetless_probe(case, model_type; energy_in = energy_in, stage = stage, + atom = atom, optimizer = optimizer, pins = pins) + status[i] = p.status + solved[i] = p.solved + residuals[i] = p.residuals + binding[i] = p.binding + prices[i] = (active = p.price_active, reactive = p.price_reactive) + if p.solved + value[i] = p.cost_stage + lambda[i] = get(p.pin_dual, b.index, NaN) + recourse_used[i] = worst_recourse(p) + end + end + + # ── Finite differences at interior grid points ─────────────────────────── + # The stage value is convex and PIECEWISE smooth in the outgoing energy: a + # binding generator, branch or voltage limit puts a kink in it. A central + # difference taken ACROSS a kink is not an approximation of anything, so the + # check first classifies each interior point by comparing its one-sided + # slopes and only compares λ against a central difference where the curve is + # locally smooth. At a kink the correct statement about a subgradient is that + # it lies BETWEEN the one-sided slopes, and that is what is recorded. + fd = fill(NaN, n) + fd_error = fill(NaN, n) + nonsmooth = falses(n) + bracketed = falses(n) + ok = true + for i in 2:(n - 1) + (solved[i - 1] && solved[i] && solved[i + 1]) || continue + endpoint[i] && continue + h⁻ = energy[i] - energy[i - 1] + h⁺ = energy[i + 1] - energy[i] + slope⁻ = (value[i] - value[i - 1]) / h⁻ + slope⁺ = (value[i + 1] - value[i]) / h⁺ + if abs(slope⁺ - slope⁻) / max(abs(slope⁻), abs(slope⁺), 1e-8) > kink_tolerance + nonsmooth[i] = true + bracketed[i] = min(slope⁻, slope⁺) - 1e-6 <= lambda[i] <= max(slope⁻, slope⁺) + 1e-6 + continue + end + # Non-uniform central difference: exact for a quadratic, which is what + # makes the check meaningful on a grid that is denser near an endpoint. + fd[i] = (h⁻^2 * value[i + 1] - h⁺^2 * value[i - 1] - + (h⁻^2 - h⁺^2) * value[i]) / (h⁻ * h⁺ * (h⁻ + h⁺)) + scale = max(abs(fd[i]), abs(lambda[i]), 1e-8) + fd_error[i] = abs(fd[i] - lambda[i]) / scale + fd_error[i] <= fd_tolerance || (ok = false) + end + + return (battery = b.index, bus = b.bus, stage = Int(stage), atom = Int(atom), + model_type = model_type, + reachable_lo = lo, reachable_hi = hi, + energy = energy, value = value, lambda = lambda, + fd = fd, fd_error = fd_error, fd_ok = ok, + nonsmooth = nonsmooth, bracketed = bracketed, + reachable = reachable, endpoint = endpoint, + solved = solved, status = status, + worst_recourse = recourse_used, residuals = residuals, + binding = binding, prices = prices) +end + +""" + compare_value_curves(case; battery, energy_in, stage, atom, grid, kwargs...) + -> NamedTuple + +Run [`energy_value_curve`](@ref) under BOTH formulations and difference their +marginal stored-energy values. + +# Returns +`(acp, soc, energy, dlambda, dlambda_interior, max_abs_dlambda, + max_rel_dlambda, reversal)` where `dlambda = λ_SOC − λ_ACP`, restricted to grid +points both formulations solved. + +# Notes +`reversal` is `true` when the two formulations disagree about the SIGN of the +marginal value at some interior point — the strongest form of mispricing, because +it means the relaxation would store energy where the true model would spend it. +A magnitude difference changes how much a policy hedges; a sign difference +changes what it does. + +`dlambda_interior` excludes two kinds of grid point, and the exclusions are the +difference between evidence and an artefact: + +- **reachable-interval endpoints**, where a multiplier is a subgradient and a + difference between two valid subgradients is not evidence of anything; +- **points that used physical recourse**, above `max_recourse` in either + formulation. There, part of the marginal value is the recourse PRICE — a number + chosen to be far above any generator — rather than the network's valuation of + stored energy, and it produces enormous, meaningless disagreements. A candidate + whose apparent mechanism lives only at such points is exactly what this phase's + review conditions say to reject. +""" +function compare_value_curves(case::BatteryCase; + battery::Integer, + energy_in::AbstractDict, + stage::Integer, + atom::Integer, + grid::AbstractVector{<:Real}, + max_recourse::Real = 1e-6, + kwargs...) + acp = energy_value_curve(case, PowerModels.ACPPowerModel; + battery = battery, energy_in = energy_in, stage = stage, + atom = atom, grid = grid, kwargs...) + soc = energy_value_curve(case, PowerModels.SOCWRConicPowerModel; + battery = battery, energy_in = energy_in, stage = stage, + atom = atom, grid = grid, kwargs...) + n = length(acp.energy) + dλ = fill(NaN, n) + for i in 1:n + (acp.solved[i] && soc.solved[i]) || continue + dλ[i] = soc.lambda[i] - acp.lambda[i] + end + clean = [i for i in 1:n if acp.solved[i] && soc.solved[i] && + acp.worst_recourse[i] <= max_recourse && + soc.worst_recourse[i] <= max_recourse] + interior = [i for i in clean if !isnan(dλ[i]) && !acp.endpoint[i] && !soc.endpoint[i]] + reversal = any(i -> sign(acp.lambda[i]) != sign(soc.lambda[i]) && + abs(acp.lambda[i]) > 1e-6 && abs(soc.lambda[i]) > 1e-6, + interior) + rel = [abs(dλ[i]) / max(abs(acp.lambda[i]), 1e-8) for i in interior] + return (acp = acp, soc = soc, energy = acp.energy, dlambda = dλ, + dlambda_interior = dλ[interior], + max_abs_dlambda = isempty(interior) ? NaN : maximum(abs, dλ[interior]), + max_rel_dlambda = isempty(rel) ? NaN : maximum(rel), + reversal = reversal, interior = interior, clean = clean, + num_recourse_excluded = count(i -> acp.solved[i] && soc.solved[i], 1:n) - + length(clean)) +end + +# ───────────────────────────────────────────────────────────────────────────── +# B4 — the multiperiod deterministic equivalent +# ───────────────────────────────────────────────────────────────────────────── + +""" + deterministic_equivalent(case, atoms; model_type=ACPPowerModel, + optimizer=nothing, energy_initial=nothing, + silent=true, time_limit=nothing) -> NamedTuple + +Solve the WHOLE horizon jointly for one complete demand path: the +perfect-foresight (wait-and-see) solution. + +# Arguments +- `atoms::AbstractVector{<:Integer}`: the atom index realized at each stage. Its + length is the horizon solved. + +# Keywords +- `model_type::Type`: `ACPPowerModel` — the physical reference — or + `SOCWRConicPowerModel`, which is a RELAXED perfect-foresight solve and is not + a physically attainable cost. +- `energy_initial`: `nothing` for the case's own ``e_{b,0}``, or a dictionary. + +# Returns +A `NamedTuple` with the solver status, the total and per-stage costs, the full +stagewise solution in the shared schema, the terminal energy, the throughput, the +simultaneous-charge audit, the recourse totals and the physical residuals. + +# Notes +There is no policy here, no target, no target slack and no future-cost +approximation: every charge, discharge and outgoing energy over the whole horizon +is chosen at once, knowing the entire demand path. The battery dynamics, the +bounds, the two-sided physical recourse and the cost accounting are the study's +own, because the model is assembled from the same per-stage builder the study +trains on — one `pm` per stage, all in ONE JuMP model, coupled only by +``e^{in}_{b,t+1} = e^{out}_{b,t}``. + +That coupling is the only thing this function writes. In particular it does not +write a network equation, and `assert_powermodels_provenance` is run on every +stage. + +**What the resulting number is.** For a single path it is the cost a clairvoyant +operator would have paid. Averaged over a protocol of paths it is a WAIT-AND-SEE +LOWER BOUND on the nonanticipative stochastic problem — see +[`perfect_foresight_panel`](@ref) for what may and may not be concluded from the +gap to a policy. + +Each stage's build replaces the JuMP objective (PowerModels' own +`objective_min_fuel_and_flow_cost` does), so the per-stage objective is captured +immediately after each build and the horizon objective is set from their sum at +the end. Reading the objective any later would return the last stage's cost. +""" +function deterministic_equivalent(case::BatteryCase, + atoms::AbstractVector{<:Integer}; + model_type::Type = PowerModels.ACPPowerModel, + optimizer = nothing, + energy_initial = nothing, + silent::Bool = true, + time_limit = nothing) + T = length(atoms) + T >= 1 || throw(ArgumentError("a deterministic equivalent needs at least one stage")) + T <= horizon(case.demand) || + throw(ArgumentError("path has $T stages but the frozen support covers $(horizon(case.demand))")) + opt = optimizer === nothing ? _default_optimizer(model_type) : optimizer + e0 = energy_initial === nothing ? + Dict{Int,Float64}(b.index => b.energy_initial for b in case.batteries) : + Dict{Int,Float64}(Int(k) => Float64(v) for (k, v) in energy_initial) + + model = JuMP.Model(opt) + silent && JuMP.set_silent(model) + time_limit === nothing || JuMP.set_time_limit_sec(model, Float64(time_limit)) + + pms = Vector{Any}(undef, T) + stage_obj = Vector{Any}(undef, T) + for t in 1:T + pms[t] = battery_stage_model(case, model_type; + jump_model = model, + stage = t, atom = atoms[t], mode = :free, + state_in = t == 1 ? e0 : nothing) + assert_powermodels_provenance(pms[t], model_type) + # Captured NOW: the next stage's build will overwrite the model objective. + stage_obj[t] = JuMP.objective_function(model) + end + + # The one thing this function writes: the interstage state coupling. + link = Dict{Tuple{Int,Int},Any}() + for t in 2:T, b in case.batteries + link[(t, b.index)] = JuMP.@constraint(model, + pms[t].ext[:battery][:e_in][b.index] == + pms[t - 1].ext[:battery][:e_out][b.index]) + end + + # ...and the terminal requirement, when the CASE declares one. Written as an + # inequality: the convention is that the horizon may not end poorer than it + # began, not that it must end at exactly the opening level, so a schedule + # that finds it worth arriving with more is not forbidden. Imposed only over + # a path that reaches the case's own horizon — a truncated path has no + # terminal stage in the sense the convention means, and silently pinning its + # last stage would charge it for a boundary it was never meant to meet. + floor_T = terminal_energy_floor(case) + terminal = Dict{Int,Any}() + if floor_T !== nothing && T == horizon(case.demand) + for b in case.batteries + haskey(floor_T, b.index) || continue + terminal[b.index] = JuMP.@constraint(model, + pms[T].ext[:battery][:e_out][b.index] >= floor_T[b.index]) + end + end + + JuMP.@objective(model, Min, sum(stage_obj)) + JuMP.optimize!(model) + status = JuMP.termination_status(model) + ok = status in ACCEPTED_STATUSES + + stages = [extract_stage_solution(pms[t]) for t in 1:T] + residuals = ok ? [physical_residuals(case.network, case.batteries, stage_hours(case), s) + for s in stages] : nothing + cost = ok ? [s.cost_stage for s in stages] : fill(NaN, T) + total = ok ? sum(cost) : NaN + Δt = stage_hours(case) + + throughput = Dict{Int,Float64}(b.index => + sum(Δt * (stages[t].p_ch[b.index] + stages[t].p_dis[b.index]) for t in 1:T) + for b in case.batteries) + simultaneous = ok ? maximum(min(stages[t].p_ch[b.index], stages[t].p_dis[b.index]) + for t in 1:T, b in case.batteries) : NaN + worst_deficit = ok ? maximum(max(0.0, maximum(values(stages[t].deficit); init = 0.0)) for t in 1:T) : NaN + worst_surplus = ok ? maximum(max(0.0, maximum(values(stages[t].surplus); init = 0.0)) for t in 1:T) : NaN + + return (status = status, solved = ok, total_cost = total, + stage_cost = cost, cumulative_cost = ok ? cumsum(cost) : fill(NaN, T), + stages = stages, residuals = residuals, + energy_initial = e0, + energy_terminal = Dict{Int,Float64}(b.index => stages[T].energy_out[b.index] + for b in case.batteries), + throughput = throughput, simultaneous = simultaneous, + worst_deficit = worst_deficit, worst_surplus = worst_surplus, + cost_deficit = ok ? sum(s.cost_deficit for s in stages) : NaN, + cost_surplus = ok ? sum(s.cost_surplus for s in stages) : NaN, + atoms = collect(Int.(atoms)), horizon = T, model_type = model_type, + terminal_floor = floor_T, terminal_binding = ok && !isempty(terminal) ? + Dict{Int,Float64}(k => stages[T].energy_out[k] - floor_T[k] for k in keys(terminal)) : + nothing, + link = link, terminal = terminal, model = model) +end + +""" + record_trajectory!(rec, result, case; scenario) + +Write every stage of a [`deterministic_equivalent`](@ref) result into a +[`SolutionRecorder`](@ref) in the shared schema. +""" +function record_trajectory!(rec::SolutionRecorder, result, case::BatteryCase; + scenario::Integer) + for t in 1:result.horizon + record_stage_solution!(rec, result.stages[t], case; scenario = scenario, stage = t) + result.residuals === nothing && continue + r = result.residuals[t] + record!(rec, scenario, t, "residual_equality", 0, + max(r.branch_flow, r.active_balance, r.reactive_balance, r.transition)) + end + return rec +end + +# ───────────────────────────────────────────────────────────────────────────── +# B5 — the perfect-foresight panel +# ───────────────────────────────────────────────────────────────────────────── + +""" + perfect_foresight_panel(case, protocol; ids=nothing, model_type=ACPPowerModel, + optimizer=nothing, retry=true, verbose=false) + -> NamedTuple + +Solve one true-ACP deterministic equivalent per GLOBAL scenario identifier and +summarize the panel. + +# Arguments +- `protocol::AbstractMatrix{<:Integer}`: the `(stages × scenarios)` atom-index + matrix, from [`scenario_index_matrix`](@ref). + +# Keywords +- `ids`: the global scenario identifiers to solve, defaulting to every column. + A column's identifier is its INDEX IN THE PROTOCOL and never changes. +- `retry::Bool`: on a first-attempt failure, re-solve once with a completely + fresh model and solver. + +# Returns +A `NamedTuple` with one row per requested identifier — cost, status, first-attempt +status, recourse, terminal energy, throughput — plus the panel mean, standard +deviation and standard error, and the full per-scenario results. + +# Notes +**Fail closed.** Every requested identifier is retained: none is replaced, +dropped or renumbered, and if any is unsolved after the retry the panel is marked +incomplete. A mean over the scenarios that happened to succeed is a mean over a +different problem, and no tolerance makes it comparable to a mean over all of +them. The first-attempt status is recorded separately from the retry status, +because "solved on the second try" is information about the case. + +**What the mean is.** The true-ACP perfect-foresight mean is a WAIT-AND-SEE LOWER +BOUND on the nonanticipative stochastic problem: a clairvoyant operator cannot be +beaten by one who must decide before seeing the future. The gap between a +policy's cost and this bound is therefore diagnostic HEADROOM, and it contains +the value of future information — which no nonanticipative policy, TS-DDR or +SDDP, can recover. It is not a target, it is not attainable, and a policy that +closes half of it has not necessarily left anything on the table. + +The bound may be computed during screening, on the screening protocol. Its +relationship to a trained policy is assessed only on PAIRED paths, after that +policy exists. +""" +function perfect_foresight_panel(case::BatteryCase, protocol::AbstractMatrix{<:Integer}; + ids = nothing, + model_type::Type = PowerModels.ACPPowerModel, + optimizer = nothing, + retry::Bool = true, + verbose::Bool = false) + columns = ids === nothing ? collect(1:size(protocol, 2)) : collect(Int.(ids)) + all(c -> 1 <= c <= size(protocol, 2), columns) || + throw(ArgumentError("scenario identifier outside the protocol's 1:$(size(protocol, 2))")) + + rows = NamedTuple[] + results = Dict{Int,Any}() + for c in columns + path = protocol[:, c] + res = deterministic_equivalent(case, path; model_type = model_type, + optimizer = optimizer) + first_status = res.status + if !res.solved && retry + # A completely fresh model and solver, not a re-solve: a re-solve + # from a failed interior point is a different algorithm, not a + # second attempt at the same one. + res = deterministic_equivalent(case, path; model_type = model_type, + optimizer = optimizer) + end + results[c] = res + push!(rows, (scenario = c, solved = res.solved, + first_status = first_status, status = res.status, + cost = res.total_cost, + worst_deficit = res.worst_deficit, worst_surplus = res.worst_surplus, + simultaneous = res.simultaneous, + terminal_energy = sum(values(res.energy_terminal)), + throughput = sum(values(res.throughput)))) + verbose && @printf(" scenario %4d %-16s cost %14.4f\n", c, string(res.status), + res.total_cost) + end + + complete = all(r -> r.solved, rows) + costs = [r.cost for r in rows if r.solved] + n = length(costs) + return (rows = rows, results = results, complete = complete, + num_requested = length(columns), num_solved = n, + mean = n == 0 ? NaN : mean(costs), + std = n < 2 ? NaN : std(costs), + sem = n < 2 ? NaN : std(costs) / sqrt(n), + worst_deficit = maximum(r -> r.worst_deficit, rows; init = 0.0), + worst_surplus = maximum(r -> r.worst_surplus, rows; init = 0.0), + model_type = model_type) +end + +""" + paired_difference(a::AbstractVector, b::AbstractVector) -> NamedTuple + +Paired mean difference `a − b` with its standard error, `t` statistic and 95 % +confidence interval. + +# Notes +Pairing is what makes a comparison of two policies on a common demand protocol +decidable: the spread of cost ACROSS scenarios is typically an order of magnitude +larger than the difference BETWEEN policies on the same scenario, so an unpaired +comparison of the same sample size would resolve nothing. + +The interval uses the normal quantile 1.96 rather than a `t` quantile; at the +sample sizes this study reports on (hundreds of paired paths) the difference is +in the third decimal of the interval and is not worth a distributions dependency +in a file every consumer loads. +""" +function paired_difference(a::AbstractVector, b::AbstractVector) + length(a) == length(b) || throw(ArgumentError("paired vectors must have equal length")) + d = Float64.(a) .- Float64.(b) + n = length(d) + n >= 2 || throw(ArgumentError("a paired difference needs at least two pairs")) + m = mean(d) + se = std(d) / sqrt(n) + return (n = n, mean = m, std = std(d), sem = se, + t = se == 0 ? NaN : m / se, + ci = (m - 1.96 * se, m + 1.96 * se), + wins = count(<(0), d)) +end + + +# ───────────────────────────────────────────────────────────────────────────── +# The deterministic-forecast reference policy +# ───────────────────────────────────────────────────────────────────────────── + +""" + central_atom_path(case, T) -> Vector{Int} + +The atom whose total multiplier is closest to the stage's PROBABILITY-WEIGHTED +mean, for each stage — i.e. the single scenario a forecaster would use. +""" +function central_atom_path(case::BatteryCase, T::Integer) + return [begin + p = atom_probabilities(case.demand, t) + μ = sum(p[k] * mean(demand_multipliers(case.demand, t, k)) + for k in eachindex(p)) + argmin([abs(mean(demand_multipliers(case.demand, t, k)) - μ) + for k in eachindex(p)]) + end for t in 1:Int(T)] +end + +""" + forecast_policy_cost(case, protocol; columns, optimizer=nothing) -> NamedTuple + +The cost of the DETERMINISTIC-FORECAST policy: optimise once against a single +forecast path, then operate those storage targets on every realized path. + +# How it is built +1. Solve one true-ACP deterministic equivalent against + [`central_atom_path`](@ref) — the forecaster's single scenario. +2. Take its outgoing battery energies as a fixed target schedule. +3. On each evaluation path, walk forward imposing those targets as STRICT + equalities, clamping each into the one-stage reachable interval of the state + actually reached. Clamping is not a fudge: a real forecast-based operator + cannot charge past its rating either, and the reachable map is the same one + the strict policy uses. + +# Why it is reported +`VSS = forecast_cost − sddp_cost` is the **value of the stochastic solution**: how +much is lost by ignoring uncertainty and operating a single forecast. It is the +direct test of whether a case's uncertainty structure requires a policy at all. +If VSS is near zero, every method — SDDP, TS-DDR, a spreadsheet — costs the same, +and no comparison between them can resolve anything, however large the other +gaps look. +""" +function forecast_policy_cost(case::BatteryCase, protocol::AbstractMatrix{<:Integer}; + columns, optimizer = nothing) + T = size(protocol, 1) + opt = optimizer === nothing ? acp_optimizer() : optimizer + plan = deterministic_equivalent(case, central_atom_path(case, T)) + plan.solved || error("the forecast deterministic equivalent did not solve") + schedule = [Dict{Int,Float64}(b.index => plan.stages[t].energy_out[b.index] + for b in case.batteries) for t in 1:T] + + Δt = stage_hours(case) + costs = Float64[] + worst_rec = 0.0 + for c in collect(Int.(columns)) + e = Dict{Int,Float64}(b.index => b.energy_initial for b in case.batteries) + total = 0.0 + for t in 1:T + tgt = Dict{Int,Float64}() + for b in case.batteries + lo, hi = reachable_interval(b, e[b.index], Δt) + tgt[b.index] = clamp(schedule[t][b.index], lo, hi) + end + sol = solve_strict_stage(case, PowerModels.ACPPowerModel; stage = t, + atom = protocol[t, c], energy_in = e, + target = tgt, optimizer = opt) + sol.solved || error("forecast policy: stage $t of scenario $c did not solve") + total += sol.cost_stage + worst_rec = max(worst_rec, worst_recourse(sol)) + e = tgt + end + push!(costs, total) + end + return (costs = costs, mean = mean(costs), max = maximum(costs), + worst_recourse = worst_rec, plan = plan) +end diff --git a/examples/BatteryStorageOPF/battery_portfolio.jl b/examples/BatteryStorageOPF/battery_portfolio.jl new file mode 100644 index 0000000..6b26842 --- /dev/null +++ b/examples/BatteryStorageOPF/battery_portfolio.jl @@ -0,0 +1,2505 @@ +# The preregistered PGLib battery portfolio. +# +# The study is a PANEL of canonical PGLib systems rather than one engineered +# case, so every construction choice has to be a documented FUNCTION of the +# benchmark's own bytes: given a case name and this file, any reader must land on +# the same network, the same storage buses, the same regions, the same finite +# demand support and the same calibrated demand level, byte for byte, on any +# machine and without the private campaign repository. +# +# That constraint is what shapes the code below. +# +# * Nothing consults a Julia RNG for a structural choice. `StableRNG` is stable +# across versions, but "stable" is a property of one library's stream and the +# panel has to survive a reader who reimplements the rule. Placement keys come +# from SHA-256 of a documented string, which is a specification, not an +# implementation. +# * Nothing iterates a `Dict` and keeps the order. Every loop that can affect a +# result runs over an explicitly sorted vector of identifiers, so a network +# parsed twice — or parsed by a different JSON reader — gives one answer. +# * Nothing is calibrated per case by hand. One rule, one margin, one search, +# frozen before any method is run, and the margin is a constant in this file +# rather than an argument a caller could tune case by case. +# +# The public entry points are `build_portfolio_case` (construct one frozen case), +# `freeze_portfolio` (construct the whole panel and write the manifest), +# `materialize_portfolio_case` (regenerate one case from the manifest and check +# it against the recorded digests) and `verify_portfolio_manifest`. +# +# Commands: +# julia --project=. battery_portfolio.jl --list +# julia --project=. battery_portfolio.jl --case pglib_opf_case118_ieee --out DIR +# julia --project=. battery_portfolio.jl --verify +# julia --project=. battery_portfolio.jl --freeze --out DIR + +using PGLib +using PowerModels +using HiGHS +using JSON +using LinearAlgebra +using Printf +using SHA +using Statistics +using TOML +import MathOptInterface as MOI + +@isdefined(acquire_pglib_case) || include(joinpath(@__DIR__, "build_battery_case.jl")) +@isdefined(base_feasibility) || include(joinpath(@__DIR__, "battery_diagnostics.jl")) + +# ───────────────────────────────────────────────────────────────────────────── +# A — the preregistered constants +# +# Everything in this section was fixed before any method was run on any case. +# Changing one of them invalidates the panel rather than adjusting it, which is +# why they are `const` in the public file and not keyword arguments. +# ───────────────────────────────────────────────────────────────────────────── + +"Schema tag of the portfolio manifest." +const PORTFOLIO_SCHEMA = "battery_storage_opf/portfolio/1" + +""" +The one global seed. It enters every derived quantity only through SHA-256, +together with the canonical digest of the network the quantity belongs to, so +two cases never share a draw and no quantity depends on the order the panel was +built in. +""" +const PORTFOLIO_SEED = 20260814 + +"Horizon of every frozen case, in stages." +const PORTFOLIO_HORIZON = 24 + +"Duration of one stage, in hours. The profile below is hourly, so this is 1." +const PORTFOLIO_STAGE_HOURS = 1.0 + +""" +The common normalized daily profile ``h_t``, multiplying the canonical PGLib +`pd` AND `qd` so every realization keeps each load's own power factor. + +# Notes +It is the same 24 numbers for every case in the panel: the panel varies the +NETWORK, and a per-case profile would confound the two. Its maximum is 1.0 and +is first attained at stage 11, which is the stage the headroom calibration +searches on. +""" +const PORTFOLIO_PROFILE = [ + 0.72, 0.68, 0.65, 0.64, 0.66, 0.72, + 0.80, 0.88, 0.94, 0.98, 1.00, 0.99, + 0.97, 0.95, 0.94, 0.96, 1.00, 1.00, + 0.98, 0.94, 0.90, 0.84, 0.79, 0.75, +] + +"Number of PTDF-sensitivity demand regions per case." +const PORTFOLIO_REGIONS = 6 + +"The multiplier a region carries in its OWN atom." +const PORTFOLIO_REGION_HIGH = 1.15 + +"The multiplier a region carries in every other region's atom." +const PORTFOLIO_REGION_LOW = 0.97 + +""" +Lower bound on a region's share of the case's nominal active demand. + +# Notes +Six regions and an unconstrained clustering do not make six LEVERS. The atoms +move demand by region, so a region carrying 0.2 % of the load is an atom that +moves nothing: measured on the first freeze, `case1951_rte` put 80.8 % of demand +in one region and `case300_ieee` 61.5 %, and on those cases the six atoms +collapse toward two directions — one region high, everything else low. + +The bounds below are what make all six atoms carry comparable demand mass. They +are a CONSTRUCTION gate, fixed before any method result existed, and applied to +every case of the panel rather than to the ones that looked worst. +""" +const PORTFOLIO_REGION_SHARE_MIN = 0.08 + +"Upper bound on a region's share of the case's nominal active demand." +const PORTFOLIO_REGION_SHARE_MAX = 0.28 + +""" +Smallest admissible effective region count, +``N_{eff} = 1/\\sum_r s_r^2`` over the region demand shares. + +# Notes +The inverse Simpson index of the demand shares: the number of EQUALLY weighted +regions that would produce the same concentration. Six exactly equal regions give +6.0; one region carrying everything gives 1.0. The share bounds imply this gate +on their own — shares in `[0.08, 0.28]` cannot concentrate below about 4.6 — so +it is a redundant check, which is exactly what makes it worth asserting. +""" +const PORTFOLIO_MIN_EFFECTIVE_REGIONS = 4.5 + +"Maximum balanced-assignment refinement sweeps." +const PORTFOLIO_BALANCE_ITERATIONS = 20 + +""" +The largest number of branch dimensions a load bus's sensitivity signature is +built from. A signature over every limited corridor of a 2000-bus system is +mostly noise from corridors no demand can move; the cap keeps the clustering +looking at the corridors the demand actually reaches. +""" +const PORTFOLIO_MAX_CORRIDORS = 64 + +"Fraction of the calibrated peak active demand the whole storage fleet can discharge." +const PORTFOLIO_POWER_SHARE = 0.10 + +""" +Cap on one bus's raw sizing weight, as a multiple of the median weight over the +SELECTED buses. Nominal demand is heavy-tailed on most PGLib systems and an +uncapped proportional split puts a third of the fleet on one bus, which measures +that bus rather than the network. +""" +const PORTFOLIO_WEIGHT_CAP = 3.0 + +"Storage duration at full discharge power, in hours." +const PORTFOLIO_DURATION_HOURS = 8.0 + +"Energy floor, as a fraction of energy capacity." +const PORTFOLIO_RESERVE_FRACTION = 0.05 + +"Initial energy, as a fraction of energy capacity." +const PORTFOLIO_INITIAL_FRACTION = 0.5 + +"Charging efficiency ``\\eta^{ch}``." +const PORTFOLIO_CHARGE_EFFICIENCY = 0.95 + +"Discharging efficiency ``\\eta^{dis}``." +const PORTFOLIO_DISCHARGE_EFFICIENCY = 0.95 + +"Self-discharge ``\\alpha`` per stage." +const PORTFOLIO_SELF_DISCHARGE = 0.999 + +"Throughput cost per unit of energy moved, in the case's own objective units." +const PORTFOLIO_THROUGHPUT_COST = 5.0 + +"Lower end of the headroom search. Below this a case is replaced, not rescaled." +const PORTFOLIO_KAPPA_LO = 0.50 + +"Upper end of the headroom search." +const PORTFOLIO_KAPPA_HI = 1.25 + +""" +Spacing of the bracketing scan that precedes the bisection. + +# Notes +The admissible set is NOT an interval anchored at `PORTFOLIO_KAPPA_LO`. On +several PGLib systems the base ACP has to spill at low demand — generators whose +`pmin` is positive cannot be switched off in an OPF, so at half load the network +is over-generating and the surplus injection carries it — while the same system +is perfectly clean at 0.9. A bisection that assumed admissibility at the bottom +of the bracket would report those systems as failures of the data gate, which +they are not. + +So the search scans this grid DOWNWARD from `PORTFOLIO_KAPPA_HI` first, stops at +the largest admissible grid point, and bisects between it and the inadmissible +point above it. The grid is a constant of this file, identical for every case. +""" +const PORTFOLIO_KAPPA_STEP = 0.05 + +"Bisection stops when the bracket is narrower than this." +const PORTFOLIO_KAPPA_TOL = 1e-3 + +""" +The margin below the largest admissible demand level. Fixed before results and +identical for every case: a per-case margin is a per-case tuning knob wearing a +safety argument. +""" +const PORTFOLIO_KAPPA_MARGIN = 0.95 + +""" +How many grid steps the frozen level may retreat when the full `24 × 6` +verification rejects it. + +# Notes +The search gates on the profile's peak and trough, which are the two extremes of +the day — but admissibility is not monotone in the demand level, so an INTERIOR +profile value can fail at a level both extremes accept. Measured on +`case2000_goc`: the search returned `κ_max = 1.159375` and the verification then +rejected `3` of the `144` stage/atom combinations. + +Gating the search on all eighteen distinct profile values instead would be +exact, but it multiplies the search by nine and a single 2000-bus ACP solve +takes about twenty-four seconds; the search alone would outlast the job. + +So the frozen level RETREATS instead: one grid step down, re-verify, up to this +many times, and the case fails the gate if none of them survives. Every attempt +is recorded. It is a fixed rule with a fixed bound, applied identically to every +case — not a per-case search for a level that happens to work. +""" +const PORTFOLIO_KAPPA_RETREATS = 3 + +"Largest physical residual a solve may leave and still count as complete (pu)." +const PORTFOLIO_RESIDUAL_TOL = 1e-7 + +""" +Largest nodal deficit or surplus injection that still counts as none (pu). + +# Notes +Deliberately looser than the residual tolerance. The recourse variables are +priced far above any generator, so an interior-point method leaves them at a +small positive value rather than exactly zero even when nothing is short; what +must not happen is a solve that USES them. +""" +const PORTFOLIO_RECOURSE_TOL = 1e-6 + +"Columns of the screening protocol." +const PORTFOLIO_SCREENING_SCENARIOS = 32 + +"Columns of the final protocol." +const PORTFOLIO_FINAL_SCENARIOS = 500 + +"The preregistered primary panel, in order." +const PORTFOLIO_PRIMARY = [ + "pglib_opf_case118_ieee", + "pglib_opf_case162_ieee_dtc", + "pglib_opf_case179_goc", + "pglib_opf_case200_activ", + "pglib_opf_case240_pserc", + "pglib_opf_case300_ieee", + "pglib_opf_case500_goc", + "pglib_opf_case588_sdet", + "pglib_opf_case793_goc", + "pglib_opf_case1354_pegase", + "pglib_opf_case1888_rte", + "pglib_opf_case2000_goc", +] + +""" +The reserve panel, in order. A primary case is replaced only when the canonical +case is unavailable from the pinned PGLib version, or when its UNMODIFIED base +ACP fails at the conservative level `PORTFOLIO_KAPPA_LO`. A failure of SOC, of +DC or of any method is never a reason to replace a case. +""" +const PORTFOLIO_RESERVE = [ + "pglib_opf_case1951_rte", + "pglib_opf_case2312_goc", + "pglib_opf_case2383wp_k", + "pglib_opf_case2736sp_k", +] + +"Smallest panel that may be frozen." +const PORTFOLIO_MIN_CASES = 10 + +"The public command a reader runs to regenerate one case." +const PORTFOLIO_COMMAND = + "julia --project=. battery_portfolio.jl --case --out " + +# ───────────────────────────────────────────────────────────────────────────── +# B — deterministic keys +# +# Every structural draw in this file is a pure function of three things: the +# portfolio seed, the canonical digest of the network being drawn on, and the +# identifier being drawn for. No RNG object, no stream position, no iteration +# order. The rule below is the specification; a reader who reimplements it in +# another language reproduces the panel exactly. +# ───────────────────────────────────────────────────────────────────────────── + +""" + network_digest(network) -> String + +SHA-256 of the canonical JSON encoding of a parsed PGLib network. + +# Notes +`canonical_json` sorts object keys and prints numbers in a fixed form, so the +digest is a function of the network's CONTENT and not of the order a `Dict` +happened to iterate in. This digest is the network's identity everywhere in the +portfolio: placement keys, region labels and protocol seeds all consume it, so +two cases that differ by one branch cannot accidentally share a draw. +""" +network_digest(network::AbstractDict) = bytes2hex(sha256(canonical_json(plain(network)))) + +""" + portfolio_key(tag, network_sha, id) -> String + +The exact byte string whose SHA-256 is the uniform draw for `id`. + +# Arguments +- `tag::AbstractString`: what is being drawn — `"placement"`, `"region-label"`, + `"protocol/final"`, … Distinct tags give independent draws from the same seed. +- `network_sha::AbstractString`: [`network_digest`](@ref) of the case. +- `id`: the identifier being drawn for; printed with `string`. + +# Notes +Written as its own function so the specification is one readable line and the +tests can assert the literal bytes. The trailing newline matters: without a +separator that cannot appear in a field, `("a", "bc")` and `("ab", "c")` would +hash the same. +""" +portfolio_key(tag::AbstractString, network_sha::AbstractString, id) = + string(PORTFOLIO_SCHEMA, "\n", tag, "\n", PORTFOLIO_SEED, "\n", + network_sha, "\n", string(id), "\n") + +""" + portfolio_uniform(tag, network_sha, id) -> Float64 + +A uniform draw on ``(0,1)`` derived from SHA-256 of [`portfolio_key`](@ref). + +# Notes +The first eight digest bytes are read big-endian into a `UInt64`, the low 11 bits +are discarded and the remaining 53 are placed on the ``2^{-53}`` grid offset by +half a step. Discarding the low bits is what makes the value exactly +representable, so the draw is the same number on any platform's floating point +rather than the same number up to rounding; the half-step offset keeps it +strictly inside ``(0,1)``, which `-log(u)` requires. +""" +function portfolio_uniform(tag::AbstractString, network_sha::AbstractString, id) + h = sha256(portfolio_key(tag, network_sha, id)) + x = zero(UInt64) + for i in 1:8 + x = (x << 8) | UInt64(h[i]) + end + return (Float64(x >> 11) + 0.5) * 2.0^-53 +end + +""" + weighted_selection(candidates, weights, k; tag, network_sha) -> Vector{Int} + +Weighted sampling WITHOUT replacement, keyed by hash rather than by an RNG. + +# Arguments +- `candidates::AbstractVector{Int}`: the pool, any order; the result does not + depend on it. +- `weights::AbstractDict{Int,<:Real}`: strictly positive weight per candidate. +- `k::Integer`: how many to select. + +# Returns +- The `k` selected identifiers, sorted ascending. + +# Notes +The rule is Efraimidis–Spirakis with exponential keys: draw +``u_b \\sim U(0,1)`` and give `b` the key + +```math +\\kappa_b = \\frac{-\\log u_b}{w_b}, +``` + +then take the `k` SMALLEST keys. Because ``-\\log u_b`` is a unit exponential, +``\\kappa_b`` is exponential with rate ``w_b``, the smallest of independent +exponentials is bus `b` with probability ``w_b / \\sum w``, and the same holds +recursively on what is left — so this is exactly sampling proportional to weight +without replacement, not an approximation of it. + +Ties are broken by ascending bus identifier. A tie needs two 53-bit draws to +coincide, but "break ties by bus id" is a one-line rule that makes the output +independent of the sort algorithm's stability, and that is worth more than the +probability argument. +""" +function weighted_selection(candidates::AbstractVector{<:Integer}, + weights::AbstractDict{<:Integer,<:Real}, + k::Integer; + tag::AbstractString, network_sha::AbstractString) + pool = sort!(collect(Int.(candidates))) + allunique(pool) || error("weighted selection pool contains duplicates") + 0 <= k <= length(pool) || + throw(ArgumentError("cannot select $k of $(length(pool)) candidates")) + keyed = Tuple{Float64,Int}[] + for b in pool + w = Float64(weights[b]) + w > 0 || error("bus $b has non-positive selection weight $w") + u = portfolio_uniform(tag, network_sha, b) + push!(keyed, (-log(u) / w, b)) + end + sort!(keyed; by = x -> (x[1], x[2])) + return sort!([b for (_, b) in keyed[1:Int(k)]]) +end + +""" + portfolio_seed_for(tag, network_sha) -> Int + +A per-case integer seed for a `StableRNG`-driven protocol, derived from the same +key rule. + +# Notes +Reduced into `1:2^31-1` so it is a valid seed on every platform and prints as a +small integer in the manifest. Two cases get independent protocols because the +network digest is part of the key; two tags get independent protocols on one case +for the same reason. +""" +function portfolio_seed_for(tag::AbstractString, network_sha::AbstractString) + h = sha256(portfolio_key(tag, network_sha, "seed")) + x = zero(UInt64) + for i in 1:8 + x = (x << 8) | UInt64(h[i]) + end + return Int(x % UInt64(2147483647)) + 1 +end + +""" + digest_of(parts...) -> String + +SHA-256 over a newline-joined list of already-stringified parts. + +# Notes +Used for the derived digests the manifest records — placement, regions, ratings. +Each caller prints its own fields, so what is hashed is visible at the call site +rather than hidden in a serializer. +""" +digest_of(parts...) = bytes2hex(sha256(join(string.(parts), "\n") * "\n")) + +# ───────────────────────────────────────────────────────────────────────────── +# C — eligibility, placement and ratings +# ───────────────────────────────────────────────────────────────────────────── + +""" + connected_component(network) -> Set{Int} + +Bus identifiers reachable from the reference bus over in-service branches. + +# Notes +`eligible_buses` already rejects a bus with no in-service branch, but a PGLib +case can contain a whole ISLAND of interconnected buses that the reference bus +cannot reach. Such an island is dropped by the solved network, so a battery +placed on it would be storage the study never dispatches. +""" +function connected_component(network::AbstractDict) + refs = sort!([Int(b["index"]) for (_, b) in network["bus"] + if Int(get(b, "bus_type", 1)) == 3]) + isempty(refs) && error("the network declares no reference bus") + adj = Dict{Int,Vector{Int}}() + for k in sort!(collect(keys(network["branch"]))) + br = network["branch"][k] + Int(get(br, "br_status", 1)) == 0 && continue + f, t = Int(br["f_bus"]), Int(br["t_bus"]) + push!(get!(adj, f, Int[]), t) + push!(get!(adj, t, Int[]), f) + end + seen = Set{Int}(refs) + stack = copy(refs) + while !isempty(stack) + v = pop!(stack) + for w in sort!(get(adj, v, Int[])) + w in seen && continue + push!(seen, w) + push!(stack, w) + end + end + return seen +end + +""" + portfolio_eligible_buses(network) -> Vector{Int} + +The storage-eligible buses of a case: in service, positive nominal active +demand, and retained in the connected solved network. +""" +function portfolio_eligible_buses(network::AbstractDict) + load_at = nominal_load_at_bus(network) + component = connected_component(network) + pool = [b for b in eligible_buses(network) + if get(load_at, b, 0.0) > 0 && b in component] + isempty(pool) && error("no storage-eligible bus remains") + return sort!(pool) +end + +""" + portfolio_battery_count(network) -> Int + +The preregistered fleet size, + +```math +n_{bat} = \\min\\Bigl(|\\mathcal B_{elig}|,\\; + \\mathrm{clamp}\\bigl(\\mathrm{round}(0.20\\, n_{bus}),\\, 24,\\, 240\\bigr)\\Bigr). +``` +""" +function portfolio_battery_count(network::AbstractDict) + n_bus = length(network["bus"]) + return min(length(portfolio_eligible_buses(network)), + clamp(round(Int, 0.20 * n_bus), 24, 240)) +end + +""" + portfolio_placement(network, network_sha) -> (buses, record) + +The reproducible storage placement: `portfolio_battery_count` buses drawn from +the eligible pool without replacement, with probability proportional to nominal +active demand, using [`weighted_selection`](@ref). +""" +function portfolio_placement(network::AbstractDict, network_sha::AbstractString) + pool = portfolio_eligible_buses(network) + load_at = nominal_load_at_bus(network) + weights = Dict{Int,Float64}(b => load_at[b] for b in pool) + k = min(length(pool), clamp(round(Int, 0.20 * length(network["bus"])), 24, 240)) + buses = weighted_selection(pool, weights, k; + tag = "placement", network_sha = network_sha) + record = Dict{String,Any}( + "algorithm" => "sha256-exponential-key weighted sampling without replacement", + "key" => "SHA256(\"$PORTFOLIO_SCHEMA\\nplacement\\n$PORTFOLIO_SEED\\n\\n\\n\")", + "weight" => "nominal in-service active demand at the bus (pu)", + "seed" => PORTFOLIO_SEED, + "network_sha256" => network_sha, + "eligible" => pool, + "count" => k, + "buses" => buses, + "digest" => digest_of("portfolio/placement/1", network_sha, PORTFOLIO_SEED, + join(buses, ",")), + ) + return buses, record +end + +""" + portfolio_ratings(network, buses, peak_demand) -> (power, budget, record) + +Split the fleet's discharge-power budget across the selected buses. + +# Arguments +- `peak_demand::Real`: the case's CALIBRATED peak active demand + ``\\kappa \\max_t h_t \\sum_i p^{d,0}_i`` (pu). + +# Returns +- `power::Dict{Int,Float64}`: discharge (and charge) rating per bus, pu. +- `budget::Float64`: ``0.10\\,\\times`` the calibrated peak, pu. +- `record::Dict`: the manifest record, including the digest of the ratings. + +# Notes +Raw weight is the bus's nominal active demand, capped at +`PORTFOLIO_WEIGHT_CAP` times the MEDIAN over the selected buses — the median of +the selection, not of the eligible pool, because the cap is there to keep one +selected bus from dominating the selected fleet. + +The budget is distributed by normalized weight and the LAST battery absorbs the +remainder, so the fleet's aggregate power equals the declared budget to within a +few ulps rather than to an accumulated relative error over up to 240 divisions. +It cannot be made to agree BITWISE and be checked as such, because the check +would then depend on the order the sum was taken in; the tolerance is a few ulps +of the budget per battery and is asserted here. Both the unrounded weights and +the resulting ratings are recorded. +""" +function portfolio_ratings(network::AbstractDict, buses::AbstractVector{<:Integer}, + peak_demand::Real) + load_at = nominal_load_at_bus(network) + ordered = sort!(collect(Int.(buses))) + raw = [max(0.0, get(load_at, b, 0.0)) for b in ordered] + all(>(0), raw) || error("a selected storage bus has non-positive nominal demand") + cap = PORTFOLIO_WEIGHT_CAP * Statistics.median(raw) + capped = min.(raw, cap) + budget = PORTFOLIO_POWER_SHARE * Float64(peak_demand) + budget > 0 || error("the calibrated peak demand must be positive, got $peak_demand") + total = sum(capped) + power = Dict{Int,Float64}() + running = 0.0 + for (i, b) in enumerate(ordered) + p = i == length(ordered) ? budget - running : budget * capped[i] / total + p > 0 || error("bus $b received a non-positive power rating $p") + power[b] = p + running += p + end + # Summed in the order the split accumulated in, because floating-point + # addition is not associative and a `Dict`'s iteration order is not that one. + total = sum(power[b] for b in ordered) + abs(total - budget) <= 8 * eps(budget) * length(ordered) || + error("fleet power $total does not equal the budget $budget") + record = Dict{String,Any}( + "power_budget_pu" => budget, + "peak_demand_pu" => Float64(peak_demand), + "system_power_share" => PORTFOLIO_POWER_SHARE, + "weight" => "nominal active demand, capped at $(PORTFOLIO_WEIGHT_CAP)× the selected-bus median", + "weight_cap_pu" => cap, + "raw_weight_pu" => Dict(string(b) => raw[i] for (i, b) in enumerate(ordered)), + "capped_weight_pu" => Dict(string(b) => capped[i] for (i, b) in enumerate(ordered)), + "power_pu" => Dict(string(b) => power[b] for b in ordered), + "duration_hours" => PORTFOLIO_DURATION_HOURS, + "reserve_fraction" => PORTFOLIO_RESERVE_FRACTION, + "initial_fraction" => PORTFOLIO_INITIAL_FRACTION, + "charge_efficiency" => PORTFOLIO_CHARGE_EFFICIENCY, + "discharge_efficiency" => PORTFOLIO_DISCHARGE_EFFICIENCY, + "self_discharge" => PORTFOLIO_SELF_DISCHARGE, + "throughput_cost" => PORTFOLIO_THROUGHPUT_COST, + "digest" => digest_of("portfolio/ratings/1", budget, + join((@sprintf("%d:%.17g", b, power[b]) for b in ordered), ",")), + ) + return power, budget, record +end + +""" + portfolio_fleet(network, buses, peak_demand) -> (fleet, record) + +Build the `BatterySpec` fleet from [`portfolio_ratings`](@ref) and the common +technology constants. +""" +function portfolio_fleet(network::AbstractDict, buses::AbstractVector{<:Integer}, + peak_demand::Real) + power, _, record = portfolio_ratings(network, buses, peak_demand) + fleet, capacity = battery_fleet(network, sort!(collect(Int.(buses))); + power = power, + energy_hours = PORTFOLIO_DURATION_HOURS, + charge_efficiency = PORTFOLIO_CHARGE_EFFICIENCY, + discharge_efficiency = PORTFOLIO_DISCHARGE_EFFICIENCY, + self_discharge = PORTFOLIO_SELF_DISCHARGE, + throughput_cost = PORTFOLIO_THROUGHPUT_COST, + initial_fraction = PORTFOLIO_INITIAL_FRACTION, + reserve_fraction = PORTFOLIO_RESERVE_FRACTION) + merged = Dict{String,Any}(record) + merged["capacity_record"] = capacity + return fleet, merged +end + +# ───────────────────────────────────────────────────────────────────────────── +# D — PTDF sensitivity regions +# +# Uniform demand scaling is the one direction that cannot switch a constraint: +# every per-bus sensitivity is averaged out and the network stays qualitatively +# where it was. To make demand a lever on the network's own binding structure the +# regions have to separate buses by HOW they load the corridors, and that map is +# the PTDF matrix. +# +# DC sensitivities are used only to CHOOSE where demand goes. Every number the +# study reports still comes from the true ACP model. +# ───────────────────────────────────────────────────────────────────────────── + +""" + ptdf_matrix(network) -> NamedTuple + +DC power transfer distribution factors of the in-service network. + +# Returns +A named tuple with +- `M::Matrix{Float64}`: `M[l, i]` is the flow induced on live branch `l` by a + unit injection at bus position `i`, withdrawn at the reference bus; +- `live::Vector{NTuple{4,Any}}`: `(branch id, f_bus, t_bus, reactance)` per row of + `M`, in ascending branch-id order; +- `pos::Dict{Int,Int}`: bus identifier → column of `M`; +- `buses::Vector{Int}`: the in-service bus identifiers, ascending; +- `ref::Int`: the reference bus identifier. + +# Notes +Built from the DC susceptance matrix ``B'`` and the branch-flow map ``B_f`` as +``M = B_f B'^{-1}`` with the reference row and column removed and the reference +column left at zero. A load increase at bus `i` is a NEGATIVE injection, so +demand at buses with large positive `M[l, i]` unloads branch `l` and demand at +large negative entries loads it — which is why the sensitivity signature below +carries a minus sign. + +Rows and columns are ordered by identifier, never by `Dict` iteration, so the +matrix is a function of the network's content alone. +""" +function ptdf_matrix(network::AbstractDict) + buses = sort!([Int(b["index"]) for (_, b) in network["bus"] + if Int(get(b, "bus_type", 1)) != 4]) + isempty(buses) && error("the network has no in-service bus") + pos = Dict(b => i for (i, b) in enumerate(buses)) + nb = length(buses) + refbus = minimum(Int(b["index"]) for (_, b) in network["bus"] + if Int(get(b, "bus_type", 1)) == 3) + ref = pos[refbus] + + live = Tuple{Int,Int,Int,Float64}[] + for k in sort!(collect(keys(network["branch"])); by = x -> Int(network["branch"][x]["index"])) + br = network["branch"][k] + Int(get(br, "br_status", 1)) == 0 && continue + x = Float64(br["br_x"]) + abs(x) > 1e-8 || continue + f, t = Int(br["f_bus"]), Int(br["t_bus"]) + (haskey(pos, f) && haskey(pos, t)) || continue + push!(live, (Int(br["index"]), f, t, x)) + end + nl = length(live) + nl > 0 || error("the network has no in-service branch with a usable reactance") + + B = zeros(nb, nb) + Bf = zeros(nl, nb) + for (e, (_, f, t, x)) in enumerate(live) + i, j, b = pos[f], pos[t], 1 / x + B[i, i] += b; B[j, j] += b; B[i, j] -= b; B[j, i] -= b + Bf[e, i] += b; Bf[e, j] -= b + end + keep = setdiff(1:nb, ref) + M = zeros(nl, nb) + M[:, keep] = Bf[:, keep] / B[keep, keep] + return (M = M, live = live, pos = pos, buses = buses, ref = refbus) +end + +""" + select_corridors(network, ptdf; limit=PORTFOLIO_MAX_CORRIDORS) -> Vector{Int} + +The branch dimensions a load bus's sensitivity signature is measured on. + +# Returns +- Row indices into `ptdf.M`, ascending. + +# Notes +One deterministic rule, applied identically to every case: among in-service +branches carrying a finite positive thermal rating, score + +```math +\\rho_l = \\frac{1}{\\overline s_l} + \\sum_{b\\,\\in\\,\\text{load buses}} \\max\\bigl(0,\\, -M_{l,b}\\bigr)\\, p^{d,0}_b , +``` + +the loading this corridor would take on if all the demand that pushes power +THROUGH it were scaled by one, relative to its own rating. It is high exactly +when a corridor is both sensitive and has load behind it — a corridor with a +large sensitivity and no demand on the sending side is not a lever — and it is a +pure function of the network data, so no solve and therefore no solver-dependent +tie enters the panel's identity. + +The `limit` highest scores are kept, ties broken by ascending branch identifier. +""" +function select_corridors(network::AbstractDict, ptdf; limit::Integer = PORTFOLIO_MAX_CORRIDORS) + load_at = nominal_load_at_bus(network) + loadbus = sort!([b for b in keys(load_at) if load_at[b] > 0 && haskey(ptdf.pos, b)]) + scored = Tuple{Float64,Int,Int}[] + for (e, (id, _, _, _)) in enumerate(ptdf.live) + rate = Float64(get(network["branch"][string(id)], "rate_a", Inf)) + (isfinite(rate) && rate > 0) || continue + ρ = sum(max(0.0, -ptdf.M[e, ptdf.pos[b]]) * load_at[b] for b in loadbus; init = 0.0) / rate + ρ > 0 || continue + push!(scored, (ρ, id, e)) + end + isempty(scored) && error("no in-service corridor carries a finite rating and reachable load") + sort!(scored; by = x -> (-x[1], x[2])) + return sort!([e for (_, _, e) in scored[1:min(Int(limit), length(scored))]]) +end + +""" + sensitivity_signatures(network, ptdf, corridors) -> (buses, S, weights) + +Normalized load-bus sensitivity signatures over the selected corridors. + +# Returns +- `buses::Vector{Int}`: the load buses, ascending; row order of `S`. +- `S::Matrix{Float64}`: row `i` is the unit-norm signature of bus `buses[i]`. +- `weights::Vector{Float64}`: nominal active demand per row (pu). + +# Notes +The raw entry is ``-M_{l,b}/\\overline s_l`` — the loading a unit of demand at +`b` puts on corridor `l`, relative to that corridor's own rating, so corridors of +very different ratings are commensurable. Rows are then scaled to unit norm, so +clustering sees the SHAPE of a bus's influence and not its size; size enters the +clustering through the demand WEIGHT instead, which is what keeps a region from +being defined by a crowd of tiny loads. +""" +function sensitivity_signatures(network::AbstractDict, ptdf, corridors::AbstractVector{<:Integer}) + load_at = nominal_load_at_bus(network) + buses = sort!([b for b in keys(load_at) if load_at[b] > 0 && haskey(ptdf.pos, b)]) + S = zeros(length(buses), length(corridors)) + for (j, e) in enumerate(corridors) + rate = Float64(get(network["branch"][string(ptdf.live[e][1])], "rate_a", Inf)) + for (i, b) in enumerate(buses) + S[i, j] = -ptdf.M[e, ptdf.pos[b]] / rate + end + end + for i in axes(S, 1) + n = sqrt(sum(abs2, view(S, i, :))) + n > 1e-12 && (S[i, :] ./= n) + end + return buses, S, [load_at[b] for b in buses] +end + +""" + weighted_kmeans(S, weights, k; iterations=100) -> Vector{Int} + +Demand-weighted k-means with deterministic initialization and explicit tie +breaking. + +# Arguments +- `S::AbstractMatrix`: one signature per ROW. +- `weights::AbstractVector`: nonnegative weight per row. +- `k::Integer`: number of clusters. + +# Returns +- `assign::Vector{Int}`: cluster index per row, in `1:k`. + +# Notes +Determinism is the whole point, so nothing here is left to a default: + +- **initialization** is farthest-point. The first centre is the row of largest + weight (ties: lowest row index); each further centre is the row farthest from + the centres already chosen (ties: lowest row index). No RNG is consulted, so + there is no seed to record and no stream to depend on; +- **assignment** takes the lowest centre index among equal distances; +- **update** is the weight-weighted mean of the members, which is what makes a + region follow the demand rather than the count of buses; +- an **empty** cluster is reseeded to the row that is farthest from its own + centre among clusters that still have at least two members (ties: lowest row + index), so `k` regions are always returned and every region is nonempty; +- the loop stops when no assignment changes, and in any case after `iterations` + sweeps, so it terminates on a case where two configurations alternate. +""" +function weighted_kmeans(S::AbstractMatrix{Float64}, weights::AbstractVector{<:Real}, + k::Integer; iterations::Integer = 100) + n = size(S, 1) + n >= k || error("cannot form $k clusters from $n signatures") + d2(i, c) = sum(abs2, view(S, i, :) .- c) + + centres = Vector{Vector{Float64}}() + first_row = argmax([(Float64(weights[i]), -i) for i in 1:n]) + push!(centres, collect(view(S, first_row, :))) + while length(centres) < k + best_i, best_d = 0, -Inf + for i in 1:n + di = minimum(d2(i, c) for c in centres) + di > best_d && (best_d = di; best_i = i) + end + push!(centres, collect(view(S, best_i, :))) + end + + assign = zeros(Int, n) + for _ in 1:Int(iterations) + changed = false + for i in 1:n + best_j, best_d = 1, Inf + for j in 1:k + dj = d2(i, centres[j]) + dj < best_d && (best_d = dj; best_j = j) + end + best_j == assign[i] || (assign[i] = best_j; changed = true) + end + for j in 1:k + members = findall(==(j), assign) + if isempty(members) + donor, donor_d = 0, -Inf + for i in 1:n + count(==(assign[i]), assign) >= 2 || continue + di = d2(i, centres[assign[i]]) + di > donor_d && (donor_d = di; donor = i) + end + donor == 0 && error("cannot repair an empty cluster") + assign[donor] = j + centres[j] = collect(view(S, donor, :)) + changed = true + continue + end + w = sum(Float64(weights[i]) for i in members) + centres[j] = w > 0 ? + vec(sum(Float64(weights[i]) .* view(S, i, :) for i in members) ./ w) : + vec(sum(view(S, i, :) for i in members) ./ length(members)) + end + changed || break + end + return assign +end + +""" + region_dispersion(S, weights, assign, centres) -> Float64 + +Demand-weighted mean squared PTDF-signature distance to the assigned +representative, + +```math +D = \\frac{\\sum_i w_i \\lVert s_i - c_{a(i)} \\rVert^2}{\\sum_i w_i}. +``` + +# Notes +This is the quantity the balance constraints trade against. Reporting it for the +constrained AND the unconstrained assignment is what keeps the tradeoff visible: +if balancing demand made the regions arbitrary bus partitions, this number would +jump, and the point of the regions — that they are levers on the network's own +binding structure — would be gone. +""" +function region_dispersion(S::AbstractMatrix, weights::AbstractVector, + assign::AbstractVector{<:Integer}, + centres::AbstractVector) + num = 0.0 + den = 0.0 + for i in axes(S, 1) + w = Float64(weights[i]) + num += w * sum(abs2, view(S, i, :) .- centres[assign[i]]) + den += w + end + return num / den +end + +"Demand-weighted centroid of each cluster, in cluster-index order." +function region_centres(S::AbstractMatrix, weights::AbstractVector, + assign::AbstractVector{<:Integer}, k::Integer) + return [begin + m = findall(==(j), assign) + w = sum(Float64(weights[i]) for i in m; init = 0.0) + isempty(m) ? zeros(size(S, 2)) : + w > 0 ? vec(sum(Float64(weights[i]) .* view(S, i, :) for i in m) ./ w) : + vec(sum(view(S, i, :) for i in m) ./ length(m)) + end for j in 1:Int(k)] +end + +""" + balanced_assignment(S, weights, k; share_min, share_max, iterations, optimizer) + -> NamedTuple + +Demand-balanced assignment of signatures to regions. + +# Returns +`(assign, centres, iterations, objective, exception, cap)`. + +# Notes +The unconstrained k-means of the first freeze minimizes signature distance and +lets the demand fall where it may. This keeps the same objective and the same +features, and adds the constraint that makes the regions usable as levers: + +```math +\\min_{x} \\sum_{i,r} w_i \\lVert s_i - c_r \\rVert^2 x_{ir} +\\quad\\text{s.t.}\\quad +\\sum_r x_{ir} = 1,\\; +\\underline s\\,W \\le \\sum_i w_i x_{ir} \\le \\overline s\\,W,\\; +x_{ir} \\in \\{0,1\\}. +``` + +Representatives are refined the way Lloyd's algorithm refines them — assign, +recompute demand-weighted centroids, repeat — but the assignment step is this +integer program rather than a nearest-centre rule, so the demand bounds hold at +every sweep and not merely at the end. Initialization is the same deterministic +farthest-point seeding the unconstrained clustering used. The sweep with the +lowest objective wins, ties going to the earliest, and the loop stops as soon as +an assignment repeats. + +**The single-heavy-bus exception.** A bus is indivisible. If one bus alone +carries more than `share_max` of the demand, no assignment can respect the upper +bound and the program is infeasible as written. The minimum necessary relaxation +is then applied and recorded: the cap rises to that bus's own share, and a +`Σ y_r ≤ 1` constraint permits exactly ONE region to use it. The other five stay +under `share_max`. + +Solved with HiGHS through JuMP — a maintained open-source solver, no proprietary +reproduction requirement. Determinism rests on HiGHS being deterministic for +fixed input, options and thread count, which is why `threads` is pinned to one +and both MIP gaps to zero; the manifest's digests catch any drift. +""" +function balanced_assignment(S::AbstractMatrix{Float64}, weights::AbstractVector{<:Real}, + k::Integer; + share_min::Real = PORTFOLIO_REGION_SHARE_MIN, + share_max::Real = PORTFOLIO_REGION_SHARE_MAX, + iterations::Integer = PORTFOLIO_BALANCE_ITERATIONS, + optimizer = HiGHS.Optimizer) + n = size(S, 1) + K = Int(k) + n >= K || error("cannot form $K regions from $n signatures") + w = Float64.(collect(weights)) + W = sum(w) + W > 0 || error("the total assignment weight is zero") + + # The indivisible-bus exception, decided from the data before any solve. + heaviest = maximum(w) / W + exception = heaviest > share_max + cap = exception ? heaviest : Float64(share_max) + share_min * (K - 1) + cap <= 1.0 + 1e-12 || + error("a single bus carries $(round(100 * heaviest; digits = 2)) % of demand; " * + "$K regions cannot also each hold $(share_min) of it") + + # Deterministic farthest-point seeding, identical to the unconstrained rule. + d2(i, c) = sum(abs2, view(S, i, :) .- c) + centres = Vector{Vector{Float64}}() + push!(centres, collect(view(S, argmax([(w[i], -i) for i in 1:n]), :))) + while length(centres) < K + best_i, best_d = 0, -Inf + for i in 1:n + di = minimum(d2(i, c) for c in centres) + di > best_d && (best_d = di; best_i = i) + end + push!(centres, collect(view(S, best_i, :))) + end + + seen = Set{Vector{Int}}() + best_assign, best_centres, best_obj, sweeps = Int[], centres, Inf, 0 + for it in 1:Int(iterations) + sweeps = it + model = JuMP.Model(optimizer) + JuMP.set_silent(model) + for (opt, val) in ("threads" => 1, "mip_rel_gap" => 0.0, + "mip_abs_gap" => 0.0, "random_seed" => 0) + try + JuMP.set_attribute(model, opt, val) + catch + # An option this solver build does not expose is not a reason to + # stop; determinism is asserted by the regression suite. + end + end + JuMP.@variable(model, x[1:n, 1:K], Bin) + JuMP.@constraint(model, [i = 1:n], sum(x[i, r] for r in 1:K) == 1) + JuMP.@constraint(model, [r = 1:K], sum(w[i] * x[i, r] for i in 1:n) >= share_min * W) + if exception + JuMP.@variable(model, y[1:K], Bin) + JuMP.@constraint(model, sum(y) <= 1) + JuMP.@constraint(model, [r = 1:K], + sum(w[i] * x[i, r] for i in 1:n) <= + share_max * W + y[r] * (cap - share_max) * W) + else + JuMP.@constraint(model, [r = 1:K], sum(w[i] * x[i, r] for i in 1:n) <= cap * W) + end + cost = [w[i] * d2(i, centres[r]) for i in 1:n, r in 1:K] + JuMP.@objective(model, Min, sum(cost[i, r] * x[i, r] for i in 1:n, r in 1:K)) + JuMP.optimize!(model) + JuMP.termination_status(model) in (MOI.OPTIMAL, MOI.LOCALLY_SOLVED) || + error("the balanced assignment did not solve: $(JuMP.termination_status(model))") + + assign = [argmax([JuMP.value(x[i, r]) for r in 1:K]) for i in 1:n] + obj = JuMP.objective_value(model) + if obj < best_obj - 1e-12 + best_obj, best_assign, best_centres = obj, copy(assign), copy(centres) + end + assign in seen && break + push!(seen, copy(assign)) + centres = region_centres(S, w, assign, K) + end + return (assign = best_assign, centres = best_centres, iterations = sweeps, + objective = best_obj, exception = exception, cap = cap) +end + +""" + portfolio_regions(network; nregions=PORTFOLIO_REGIONS) -> NamedTuple + +The case's demand regions, from PTDF sensitivity signatures. + +# Returns +A named tuple with `regions::Vector{Vector{Int}}` (bus identifiers per region, +ascending), `sizes`, `demand_pu`, `demand_share`, `corridors` (branch +IDENTIFIERS, ascending) and `digest`. + +# Notes +Region IDENTITY is canonicalized after clustering: regions are relabelled in +descending total nominal demand, ties broken by the lowest bus identifier they +contain. Without that step the labels would carry the order the initialization +happened to pick centres in, and "region 3" would not mean the same thing to a +reader who reparsed the network. + +Every region is nonempty and contains load by construction — the rows being +clustered are exactly the buses with positive nominal active demand — and the +regions partition those buses, which is what makes each region's support mean +exactly one. +""" +function portfolio_regions(network::AbstractDict; nregions::Integer = PORTFOLIO_REGIONS) + ptdf = ptdf_matrix(network) + corridors = select_corridors(network, ptdf) + buses, S, weights = sensitivity_signatures(network, ptdf, corridors) + + balanced = balanced_assignment(S, weights, nregions) + assign = balanced.assign + + # The unconstrained clustering is still computed, for one reason: it is the + # baseline the balance constraints are traded against, and a tradeoff nobody + # measured is a tradeoff nobody made. + free_assign = weighted_kmeans(S, weights, nregions) + free_centres = region_centres(S, weights, free_assign, nregions) + dispersion = region_dispersion(S, weights, assign, balanced.centres) + free_dispersion = region_dispersion(S, weights, free_assign, free_centres) + + raw = [[buses[i] for i in 1:length(buses) if assign[i] == j] for j in 1:Int(nregions)] + load = [sum(weights[i] for i in 1:length(buses) if assign[i] == j; init = 0.0) + for j in 1:Int(nregions)] + all(!isempty, raw) || error("the balanced assignment left a region empty") + all(>(0), load) || error("a region carries no demand") + order = sort(1:Int(nregions); by = j -> (-load[j], minimum(raw[j]))) + regions = [sort!(raw[j]) for j in order] + demand = [load[j] for j in order] + total = sum(demand) + share = demand ./ total + n_eff = 1 / sum(abs2, share) + ids = [ptdf.live[e][1] for e in corridors] + + # Fail closed on the gate, not on a report of it. The bounds are checked + # against the SOLVED shares rather than against the constraints that were + # written, because a constraint the solver satisfied to its own feasibility + # tolerance is not the same statement as a share that is inside the band. + all(s -> s >= PORTFOLIO_REGION_SHARE_MIN - 1e-9, share) || + error("a region share fell below $PORTFOLIO_REGION_SHARE_MIN: $share") + over = findall(s -> s > PORTFOLIO_REGION_SHARE_MAX + 1e-9, share) + isempty(over) || balanced.exception || + error("a region share exceeded $PORTFOLIO_REGION_SHARE_MAX without a " * + "single-bus exception: $share") + length(over) <= 1 || + error("more than one region exceeded $PORTFOLIO_REGION_SHARE_MAX: $share") + n_eff >= PORTFOLIO_MIN_EFFECTIVE_REGIONS - 1e-9 || balanced.exception || + error("effective region count $n_eff is below $PORTFOLIO_MIN_EFFECTIVE_REGIONS") + + return (regions = regions, + sizes = [length(r) for r in regions], + demand_pu = demand, + demand_share = share, + effective_regions = n_eff, + dispersion = dispersion, + unconstrained_dispersion = free_dispersion, + unconstrained_share = sort( + [sum(weights[i] for i in 1:length(buses) if free_assign[i] == j; init = 0.0) + for j in 1:Int(nregions)] ./ total; rev = true), + exception = balanced.exception, + cap = balanced.cap, + sweeps = balanced.iterations, + corridors = ids, + # Digest tag bumped to /2: the assignment rule changed, so a region + # digest from the first freeze must not be mistaken for one of these. + digest = digest_of("portfolio/regions/2", nregions, join(ids, ","), + join((join(r, ",") for r in regions), ";"))) +end + +# ───────────────────────────────────────────────────────────────────────────── +# E — the finite joint demand support +# ───────────────────────────────────────────────────────────────────────────── + +""" + portfolio_mode_matrix(nregions=PORTFOLIO_REGIONS) -> Matrix{Float64} + +The `(nregions × nregions)` matrix of joint multiplier atoms: atom `r` gives +region `r` the high multiplier and every other region the low one. + +# Notes +With `PORTFOLIO_REGION_HIGH = 1.15`, `PORTFOLIO_REGION_LOW = 0.97` and six +equiprobable atoms every region's support mean is exactly one, + +```math +\\frac{1.15 + 5 \\times 0.97}{6} = \\frac{6.00}{6} = 1 , +``` + +so the uncertainty adds no expected load and the deterministic profile alone +carries the level. The atoms are locationally NEGATIVELY correlated — a region is +high precisely when the other five are low — which is the structure a single +system-wide multiplier cannot express and the reason a policy has to know WHERE +demand went, not only how much of it there is. +""" +function portfolio_mode_matrix(nregions::Integer = PORTFOLIO_REGIONS) + M = fill(PORTFOLIO_REGION_LOW, Int(nregions), Int(nregions)) + for r in 1:Int(nregions) + M[r, r] = PORTFOLIO_REGION_HIGH + end + return M +end + +""" + portfolio_sampler(regions) -> JointRegionMultiplier + +The authoring sampler of the frozen support: one equiprobable joint atom per +region, applied by BUS. +""" +portfolio_sampler(regions::AbstractVector) = + JointRegionMultiplier(regions, portfolio_mode_matrix(length(regions)), + fill(1 / length(regions), length(regions)); by = :bus) + +""" + portfolio_support(network, regions, κ; horizon=PORTFOLIO_HORIZON, seed) -> DemandSupport + +Freeze the panel's finite demand support at demand level `κ`. + +# Notes +The temporal profile handed to `freeze_demand_support` is `κ * PORTFOLIO_PROFILE` +— the calibration scales the COMMON profile rather than the atoms, so every case +faces the identically shaped day and the same six multiplier vectors at every +stage, and only the level differs. The freeze is `:exact`: the sampler declares a +finite support, so the atoms are its own and nothing is resampled or reweighted. +""" +function portfolio_support(network::AbstractDict, regions::AbstractVector, κ::Real; + horizon::Integer = PORTFOLIO_HORIZON, seed::Integer) + return freeze_demand_support(portfolio_sampler(regions), network, Int(horizon); + seed = Int(seed), + method = :exact, + profile = Float64(κ) .* PORTFOLIO_PROFILE, + profile_period = length(PORTFOLIO_PROFILE), + protocol_seed = Int(seed), + stage_hours = PORTFOLIO_STAGE_HOURS) +end + +# ───────────────────────────────────────────────────────────────────────────── +# F — method-independent ACP headroom calibration +# +# The level is calibrated on the UNMODIFIED network with no batteries and no +# policy target, so nothing a method does can move it. What is being asked is a +# property of the benchmark alone: how much of this common profile can this +# system serve, on true ACP, at every atom, without reaching for the recourse +# injections. +# ───────────────────────────────────────────────────────────────────────────── + +""" + portfolio_probe_case(name, network, support) -> BatteryCase + +An in-memory `BatteryCase` carrying no batteries, for the headroom gate. + +# Notes +Built in memory rather than through `build_case` because the gate evaluates a +dozen demand levels and writing a 2000-bus network to disk for each of them +would dominate the calibration. Nothing is frozen here and nothing is hashed: +the FROZEN case still goes through `build_case`, which reads it back through the +verifier. The fleet is empty and every gate solve runs with `mode = :none`, so +this is literally the PGLib network plus the two nodal recourse injections. +""" +function portfolio_probe_case(name::AbstractString, network::AbstractDict, + support::DemandSupport) + manifest = Dict{String,Any}("schema" => BATTERY_MANIFEST_SCHEMA, + "case" => String(name), + "stage_hours" => support.stage_hours) + return BatteryCase(".", String(name), network, BatterySpec[], + recourse_prices(network), support, manifest) +end + +""" + physical_residual(r) -> Float64 + +Collapse the named tuple `physical_residuals` returns into one number. + +# Notes +Its fields are not all of one kind: `branch_flow`, `active_balance`, +`reactive_balance` and `transition` are absolute equation errors and are +nonnegative, while `thermal`, `angle` and `voltage` are signed SLACKS — a +satisfied limit reports a negative number. Taking an absolute value over all +seven would turn the healthiest possible voltage margin into the largest possible +residual, so the slacks are clamped at zero and only a genuine VIOLATION +contributes. +""" +physical_residual(r) = max(r.branch_flow, r.active_balance, r.reactive_balance, + r.transition, max(0.0, r.thermal), max(0.0, r.angle), + max(0.0, r.voltage)) + +""" + admissible_stage(case, t, atom; optimizer=nothing) -> NamedTuple + +Solve one base ACP stage and report whether it clears the headroom gate. + +# Returns +`(ok, solved, status, cost, residual, recourse)`. + +# Notes +Three conditions, all of them fail-closed. The solve must COMPLETE +(`OPTIMAL`/`LOCALLY_SOLVED`); the physical residuals — recomputed from the +solution by `physical_residuals`, independently of whatever the solver reported — +must be at most `PORTFOLIO_RESIDUAL_TOL`; and the worst nodal deficit or surplus +must be at most `PORTFOLIO_RECOURSE_TOL`. A level that needs recourse is a level +at which the case measures the recourse price rather than the network. +""" +function admissible_stage(case::BatteryCase, t::Integer, atom::Integer; optimizer = nothing) + b = base_feasibility(case, PowerModels.ACPPowerModel; + stage = t, atom = atom, optimizer = optimizer) + if !b.solved + return (ok = false, solved = false, status = string(b.status), + cost = NaN, residual = NaN, recourse = NaN) + end + r = physical_residual(b.residuals) + rec = b.worst_recourse + return (ok = r <= PORTFOLIO_RESIDUAL_TOL && rec <= PORTFOLIO_RECOURSE_TOL, + solved = true, status = string(b.status), cost = b.cost_stage, + residual = r, recourse = rec) +end + +""" + peak_stage(profile=PORTFOLIO_PROFILE) -> Int + +The FIRST stage attaining the profile's maximum. Three stages of the common +profile reach 1.00; taking the first makes the choice a rule rather than a +preference. +""" +peak_stage(profile::AbstractVector = PORTFOLIO_PROFILE) = argmax(profile) + +""" + trough_stage(profile=PORTFOLIO_PROFILE) -> Int + +The FIRST stage attaining the profile's minimum. +""" +trough_stage(profile::AbstractVector = PORTFOLIO_PROFILE) = argmin(profile) + +""" + gate_stages(profile=PORTFOLIO_PROFILE) -> Vector{Int} + +The stages the headroom search evaluates: the profile's peak AND its trough. + +# Notes +The search cannot run on the peak alone. The profile takes the day down to +`0.64` of the calibrated level, and several PGLib systems cannot follow it: their +in-service generators carry a positive `pmin`, an OPF has no unit commitment to +switch one off, and at low demand the system over-generates and the SURPLUS +injection takes the difference. That is exactly the "meaningful recourse" the +gate exists to forbid, and a level chosen on the peak alone would fail the +`24 × 6` verification at the trough instead. + +Both extremes, then, and both at every atom. The interior stages are not implied +by the extremes — admissibility is not monotone in the level — which is why the +full grid is still verified afterwards; what the two extremes buy is that the +verification almost always passes, rather than rejecting a level the search had +no way of knowing was bad. +""" +gate_stages(profile::AbstractVector = PORTFOLIO_PROFILE) = + sort!(unique([peak_stage(profile), trough_stage(profile)])) + +""" + admissible_level(name, network, regions, κ; stages, seed, optimizer=nothing) + -> NamedTuple + +Whether EVERY atom of every gate stage clears the gate at demand level `κ`. + +# Notes +Short-circuits on the first combination that fails, because the gate is a +conjunction and the search only needs the verdict. The stage and atom that failed +are returned, which is what a replacement decision has to be able to cite. +""" +function admissible_level(name::AbstractString, network::AbstractDict, + regions::AbstractVector, κ::Real; + stages::AbstractVector{<:Integer} = gate_stages(), + seed::Integer, optimizer = nothing) + support = portfolio_support(network, regions, κ; seed = seed) + case = portfolio_probe_case(name, network, support) + for t in stages, a in 1:num_atoms(support, Int(t)) + r = admissible_stage(case, t, a; optimizer = optimizer) + r.ok || return (ok = false, stage = Int(t), atom = a, detail = r) + end + return (ok = true, stage = 0, atom = 0, + detail = (ok = true, solved = true, status = "OPTIMAL", + cost = NaN, residual = NaN, recourse = NaN)) +end + +""" + calibrate_kappa(name, network, regions; seed, optimizer=nothing, log=true) + -> NamedTuple + +The case's method-independent demand-level calibration. + +# Returns +`(ok, kappa_max, kappa_case, evaluations, trace, reason)`. `ok = false` means the +case fails the base-ACP data gate at `PORTFOLIO_KAPPA_LO` and is replaced from +the reserve panel. + +# Notes +The search is [`search_kappa_max`](@ref): a downward scan of a fixed grid to +bracket the largest admissible level, then bisection to `PORTFOLIO_KAPPA_TOL`. +The reported `kappa_max` is the largest level actually PROVEN admissible, never +the midpoint of a bracket, so the frozen level is backed by a solve rather than +by an interpolation. + +The frozen level is then + +```math +\\kappa_{case} = 0.95\\, \\kappa_{\\max}, +``` + +with the margin a constant of this file. No solver setting is touched: the same +optimizer, the same tolerances and the same formulation are used for every case +and every level, because a per-case solver adjustment would make the calibration +a property of the solver rather than of the benchmark. +""" +function calibrate_kappa(name::AbstractString, network::AbstractDict, + regions::AbstractVector; seed::Integer, + optimizer = nothing, log::Bool = true) + ts = gate_stages() + trace = Dict{String,Any}[] + evals = Ref(0) + function gate(κ) + r = admissible_level(name, network, regions, κ; stages = ts, seed = seed, + optimizer = optimizer) + evals[] += 1 + push!(trace, Dict{String,Any}("kappa" => Float64(κ), "admissible" => r.ok, + "first_failing_stage" => r.stage, + "first_failing_atom" => r.atom, + "status" => r.detail.status, + "residual" => r.detail.residual, + "recourse" => r.detail.recourse)) + log && @printf(" κ=%.6f %s%s\n", κ, r.ok ? "admissible" : "REJECTED", + r.ok ? "" : @sprintf(" (stage %d atom %d, %s, residual %.2e, recourse %.2e)", + r.stage, r.atom, r.detail.status, + r.detail.residual, r.detail.recourse)) + return r.ok + end + + r = search_kappa_max(gate) + r.ok || return (ok = false, kappa_max = NaN, kappa_case = NaN, + evaluations = evals[], trace = trace, reason = r.reason) + return (ok = true, kappa_max = r.kappa_max, + kappa_case = PORTFOLIO_KAPPA_MARGIN * r.kappa_max, + evaluations = evals[], trace = trace, reason = "") +end + +""" + kappa_grid(lo=PORTFOLIO_KAPPA_LO, hi=PORTFOLIO_KAPPA_HI, + step=PORTFOLIO_KAPPA_STEP) -> Vector{Float64} + +The bracketing grid, DESCENDING from `hi`, with `lo` exactly as its last point. + +# Notes +Built by subtracting whole multiples of `step` from `hi` rather than by +accumulating additions, so the same values come out on any machine, and the last +point is written as `lo` rather than computed, so the bottom of the bracket is +the number the constant says it is and not one ulp away from it. +""" +function kappa_grid(lo::Real = PORTFOLIO_KAPPA_LO, hi::Real = PORTFOLIO_KAPPA_HI, + step::Real = PORTFOLIO_KAPPA_STEP) + n = round(Int, (hi - lo) / step) + g = [Float64(hi) - k * Float64(step) for k in 0:n] + g[end] = Float64(lo) + return g +end + +""" + search_kappa_max(gate; lo=PORTFOLIO_KAPPA_LO, hi=PORTFOLIO_KAPPA_HI, + step=PORTFOLIO_KAPPA_STEP, tol=PORTFOLIO_KAPPA_TOL) + -> NamedTuple + +The panel's deterministic headroom search, isolated from what it is searching on. + +# Arguments +- `gate`: a predicate `κ -> Bool` saying whether the case is admissible at that + demand level. + +# Returns +`(ok, kappa_max, reason)`. `ok = false` means NO point of the grid is +admissible, which is the preregistered replacement condition. + +# Notes +Two steps, both deterministic: + +1. **bracket** — walk [`kappa_grid`](@ref) downward from `hi` and stop at the + first admissible point. Downward, because the quantity wanted is the LARGEST + admissible level; and by scan rather than by bisection, because admissibility + is not monotone in the demand level (see `PORTFOLIO_KAPPA_STEP`). +2. **refine** — bisect between that point and the inadmissible grid point + immediately above it, until the bracket is narrower than `tol`. + +The returned level is always one the gate ACCEPTED, never a midpoint that was +merely bracketed. Separated from [`calibrate_kappa`](@ref) so the search itself +can be tested against a predicate with a known threshold rather than only +through hundreds of ACP solves. +""" +function search_kappa_max(gate; lo::Real = PORTFOLIO_KAPPA_LO, + hi::Real = PORTFOLIO_KAPPA_HI, + step::Real = PORTFOLIO_KAPPA_STEP, + tol::Real = PORTFOLIO_KAPPA_TOL) + grid = kappa_grid(lo, hi, step) + for (i, κ) in enumerate(grid) + gate(κ) || continue + i == 1 && return (ok = true, kappa_max = κ, reason = "") + a, b = κ, grid[i - 1] + while b - a > tol + mid = 0.5 * (a + b) + gate(mid) ? (a = mid) : (b = mid) + end + return (ok = true, kappa_max = a, reason = "") + end + return (ok = false, kappa_max = NaN, + reason = "the unmodified base ACP clears the gate at no level of " * + "[$lo, $hi]; the conservative level $lo fails") +end + +""" + verify_kappa(name, network, regions, κ; seed, optimizer=nothing, log=true) + -> NamedTuple + +Check EVERY stage/atom combination at the frozen level. + +# Returns +`(ok, checked, failures)` where `failures` lists `(stage, atom, status, residual, +recourse)` for each combination that did not clear the gate. + +# Notes +The search runs on the peak stage alone; this runs on all `24 × 6` of them. Lower +profile stages are not automatically easier — a lightly loaded system can be the +one that has to spill — so the verification is exhaustive rather than argued. +""" +function verify_kappa(name::AbstractString, network::AbstractDict, + regions::AbstractVector, κ::Real; seed::Integer, + optimizer = nothing, log::Bool = true) + support = portfolio_support(network, regions, κ; seed = seed) + case = portfolio_probe_case(name, network, support) + failures = Dict{String,Any}[] + checked = 0 + for t in 1:support.horizon, a in 1:num_atoms(support, t) + r = admissible_stage(case, t, a; optimizer = optimizer) + checked += 1 + r.ok || push!(failures, Dict{String,Any}("stage" => t, "atom" => a, + "status" => r.status, + "residual" => r.residual, + "recourse" => r.recourse)) + end + log && @printf(" verified %d stage/atom combinations, %d failures\n", + checked, length(failures)) + return (ok = isempty(failures), checked = checked, failures = failures) +end + +# ───────────────────────────────────────────────────────────────────────────── +# G — building one frozen portfolio case +# ───────────────────────────────────────────────────────────────────────────── + +""" + build_portfolio_case(name; dir, kappa=nothing, verify_all=true, + optimizer=nothing, quiet=false) -> NamedTuple + +Construct one frozen portfolio case from its PGLib name. + +# Keywords +- `dir::AbstractString`: where the four case artifacts are written. +- `kappa`: `nothing` to calibrate the demand level here, or the frozen + `κ_case` from the portfolio manifest to reproduce a case without repeating the + search. +- `verify_all::Bool`: check all `24 × 6` stage/atom combinations at the frozen + level. + +# Returns +A named tuple with the read-back `case`, the `record` the portfolio manifest +stores, and the intermediate `regions`, `placement` and `calibration` results. + +# Notes +The order of construction is forced by what depends on what: regions are needed +before the support, the support before the headroom gate, the gate before the +peak demand, and the peak demand before the storage ratings. Placement does not +depend on any of them — it is a function of the network alone — but the fleet's +ratings do, which is why the buses are drawn early and rated late. +""" +function build_portfolio_case(name::AbstractString; + dir::AbstractString, + kappa = nothing, + verify_all::Bool = true, + optimizer = nothing, + regions = nothing, + quiet::Bool = false) + src = acquire_pglib_case(name) + sha = network_digest(src.network) + quiet || @printf(" %s: network sha256 %s\n", name, first(sha, 16)) + + # The regions may be supplied by a caller that has already computed them. + # This is a pure cache, never a second source of truth: the balanced + # assignment is an integer program that takes 38 minutes on `case2000_goc`, + # and a reader who regenerates a case should pay for it ONCE rather than + # once for the digest check and again for the build. + regions = regions === nothing ? portfolio_regions(src.network) : regions + buses, placement = portfolio_placement(src.network, sha) + final_seed = portfolio_seed_for("protocol/final", sha) + screen_seed = portfolio_seed_for("protocol/screening", sha) + + calibration = if kappa === nothing + quiet || println(" calibrating the ACP headroom") + calibrate_kappa(name, src.network, regions.regions; + seed = final_seed, optimizer = optimizer, log = !quiet) + else + (ok = true, kappa_max = Float64(kappa) / PORTFOLIO_KAPPA_MARGIN, + kappa_case = Float64(kappa), evaluations = 0, + trace = Dict{String,Any}[], reason = "taken from the portfolio manifest") + end + calibration.ok || + return (ok = false, case = nothing, record = nothing, regions = regions, + placement = placement, calibration = calibration, + verification = nothing) + + # The frozen level, and the retreat if the full grid rejects it. `retreats` + # counts the grid steps taken; a case that exhausts them fails the gate and + # is recorded, never quietly frozen at a level the verification rejected. + κmax = calibration.kappa_max + κ = calibration.kappa_case + retreats = 0 + attempts = Dict{String,Any}[] + verification = (ok = true, checked = 0, failures = Dict{String,Any}[]) + if verify_all + while true + verification = verify_kappa(name, src.network, regions.regions, κ; + seed = final_seed, optimizer = optimizer, + log = !quiet) + push!(attempts, Dict{String,Any}("kappa_max" => κmax, "kappa_case" => κ, + "retreat" => retreats, + "checked" => verification.checked, + "failures" => verification.failures)) + verification.ok && break + retreats += 1 + quiet || @printf(" verification rejected κ_case=%.6f on %d of %d combinations; retreating one grid step\n", + κ, length(verification.failures), verification.checked) + retreats <= PORTFOLIO_KAPPA_RETREATS || break + κmax -= PORTFOLIO_KAPPA_STEP + κ = PORTFOLIO_KAPPA_MARGIN * κmax + κmax >= PORTFOLIO_KAPPA_LO || + return (ok = false, case = nothing, record = nothing, regions = regions, + placement = placement, + calibration = (ok = false, kappa_max = NaN, kappa_case = NaN, + evaluations = calibration.evaluations, + trace = calibration.trace, + reason = "retreating below $PORTFOLIO_KAPPA_LO"), + verification = verification) + end + end + verification.ok || + return (ok = false, case = nothing, record = nothing, regions = regions, + placement = placement, + calibration = (ok = false, kappa_max = NaN, kappa_case = NaN, + evaluations = calibration.evaluations, + trace = calibration.trace, + reason = "no level within $PORTFOLIO_KAPPA_RETREATS grid " * + "steps of κ_max cleared all 24×6 stage/atom " * + "combinations; the last rejected " * + "$(length(verification.failures)) of " * + "$(verification.checked)"), + verification = verification) + + load_at = nominal_load_at_bus(src.network) + peak = κ * maximum(PORTFOLIO_PROFILE) * sum(values(load_at)) + fleet, ratings = portfolio_fleet(src.network, buses, peak) + support = portfolio_support(src.network, regions.regions, κ; seed = final_seed) + + case = build_case(src; dir = dir, batteries = fleet, support = support, + placement = Dict{String,Any}("buses" => placement, + "capacity" => ratings, + "regions" => Dict{String,Any}( + "count" => length(regions.regions), + "corridors" => regions.corridors, + "sizes" => regions.sizes, + "demand_pu" => regions.demand_pu, + "digest" => regions.digest), + # κ_case ONLY. The largest + # admissible level is a + # byproduct of the search and + # belongs to the portfolio + # manifest; putting it in a + # HASHED case artifact would + # mean a reader who + # regenerates the case from + # the frozen κ_case has to + # reproduce κ_max = κ_case/0.95 + # to the last bit, and float + # division does not promise + # that. + "kappa" => Dict{String,Any}( + "kappa_case" => κ, + "margin" => PORTFOLIO_KAPPA_MARGIN)), + protocol_stages = PORTFOLIO_HORIZON, + protocol_scenarios = PORTFOLIO_FINAL_SCENARIOS, + screening_seed = screen_seed, + screening_scenarios = PORTFOLIO_SCREENING_SCENARIOS, + quiet = true) + + record = Dict{String,Any}( + "case" => String(name), + "network_sha256" => sha, + "counts" => Dict{String,Any}( + "bus" => length(src.network["bus"]), + "gen" => length(src.network["gen"]), + "branch" => length(src.network["branch"]), + "load" => length(src.network["load"]), + "eligible_bus" => length(portfolio_eligible_buses(src.network)), + "battery" => length(fleet)), + "kappa_max" => κmax, + "kappa_case" => κ, + "kappa_evaluations" => calibration.evaluations, + "kappa_retreats" => retreats, + "peak_demand_pu" => peak, + "nominal_demand_pu" => sum(values(load_at)), + "placement" => Dict{String,Any}( + "algorithm" => placement["algorithm"], + "count" => placement["count"], + "eligible" => length(placement["eligible"]), + "buses" => buses, + "digest" => placement["digest"]), + "battery" => Dict{String,Any}( + "power_budget_pu" => ratings["power_budget_pu"], + "power_min_pu" => minimum(values(ratings["power_pu"])), + "power_max_pu" => maximum(values(ratings["power_pu"])), + "energy_budget_puh" => ratings["power_budget_pu"] * PORTFOLIO_DURATION_HOURS, + "digest" => ratings["digest"]), + "regions" => Dict{String,Any}( + "count" => length(regions.regions), + "corridors" => regions.corridors, + "sizes" => regions.sizes, + "demand_pu" => regions.demand_pu, + "demand_share" => regions.demand_share, + "effective_regions" => regions.effective_regions, + # The tradeoff, recorded next to the thing it was traded for. + "dispersion" => regions.dispersion, + "unconstrained_dispersion" => regions.unconstrained_dispersion, + "unconstrained_share" => regions.unconstrained_share, + "single_bus_exception" => regions.exception, + "share_cap" => regions.cap, + "sweeps" => regions.sweeps, + "digest" => regions.digest), + "support" => Dict{String,Any}("sha256" => support_digest(support)), + "protocols" => Dict{String,Any}( + "final" => Dict{String,Any}( + "seed" => final_seed, + "num_stages" => PORTFOLIO_HORIZON, + "num_scenarios" => PORTFOLIO_FINAL_SCENARIOS, + "sha256" => case.manifest["protocol"]["sha256"]), + "screening" => Dict{String,Any}( + "seed" => screen_seed, + "num_stages" => PORTFOLIO_HORIZON, + "num_scenarios" => PORTFOLIO_SCREENING_SCENARIOS, + "excludes" => "final", + "sha256" => case.manifest["screening"]["sha256"])), + "artifacts" => Dict{String,Any}(case.manifest["artifacts"]), + "verification" => Dict{String,Any}( + "stage_atom_checked" => verification.checked, + "stage_atom_failures" => length(verification.failures), + "retreats" => retreats), + ) + return (ok = true, case = case, record = record, regions = regions, + placement = placement, calibration = calibration, + verification = verification, attempts = attempts) +end + +# ───────────────────────────────────────────────────────────────────────────── +# G2 — strict-stage validation of a frozen case +# +# The headroom calibration answers a question about the NETWORK. This answers the +# question about the CASE: with the fleet in place, is the strict formulation +# well posed at every stage — is the state admissible, is the reachable interval +# nonempty, is holding the state feasible, and does an ordinary strict solve +# complete without reaching for recourse. +# ───────────────────────────────────────────────────────────────────────────── + +""" + hold_trajectory(case) -> (states, ok, worst_slack) + +Follow the "do nothing" policy — every battery targets the energy it came in +with — and report whether that target is reachable at every stage. + +# Returns +- `states::Vector{Dict{Int,Float64}}`: the incoming state at each stage; with a + hold policy they are all the initial state. +- `ok::Bool`: every reachable interval was nonempty and contained the hold target. +- `worst_slack::Float64`: the smallest distance from a hold target to the nearer + end of its reachable interval, over all stages and batteries. Negative means + the hold target was outside. + +# Notes +Holding is not free: self-discharge means a battery must charge +``(1-\\alpha)e`` every stage just to stand still, so a hold target is only +reachable when the charging rating covers that. With +``\\alpha = `` `PORTFOLIO_SELF_DISCHARGE` and a duration of +`PORTFOLIO_DURATION_HOURS` hours it does, by a wide margin — but the margin is +measured here rather than argued, because it is a joint property of three +constants that a later revision could change one of. +""" +function hold_trajectory(case::BatteryCase) + Δt = stage_hours(case) + e = Dict(b.index => b.energy_initial for b in case.batteries) + states = Dict{Int,Float64}[] + ok = true + worst = Inf + for _ in 1:case.demand.horizon + push!(states, copy(e)) + for b in case.batteries + lo, hi = reachable_interval(b, e[b.index], Δt) + hi >= lo || (ok = false) + worst = min(worst, e[b.index] - lo, hi - e[b.index]) + (lo <= e[b.index] <= hi) || (ok = false) + end + end + return states, ok, worst +end + +""" + validate_portfolio_case(case; stages=nothing, atoms=nothing, optimizer=nothing, + quiet=false) -> NamedTuple + +The strict-stage validation every frozen case must pass. + +# Keywords +- `stages`, `atoms`: which combinations to solve. `nothing` means all of them. + +# Returns +A named tuple with `ok`, the state and reachability verdicts, the number of +strict solves attempted and completed, the worst residual and recourse over the +completed ones, and the list of failures. + +# Notes +Six properties, each fail-closed: + +1. every initial energy lies inside its own bounds; +2. every reachable interval is nonempty at every stage of the hold trajectory; +3. the idle/hold target lies inside that interval; +4. every requested strict-stage ACP solve completes; +5. the independently recomputed physical residual is at most + `PORTFOLIO_RESIDUAL_TOL`; +6. no solve uses a meaningful nodal deficit or surplus. + +Properties 5 and 6 are recomputed from the returned solution rather than read +from the solver, because a solver that reports success is asserting its own +convergence criterion and not the network's equations. +""" +function validate_portfolio_case(case::BatteryCase; + stages = nothing, atoms = nothing, + optimizer = nothing, quiet::Bool = false) + opt = optimizer === nothing ? acp_optimizer() : optimizer + Δt = stage_hours(case) + failures = Dict{String,Any}[] + + states_ok = all(b.energy_min <= b.energy_initial <= b.energy_max for b in case.batteries) + states_ok || push!(failures, Dict{String,Any}("kind" => "initial state out of bounds")) + _, reach_ok, slack = hold_trajectory(case) + reach_ok || push!(failures, Dict{String,Any}("kind" => "hold target unreachable", + "slack" => slack)) + + ts = stages === nothing ? (1:case.demand.horizon) : stages + hold = Dict(b.index => b.energy_initial for b in case.batteries) + attempted, completed = 0, 0 + worst_res, worst_rec = 0.0, 0.0 + for t in ts + as = atoms === nothing ? (1:num_atoms(case.demand, t)) : atoms + for a in as + attempted += 1 + sol = solve_strict_stage(case, PowerModels.ACPPowerModel; + stage = t, atom = a, energy_in = hold, + target = hold, optimizer = opt) + if !sol.solved + push!(failures, Dict{String,Any}("kind" => "strict solve incomplete", + "stage" => t, "atom" => a, + "status" => string(sol.status))) + continue + end + completed += 1 + r = physical_residual(physical_residuals(case.network, case.batteries, Δt, sol)) + rec = worst_recourse(sol) + worst_res = max(worst_res, r) + worst_rec = max(worst_rec, rec) + r <= PORTFOLIO_RESIDUAL_TOL || + push!(failures, Dict{String,Any}("kind" => "residual above tolerance", + "stage" => t, "atom" => a, "residual" => r)) + rec <= PORTFOLIO_RECOURSE_TOL || + push!(failures, Dict{String,Any}("kind" => "recourse used", + "stage" => t, "atom" => a, "recourse" => rec)) + end + end + quiet || @printf(" strict stages: %d/%d complete, worst residual %.2e, worst recourse %.2e\n", + completed, attempted, worst_res, worst_rec) + return (ok = isempty(failures), states_ok = states_ok, reachable_ok = reach_ok, + hold_slack = slack, attempted = attempted, completed = completed, + worst_residual = worst_res, worst_recourse = worst_rec, failures = failures) +end + +""" + aggressive_charge_probe(case; stage=peak_stage(), atom=1, optimizer=nothing) + -> NamedTuple + +The deliberately aggressive diagnostic: every battery targets the TOP of its +reachable interval at one stage. + +# Returns +`(solved, recourse, residual, rejected)`; `rejected` is `true` when the solve +used more recourse than `PORTFOLIO_RECOURSE_TOL`. + +# Notes +This is a DIAGNOSTIC, not a gate. Charging the whole fleet at full rate on one +stage is an admissible target — it is inside the reachable interval by +construction — that the NETWORK may not be able to serve, and when it cannot, the +right outcome is that the admissibility check rejects the solution. A case is +never altered because this probe draws recourse; a probe that never drew any +would mean the fleet was too small to matter. +""" +function aggressive_charge_probe(case::BatteryCase; stage::Integer = peak_stage(), + atom::Integer = 1, optimizer = nothing) + opt = optimizer === nothing ? acp_optimizer() : optimizer + Δt = stage_hours(case) + e_in = Dict(b.index => b.energy_initial for b in case.batteries) + target = Dict{Int,Float64}() + for b in case.batteries + _, hi = reachable_interval(b, e_in[b.index], Δt) + target[b.index] = hi + end + sol = solve_strict_stage(case, PowerModels.ACPPowerModel; + stage = stage, atom = atom, energy_in = e_in, + target = target, optimizer = opt) + sol.solved || return (solved = false, recourse = NaN, residual = NaN, rejected = true) + rec = worst_recourse(sol) + r = physical_residual(physical_residuals(case.network, case.batteries, Δt, sol)) + return (solved = true, recourse = rec, residual = r, + rejected = rec > PORTFOLIO_RECOURSE_TOL) +end + +# ───────────────────────────────────────────────────────────────────────────── +# H — the portfolio manifest +# ───────────────────────────────────────────────────────────────────────────── + +""" + portfolio_manifest_path(dir=@__DIR__) -> String + +Path of the byte-identical portfolio manifest inside an example directory. +""" +portfolio_manifest_path(dir::AbstractString = @__DIR__) = + joinpath(dir, "battery_portfolio.json") + +""" + portfolio_digest(manifest) -> String + +SHA-256 of the canonical JSON of a manifest with its own `digest` field removed. + +# Notes +A manifest that hashed itself including the hash could not be checked, so the +field is excluded and nothing else is. Any edit to any other field — a `κ`, one +bus of one placement, a protocol seed — changes the digest, which is what makes +[`verify_portfolio_manifest`](@ref) a tamper check and not a formatting check. +""" +function portfolio_digest(manifest::AbstractDict) + body = Dict{String,Any}(k => v for (k, v) in manifest if k != "digest") + return bytes2hex(sha256(canonical_json(plain(body)))) +end + +""" + write_portfolio_manifest(path, cases; replacements=[]) -> Dict + +Write the panel manifest as canonical JSON and return it. + +# Arguments +- `cases`: the per-case records returned by [`build_portfolio_case`](@ref), in + panel order. +- `replacements`: one record per primary case that was replaced, each carrying + the case dropped, the reserve case promoted and the reason. + +# Notes +The manifest is deliberately SMALL: it records what cannot be recomputed (the +calibrated levels, the panel order and the replacements) plus the digest of +everything that can, so a reader regenerates the artifacts and checks them +against the record rather than downloading them. Region MEMBERSHIP is a +recomputable quantity and only its digest, sizes and demand shares are stored; +the selected storage buses are stored in full because they are the panel's most +consequential single choice and a reader should be able to read them without +running anything. + +Nothing here is a timestamp, a hostname or a path, so two machines that freeze +the same panel write the same bytes. +""" +function write_portfolio_manifest(path::AbstractString, cases::AbstractVector; + replacements::AbstractVector = Any[], + unfilled::AbstractVector = Any[], + min_cases::Integer = PORTFOLIO_MIN_CASES) + length(cases) >= min_cases || + error("the panel freezes at least $min_cases cases, got $(length(cases))") + manifest = Dict{String,Any}( + "schema" => PORTFOLIO_SCHEMA, + "seed" => PORTFOLIO_SEED, + "horizon" => PORTFOLIO_HORIZON, + "stage_hours" => PORTFOLIO_STAGE_HOURS, + "profile" => copy(PORTFOLIO_PROFILE), + "panel" => Dict{String,Any}( + "primary" => copy(PORTFOLIO_PRIMARY), + "reserve" => copy(PORTFOLIO_RESERVE), + "accepted" => [String(c["case"]) for c in cases], + "replacements" => collect(replacements), + # Preregistered slots that no case could fill, because the primary + # failed the base-ACP data gate and the reserve list ran out. They + # are recorded rather than backfilled: a case chosen outside the + # preregistered order to make the count come out right is exactly + # the selection freedom the preregistration removes. + "unfilled" => collect(unfilled), + "replacement_rule" => "a primary case is replaced only when it is unavailable " * + "from the pinned PGLib version or when its unmodified base " * + "ACP fails at demand level $PORTFOLIO_KAPPA_LO; a SOC, DC " * + "or method failure is never a reason to replace a case"), + "regions" => Dict{String,Any}( + "count" => PORTFOLIO_REGIONS, + "high" => PORTFOLIO_REGION_HIGH, + "low" => PORTFOLIO_REGION_LOW, + "max_corridors" => PORTFOLIO_MAX_CORRIDORS, + "algorithm" => "demand-BALANCED assignment over unit-norm PTDF sensitivity " * + "signatures on the highest-reach rated corridors: farthest-point " * + "initialization, then Lloyd sweeps whose assignment step is a " * + "HiGHS integer program minimizing demand-weighted signature " * + "distance subject to per-region demand-share bounds; regions " * + "relabelled by descending demand", + "share_min" => PORTFOLIO_REGION_SHARE_MIN, + "share_max" => PORTFOLIO_REGION_SHARE_MAX, + "min_effective_regions" => PORTFOLIO_MIN_EFFECTIVE_REGIONS, + "balance_iterations" => PORTFOLIO_BALANCE_ITERATIONS, + "modes" => [PORTFOLIO_REGION_HIGH, PORTFOLIO_REGION_LOW], + "probabilities" => fill(1 / PORTFOLIO_REGIONS, PORTFOLIO_REGIONS), + "support_mean" => (PORTFOLIO_REGION_HIGH + + (PORTFOLIO_REGIONS - 1) * PORTFOLIO_REGION_LOW) / PORTFOLIO_REGIONS), + "battery" => Dict{String,Any}( + "system_power_share" => PORTFOLIO_POWER_SHARE, + "weight_cap" => PORTFOLIO_WEIGHT_CAP, + "duration_hours" => PORTFOLIO_DURATION_HOURS, + "reserve_fraction" => PORTFOLIO_RESERVE_FRACTION, + "initial_fraction" => PORTFOLIO_INITIAL_FRACTION, + "charge_efficiency" => PORTFOLIO_CHARGE_EFFICIENCY, + "discharge_efficiency" => PORTFOLIO_DISCHARGE_EFFICIENCY, + "self_discharge" => PORTFOLIO_SELF_DISCHARGE, + "throughput_cost" => PORTFOLIO_THROUGHPUT_COST), + "calibration" => Dict{String,Any}( + "bracket" => [PORTFOLIO_KAPPA_LO, PORTFOLIO_KAPPA_HI], + "tolerance" => PORTFOLIO_KAPPA_TOL, + "margin" => PORTFOLIO_KAPPA_MARGIN, + "residual_tolerance" => PORTFOLIO_RESIDUAL_TOL, + "recourse_tolerance" => PORTFOLIO_RECOURSE_TOL, + "step" => PORTFOLIO_KAPPA_STEP, + "gate_stages" => gate_stages(), + "formulation" => "PowerModels.ACPPowerModel, unmodified network, no batteries"), + "protocols" => Dict{String,Any}( + "screening_scenarios" => PORTFOLIO_SCREENING_SCENARIOS, + "final_scenarios" => PORTFOLIO_FINAL_SCENARIOS, + "num_stages" => PORTFOLIO_HORIZON, + "algorithm" => "stage-major StableRNG draw from the frozen support; the " * + "screening protocol is drawn from an independent seed and " * + "repaired against the final protocol's columns, so the two " * + "panels are disjoint by construction", + "seed_rule" => "SHA256(\"$PORTFOLIO_SCHEMA\\nprotocol/\\n$PORTFOLIO_SEED\\n" * + "\\nseed\\n\") reduced into 1:2^31-1"), + "versions" => portfolio_versions(), + "command" => PORTFOLIO_COMMAND, + "cases" => collect(cases), + ) + manifest["digest"] = portfolio_digest(manifest) + write_canonical_json(path, manifest) + return manifest +end + +""" + portfolio_versions() -> Dict{String,Any} + +The package identities the panel's bytes came from. + +# Notes +Every version is recorded as a STRING. `_package_version` returns a +`VersionNumber`, which prints one way and serializes another; the manifest is +hashed through `canonical_json`, so anything it carries has to be a JSON scalar +and not a Julia type that happens to have a `show` method. +""" +function portfolio_versions() + out = Dict{String,Any}("julia" => string(VERSION)) + for p in ("PGLib", "PowerModels", "SDDP", "JuMP", "Ipopt", "Clarabel", "StableRNGs") + v = _package_version(p) + out[p] = v === nothing ? "" : string(v) + end + return out +end + +""" + read_portfolio_manifest(path=portfolio_manifest_path()) -> Dict + +Read a portfolio manifest and check its schema and its self-digest. +""" +function read_portfolio_manifest(path::AbstractString = portfolio_manifest_path()) + isfile(path) || error("no portfolio manifest at $path") + m = JSON.parsefile(path) + m["schema"] == PORTFOLIO_SCHEMA || + error("unexpected portfolio schema $(m["schema"]); expected $PORTFOLIO_SCHEMA") + got = portfolio_digest(m) + got == m["digest"] || + error("portfolio manifest digest $got does not match the recorded $(m["digest"])") + return m +end + +""" + verify_portfolio_manifest(path=portfolio_manifest_path()) -> Dict + +Check a portfolio manifest against everything it can be checked against WITHOUT +solving anything: its self-digest, the preregistered constants it was frozen +under, the panel size, and the internal consistency of each case record. + +# Notes +This is the cheap check a reader runs first. It does not acquire a PGLib case and +does not touch a solver, so it cannot confirm that the recorded digests are the +digests of the artifacts the constants produce — that is what +[`materialize_portfolio_case`](@ref) does, one case at a time. +""" +function verify_portfolio_manifest(path::AbstractString = portfolio_manifest_path(); + min_cases::Integer = PORTFOLIO_MIN_CASES) + m = read_portfolio_manifest(path) + m["seed"] == PORTFOLIO_SEED || error("manifest seed $(m["seed"]) is not $PORTFOLIO_SEED") + m["horizon"] == PORTFOLIO_HORIZON || error("manifest horizon is not $PORTFOLIO_HORIZON") + Float64.(m["profile"]) == PORTFOLIO_PROFILE || + error("manifest profile differs from the preregistered common profile") + m["regions"]["count"] == PORTFOLIO_REGIONS || error("manifest region count is not $PORTFOLIO_REGIONS") + isapprox(Float64(m["regions"]["support_mean"]), 1.0; atol = 1e-15) || + error("the recorded atom mean is $(m["regions"]["support_mean"]), not 1") + length(m["cases"]) >= min_cases || + error("the panel has $(length(m["cases"])) cases, fewer than $min_cases") + for c in m["cases"] + c["counts"]["battery"] == length(c["placement"]["buses"]) || + error("$(c["case"]): battery count disagrees with the placement") + allunique(c["placement"]["buses"]) || + error("$(c["case"]): the placement repeats a bus") + issorted(c["placement"]["buses"]) || + error("$(c["case"]): the placement is not sorted") + expected = min(c["counts"]["eligible_bus"], + clamp(round(Int, 0.20 * c["counts"]["bus"]), 24, 240)) + c["counts"]["battery"] == expected || + error("$(c["case"]): $(c["counts"]["battery"]) batteries where the rule gives $expected") + isapprox(Float64(c["kappa_case"]), + PORTFOLIO_KAPPA_MARGIN * Float64(c["kappa_max"]); rtol = 1e-12) || + error("$(c["case"]): κ_case is not $(PORTFOLIO_KAPPA_MARGIN) × κ_max") + PORTFOLIO_KAPPA_LO <= Float64(c["kappa_max"]) <= PORTFOLIO_KAPPA_HI || + error("$(c["case"]): κ_max is outside the preregistered bracket") + sum(Int.(c["regions"]["sizes"])) > 0 || error("$(c["case"]): empty regions") + all(>(0), Int.(c["regions"]["sizes"])) || + error("$(c["case"]): a region contains no bus") + isapprox(sum(Float64.(c["regions"]["demand_share"])), 1.0; atol = 1e-9) || + error("$(c["case"]): region demand shares do not sum to 1") + # The balance gate, re-checked from the recorded shares themselves. + share = Float64.(c["regions"]["demand_share"]) + exception = Bool(get(c["regions"], "single_bus_exception", false)) + all(s -> s >= PORTFOLIO_REGION_SHARE_MIN - 1e-9, share) || + error("$(c["case"]): a region share is below $PORTFOLIO_REGION_SHARE_MIN") + over = count(s -> s > PORTFOLIO_REGION_SHARE_MAX + 1e-9, share) + over == 0 || (exception && over == 1) || + error("$(c["case"]): $over regions exceed $PORTFOLIO_REGION_SHARE_MAX " * + "with single_bus_exception = $exception") + n_eff = 1 / sum(abs2, share) + isapprox(n_eff, Float64(c["regions"]["effective_regions"]); rtol = 1e-9) || + error("$(c["case"]): the recorded effective region count does not match its shares") + n_eff >= PORTFOLIO_MIN_EFFECTIVE_REGIONS - 1e-9 || exception || + error("$(c["case"]): effective region count $n_eff is below " * + "$PORTFOLIO_MIN_EFFECTIVE_REGIONS") + c["protocols"]["screening"]["excludes"] == "final" || + error("$(c["case"]): the screening protocol does not exclude the final one") + c["protocols"]["final"]["sha256"] == c["protocols"]["screening"]["sha256"] && + error("$(c["case"]): the two protocols are identical") + end + return m +end + +""" + materialize_portfolio_case(name; dir, manifest=nothing, verify_all=false, + optimizer=nothing, quiet=false) -> BatteryCase + +Regenerate one frozen case into `dir` and check it against the manifest. + +# Notes +This is the reader-facing reproduction path and it is fail-closed at four points: +the acquired network must hash to the recorded digest, the recomputed regions and +placement must hash to the recorded digests, the frozen support must hash to the +recorded digest, and every written artifact must hash to what the manifest +records. `κ_case` is READ from the manifest rather than recalibrated, because a +reader should not have to spend hundreds of ACP solves to obtain a case — the +calibration is checked instead by `verify_all`, which is off by default and +re-runs the full `24 × 6` gate when it is on. +""" +function materialize_portfolio_case(name::AbstractString; + dir::AbstractString, + manifest = nothing, + verify_all::Bool = false, + optimizer = nothing, + quiet::Bool = false) + m = manifest === nothing ? read_portfolio_manifest() : manifest + idx = findfirst(c -> String(c["case"]) == String(name), m["cases"]) + idx === nothing && error("$name is not in the frozen panel") + rec = m["cases"][idx] + + src = acquire_pglib_case(name) + sha = network_digest(src.network) + sha == rec["network_sha256"] || + error("$name: the acquired network hashes to $sha, the manifest records $(rec["network_sha256"])") + + regions = portfolio_regions(src.network) + regions.digest == rec["regions"]["digest"] || + error("$name: regenerated region digest $(regions.digest) does not match the manifest") + buses, _ = portfolio_placement(src.network, sha) + buses == Int.(rec["placement"]["buses"]) || + error("$name: the regenerated placement differs from the manifest") + + built = build_portfolio_case(name; dir = dir, kappa = Float64(rec["kappa_case"]), + verify_all = verify_all, optimizer = optimizer, + regions = regions, quiet = quiet) + built.ok || error("$name: reconstruction failed") + for (f, want) in rec["artifacts"] + got = built.record["artifacts"][f] + got == want || + error("$name: regenerated $f hashes to $got, the manifest records $want") + end + built.record["support"]["sha256"] == rec["support"]["sha256"] || + error("$name: the regenerated support digest does not match the manifest") + quiet || @printf(" %s regenerated into %s and matched every recorded digest\n", name, dir) + return built.case +end + +""" + freeze_portfolio(; dir, manifest_paths, quiet=false) -> Dict + +Build the whole panel and write the manifest. + +# Keywords +- `dir`: root the per-case artifact directories are written under. +- `manifest_paths`: every location the byte-identical manifest is written to — + in practice both public example directories. + +# Notes +Cases are attempted in the preregistered order. A case that fails the base-ACP +data gate is recorded as a replacement and the next RESERVE case is promoted, in +reserve order; a case that fails for any other reason stops the freeze, because +"this case broke and we moved on" is exactly the selection freedom the +preregistration exists to remove. +""" +function freeze_portfolio(; dir::AbstractString, + manifest_paths::AbstractVector{<:AbstractString}, + quiet::Bool = false) + records = Dict{String,Any}[] + replacements = Dict{String,Any}[] + reserve = copy(PORTFOLIO_RESERVE) + for name in PORTFOLIO_PRIMARY + candidate = name + while true + quiet || println("\n", candidate) + built = try + build_portfolio_case(candidate; dir = joinpath(dir, candidate), quiet = quiet) + catch e + e isa ErrorException && occursin("unavailable", string(e)) ? + (ok = false, record = nothing, + calibration = (reason = "unavailable from the pinned PGLib version",)) : + rethrow() + end + if built.ok + push!(records, built.record) + break + end + push!(replacements, Dict{String,Any}( + "dropped" => candidate, "reason" => built.calibration.reason, + "promoted" => isempty(reserve) ? "" : first(reserve))) + isempty(reserve) && error("the reserve panel is exhausted; $candidate cannot be replaced") + candidate = popfirst!(reserve) + end + end + manifest = nothing + for p in manifest_paths + manifest = write_portfolio_manifest(p, records; replacements = replacements) + end + hashes = unique([bytes2hex(sha256(read(p))) for p in manifest_paths]) + length(hashes) == 1 || + error("the portfolio manifest was not written byte-identically to every location") + return manifest +end + +# ───────────────────────────────────────────────────────────────────────────── +# J — regional demand variants +# +# Experimental supports built ON a frozen portfolio case: the network, fleet, +# κ, profile, seeds and stage duration are the case's own, and only the demand +# multiplier changes. Every choice lives in `demand_variants.toml`; nothing in +# this section alters `portfolio_support` or the reference manifest. +# ───────────────────────────────────────────────────────────────────────────── + +"Schema tag the variant recipe must carry." +const DEMAND_VARIANT_SCHEMA = "battery_storage_opf/demand_variants/1" + +"Path of the variant recipe inside an example directory." +demand_variant_recipe_path(dir::AbstractString = @__DIR__) = joinpath(dir, "demand_variants.toml") + +""" + demand_variant_recipe(path=demand_variant_recipe_path()) -> Dict{String,Any} + +Read the variant recipe and refuse any rule this code does not implement. + +# Notes +The recipe names its rules (`rank_pairs`, `weighted_phasor_triangle`, …) rather +than leaving them implicit, so a reader of the file alone knows what was built. +The check below is what makes that naming binding: a recipe asking for a rule +that is not implemented here is an error, never a silent substitution. +""" +function demand_variant_recipe(path::AbstractString = demand_variant_recipe_path()) + r = TOML.parsefile(path) + r["schema"] == DEMAND_VARIANT_SCHEMA || + error("variant recipe schema $(r["schema"]) is not $DEMAND_VARIANT_SCHEMA") + r["recipe_version"] == 1 || error("unsupported variant recipe_version $(r["recipe_version"])") + r["groups"]["rule"] == "rank_pairs" || error("unsupported grouping rule $(r["groups"]["rule"])") + m = r["means"] + (m["form"], m["phase_rule"], m["centering"]) == + ("centered_cosine", "weighted_phasor_triangle", "profile_weighted_daily") || + error("unsupported mean-modulation rules in the variant recipe") + s = r["shocks"] + (s["scale"], s["allocation"], s["probability"]) == + ("reference_region_variance", "partner_equal_mw", "uniform") || + error("unsupported shock rules in the variant recipe") + pairs = [Int.(p) for p in r["groups"]["pairs"]] + sort!(vcat(pairs...)) == collect(1:PORTFOLIO_REGIONS) || + error("variant pairs must partition 1:$PORTFOLIO_REGIONS, got $pairs") + length(pairs) == 3 || error("the variant recipe implements exactly three groups") + return r +end + +""" + variant_parameters(recipe, demand_pu) -> NamedTuple + +The host-dependent constants of the variant construction. + +# Arguments +- `demand_pu::AbstractVector`: region nominal active demand ``D_r``, manifest order. + +# Returns +`(pairs, group, sigma, G, phi, beta, delta, u)` with `group[r]` the group of +region `r` and `sigma[r] = ±1` its sign inside the group. + +# Notes +```math +\\delta^2 = \\tfrac1n (H-1)^2 + \\tfrac{n-1}{n}(L-1)^2,\\qquad +u_r = \\delta\\sqrt{D_{partner}/D_r}, +``` +so ``D_r u_r = D_{partner} u_{partner}`` and ``\\sum_r w_r u_r^2 = \\delta^2``. +The phases close the triangle ``\\sum_p G_p e^{i\\phi_p} = 0``. +""" +function variant_parameters(recipe::AbstractDict, demand_pu::AbstractVector{<:Real}) + n = PORTFOLIO_REGIONS + length(demand_pu) == n || error("expected $n region demands, got $(length(demand_pu))") + D = Float64.(demand_pu) + all(>(0), D) || error("every region must carry positive demand") + pairs = [Int.(p) for p in recipe["groups"]["pairs"]] + group = zeros(Int, n) + sigma = zeros(Float64, n) + for (p, (a, b)) in enumerate(pairs) + group[a] = p; sigma[a] = 1.0 + group[b] = p; sigma[b] = -1.0 + end + G = [D[a] + D[b] for (a, b) in pairs] + 2 * maximum(G) < sum(G) || + error("no phase triangle exists: the largest group mass $(maximum(G)) is at least half of $(sum(G))") + C = acos((G[1]^2 + G[2]^2 - G[3]^2) / (2 * G[1] * G[2])) + phi = [0.0, π - C, 0.0] + phi[3] = angle(-(G[1] + G[2] * cis(phi[2]))) + + m = recipe["means"] + P, t0 = Int(m["period"]), Int(m["anchor_stage"]) + θ = [2π * (t - t0) / P for t in 1:PORTFOLIO_HORIZON] + beta = [sum(PORTFOLIO_PROFILE[t] * cos(θ[t] - phi[p]) for t in 1:PORTFOLIO_HORIZON) / + sum(PORTFOLIO_PROFILE) for p in 1:3] + + delta = sqrt((1 / n) * (PORTFOLIO_REGION_HIGH - 1)^2 + ((n - 1) / n) * (PORTFOLIO_REGION_LOW - 1)^2) + u = zeros(Float64, n) + for (a, b) in pairs + u[a] = delta * sqrt(D[b] / D[a]) + u[b] = delta * sqrt(D[a] / D[b]) + end + return (pairs = pairs, group = group, sigma = sigma, G = G, phi = phi, beta = beta, + theta = θ, delta = delta, u = u) +end + +""" + variant_mean_modulation(recipe, params, amplitude) -> Matrix{Float64} + +``a_{p,t}`` as a `3 × PORTFOLIO_HORIZON` matrix. +""" +variant_mean_modulation(recipe::AbstractDict, params, amplitude::Real) = + [Float64(amplitude) * (cos(params.theta[t] - params.phi[p]) - params.beta[p]) + for p in 1:3, t in 1:PORTFOLIO_HORIZON] + +""" + variant_signs(recipe, variant, t) -> Matrix{Int} + +The `4 × 3` group sign matrix ``s_{p,k}(t)`` of `variant` (`"A"` or `"B"`) at +stage `t`; rows are atoms, columns groups. +""" +function variant_signs(recipe::AbstractDict, variant::AbstractString, t::Integer) + A = reduce(vcat, [permutedims(Int.(row)) for row in recipe["variants"]["A"]["signs"]]) + size(A) == (4, 3) || error("variant A must declare 4 atoms over 3 groups") + variant == "A" && return A + variant == "B" || error("unknown demand variant \"$variant\"") + recipe["variants"]["B"]["base"] == "A" || error("variant B must be based on A") + blocks = [b for b in recipe["variants"]["B"]["blocks"] if b["first"] <= t <= b["last"]] + length(blocks) == 1 || error("stage $t is covered by $(length(blocks)) blocks of variant B") + blk = only(blocks) + S = copy(A) + S[:, Int(blk["overwrite"])] .= A[:, Int(blk["keep"])] + return S +end + +""" + variant_modes(recipe, variant, demand_pu, t; amplitude) -> Matrix{Float64} + +The `(regions × 4)` joint multiplier modes of `variant` at stage `t`: +``m_{r,t,k} = (1 + a_{p,t})(1 + \\sigma_r u_r s_{p,k}(t))``. +""" +function variant_modes(recipe::AbstractDict, variant::AbstractString, + demand_pu::AbstractVector{<:Real}, t::Integer; amplitude::Real) + par = variant_parameters(recipe, demand_pu) + a = variant_mean_modulation(recipe, par, amplitude) + S = variant_signs(recipe, variant, t) + return [(1 + a[par.group[r], t]) * (1 + par.sigma[r] * par.u[r] * S[k, par.group[r]]) + for r in 1:PORTFOLIO_REGIONS, k in 1:4] +end + +""" + portfolio_variant_sampler(regions, demand_pu, recipe, variant; amplitude) -> StageMultiplier + +The authoring sampler of a variant: one equiprobable four-atom +[`JointRegionMultiplier`](@ref) per stage, applied by BUS exactly as the +reference sampler is. +""" +function portfolio_variant_sampler(regions::AbstractVector, demand_pu::AbstractVector{<:Real}, + recipe::AbstractDict, variant::AbstractString; amplitude::Real) + by_stage = [JointRegionMultiplier(regions, + variant_modes(recipe, variant, demand_pu, t; amplitude = amplitude), + fill(0.25, 4); by = :bus) + for t in 1:PORTFOLIO_HORIZON] + return StageMultiplier(by_stage) +end + +""" + portfolio_variant_support(network, regions, demand_pu, κ, recipe, variant; + amplitude, seed) -> DemandSupport + +Freeze a variant support with the reference's own profile, seed and stage +duration. + +# Notes +The freeze is the existing `:exact` path. Two properties are asserted HERE, +locally to the variants, because the evaluation protocol draws atom indices +uniformly: every stage has exactly four DISTINCT atoms (so duplicate merging +cannot have reweighted anything) and exactly uniform probabilities. +""" +function portfolio_variant_support(network::AbstractDict, regions::AbstractVector, + demand_pu::AbstractVector{<:Real}, κ::Real, + recipe::AbstractDict, variant::AbstractString; + amplitude::Real, seed::Integer) + sampler = portfolio_variant_sampler(regions, demand_pu, recipe, variant; amplitude = amplitude) + support = freeze_demand_support(sampler, network, PORTFOLIO_HORIZON; + seed = Int(seed), method = :exact, + profile = Float64(κ) .* PORTFOLIO_PROFILE, + profile_period = length(PORTFOLIO_PROFILE), + protocol_seed = Int(seed), + stage_hours = PORTFOLIO_STAGE_HOURS) + for t in 1:support.horizon + num_atoms(support, t) == 4 || + error("variant $variant stage $t has $(num_atoms(support, t)) atoms after freezing, expected 4") + all(==(0.25), atom_probabilities(support, t)) || + error("variant $variant stage $t probabilities are not exactly uniform") + allunique(eachcol(support.atoms[t])) || + error("variant $variant stage $t has duplicate atoms") + end + return support +end + +""" + region_demand_pu(network, regions) -> Vector{Float64} + +Nominal active demand per region, summed in ascending bus order — the same +quantity and order `portfolio_regions` records as `demand_pu`. +""" +function region_demand_pu(network::AbstractDict, regions::AbstractVector) + load_at = nominal_load_at_bus(network) + return [sum(load_at[b] for b in sort(collect(Int.(r))); init = 0.0) for r in regions] +end + +# ───────────────────────────────────────────────────────────────────────────── +# I — command line +# ───────────────────────────────────────────────────────────────────────────── + +""" + portfolio_main(args) -> Int + +The public command. Returns a process exit code. +""" +function portfolio_main(args::AbstractVector{<:AbstractString}) + opt(flag, default = nothing) = begin + i = findfirst(==(flag), args) + i === nothing || i == length(args) ? default : args[i + 1] + end + has(flag) = flag in args + # `--manifest` exists so a reader can point the command at a manifest that is + # not the one shipped beside this file — the panel's own freeze does exactly + # that, and so does the regression suite. + mpath = opt("--manifest", portfolio_manifest_path()) + + if has("--list") + m = read_portfolio_manifest(mpath) + @printf("%-28s %7s %8s %8s %10s %s\n", + "case", "bus", "battery", "κ_case", "regions", "network sha256") + for c in m["cases"] + @printf("%-28s %7d %8d %8.4f %10s %s\n", + c["case"], c["counts"]["bus"], c["counts"]["battery"], + c["kappa_case"], join(c["regions"]["sizes"], "/"), + first(String(c["network_sha256"]), 16)) + end + return 0 + end + + if has("--verify") + # `--min-cases` exists for the same reason as `--manifest`: a manifest + # that is not the frozen panel — a single-case one written by the + # regression suite — is still a manifest and must be checkable. + m = verify_portfolio_manifest(mpath; + min_cases = parse(Int, opt("--min-cases", + string(PORTFOLIO_MIN_CASES)))) + @printf("portfolio manifest verified: %d cases, digest %s\n", + length(m["cases"]), first(String(m["digest"]), 16)) + return 0 + end + + if has("--freeze") + out = opt("--out", joinpath(@__DIR__, "case")) + exa = opt("--mirror", nothing) + paths = [portfolio_manifest_path()] + exa === nothing || push!(paths, portfolio_manifest_path(exa)) + m = freeze_portfolio(; dir = out, manifest_paths = paths) + @printf("\nfrozen %d cases, manifest digest %s\n", + length(m["cases"]), first(String(m["digest"]), 16)) + return 0 + end + + name = opt("--case", nothing) + name === nothing && (println("usage: ", PORTFOLIO_COMMAND); return 2) + out = opt("--out", joinpath(@__DIR__, "case")) + case = materialize_portfolio_case(name; dir = joinpath(out, name), + manifest = read_portfolio_manifest(mpath), + verify_all = has("--verify-all")) + print(describe(case)) + return 0 +end + +abspath(PROGRAM_FILE) == abspath(@__FILE__) && exit(portfolio_main(ARGS)) diff --git a/examples/BatteryStorageOPF/battery_portfolio.json b/examples/BatteryStorageOPF/battery_portfolio.json new file mode 100644 index 0000000..6e79df1 --- /dev/null +++ b/examples/BatteryStorageOPF/battery_portfolio.json @@ -0,0 +1,3342 @@ +{ + "battery": { + "charge_efficiency": 0.95, + "discharge_efficiency": 0.95, + "duration_hours": 8.0, + "initial_fraction": 0.5, + "reserve_fraction": 0.05, + "self_discharge": 0.999, + "system_power_share": 0.1, + "throughput_cost": 5.0, + "weight_cap": 3.0 + }, + "calibration": { + "bracket": [ + 0.5, + 1.25 + ], + "formulation": "PowerModels.ACPPowerModel, unmodified network, no batteries", + "gate_stages": [ + 4, + 11 + ], + "margin": 0.95, + "recourse_tolerance": 1.0e-6, + "residual_tolerance": 1.0e-7, + "step": 0.05, + "tolerance": 0.001 + }, + "cases": [ + { + "artifacts": { + "batteries.json": "7c6d8cc62a1582672db38411a27aa9b83fc61251c10c797d4f4906089a35b153", + "demand.json": "f9e48de85d50134f8efcfc224a3ceb85f9267c1c92182dc231576edde6c10a8e", + "network.json": "831a859593f53797ebb64d7a9de435ee5b1a4e01e7650e10db03ba8cb9fbff26" + }, + "battery": { + "digest": "99ab113f3a8a9fc82fef6af1d8b5f16b55caa2b691f124f7a3e909083a275767", + "energy_budget_puh": 37.604004374999995, + "power_budget_pu": 4.700500546874999, + "power_max_pu": 0.46367649462394134, + "power_min_pu": 0.0478017004766949 + }, + "case": "pglib_opf_case118_ieee", + "counts": { + "battery": 24, + "branch": 186, + "bus": 118, + "eligible_bus": 99, + "gen": 54, + "load": 99 + }, + "kappa_case": 1.1080859374999998, + "kappa_evaluations": 9, + "kappa_max": 1.1664062499999999, + "kappa_retreats": 0, + "network_sha256": "6c92fd1dfa9021b964d77a1627906092081bf46cdc989699a40bb3931b7b50e2", + "nominal_demand_pu": 42.42, + "peak_demand_pu": 47.00500546874999, + "placement": { + "algorithm": "sha256-exponential-key weighted sampling without replacement", + "buses": [ + 6, + 8, + 12, + 18, + 36, + 41, + 42, + 49, + 56, + 60, + 66, + 80, + 90, + 92, + 95, + 97, + 98, + 99, + 104, + 107, + 110, + 112, + 115, + 116 + ], + "count": 24, + "digest": "7f5c2a7f2654fd622fad61b8fe14459c3fb36e1cd58a2a157b2704c7c4365994", + "eligible": 99 + }, + "protocols": { + "final": { + "num_scenarios": 500, + "num_stages": 24, + "seed": 594161976, + "sha256": "7de27f0b33175fd37eee8f028be7040bac3757f73281ea928b202f7afeb1ac40" + }, + "screening": { + "excludes": "final", + "num_scenarios": 32, + "num_stages": 24, + "seed": 1994805207, + "sha256": "b0a5bae2e0ecd7320c4912188c304ccdb1ed61ba31b66aa021c1ba38b8157e4f" + } + }, + "regions": { + "corridors": [ + 5, + 12, + 23, + 31, + 33, + 39, + 41, + 52, + 53, + 55, + 56, + 65, + 70, + 71, + 75, + 76, + 78, + 81, + 83, + 102, + 103, + 108, + 110, + 112, + 114, + 115, + 116, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 135, + 136, + 137, + 139, + 148, + 149, + 151, + 152, + 153, + 155, + 158, + 159, + 163, + 164, + 167, + 168, + 171, + 173, + 174, + 177, + 179, + 185 + ], + "count": 6, + "demand_pu": [ + 11.58, + 10.03, + 9.95, + 3.72, + 3.71, + 3.43 + ], + "demand_share": [ + 0.272984441301273, + 0.2364450730787364, + 0.23455917020273453, + 0.0876944837340877, + 0.08745874587458745, + 0.08085808580858087 + ], + "digest": "875923af8cb35b87de1f7ddb081751542c35e20d51aa99c34f8ff7b2bb3f4dda", + "dispersion": 0.3965562759141299, + "effective_regions": 4.823410902915079, + "share_cap": 0.28, + "single_bus_exception": false, + "sizes": [ + 37, + 14, + 28, + 7, + 7, + 6 + ], + "sweeps": 3, + "unconstrained_dispersion": 0.40605418717126485, + "unconstrained_share": [ + 0.2885431400282885, + 0.2670909948137671, + 0.23455917020273453, + 0.09123055162659123, + 0.0876944837340877, + 0.03088165959453088 + ] + }, + "support": { + "sha256": "0a9763629bf78cd91ef289198dab89edd6489194c30f030b5f96bf341315f349" + }, + "verification": { + "retreats": 0, + "stage_atom_checked": 144, + "stage_atom_failures": 0 + } + }, + { + "artifacts": { + "batteries.json": "3e40b98943ab44224a5fd2fa9ad4f492d2a7942e8cacb8f2716de80da729f86d", + "demand.json": "2e92e4f4afa54f4d492ac4bd9e4835870674ede47925be9e23c550615716d3a5", + "network.json": "ea9c702f68ac9ca4d39a27b598d49eebd06ddc2d1dee9fdde41dec04e98edbe4" + }, + "battery": { + "digest": "d3e56bde42d5337316cffc9e6372415e6cd9a0c9de2b4d057e6bad2930616442", + "energy_budget_puh": 599.1013589062499, + "power_budget_pu": 74.88766986328123, + "power_max_pu": 0.7914335803966963, + "power_min_pu": 0.022400109743608326 + }, + "case": "pglib_opf_case1951_rte", + "counts": { + "battery": 240, + "branch": 2596, + "bus": 1951, + "eligible_bus": 940, + "gen": 391, + "load": 1015 + }, + "kappa_case": 0.9284765624999999, + "kappa_evaluations": 13, + "kappa_max": 0.97734375, + "kappa_retreats": 0, + "network_sha256": "1e5eb455fb49b7cce6c050bcd3a9c3f7e663065f6b2eedc7e61b193b9c242cea", + "nominal_demand_pu": 806.5649999999999, + "peak_demand_pu": 748.8766986328123, + "placement": { + "algorithm": "sha256-exponential-key weighted sampling without replacement", + "buses": [ + 3, + 13, + 19, + 21, + 22, + 40, + 42, + 43, + 44, + 45, + 49, + 53, + 55, + 57, + 71, + 82, + 84, + 87, + 98, + 107, + 111, + 114, + 123, + 149, + 199, + 209, + 213, + 222, + 237, + 247, + 263, + 273, + 276, + 280, + 295, + 297, + 303, + 309, + 313, + 317, + 325, + 328, + 330, + 342, + 344, + 345, + 354, + 367, + 387, + 389, + 394, + 398, + 404, + 406, + 407, + 409, + 413, + 414, + 428, + 440, + 441, + 444, + 461, + 464, + 468, + 469, + 472, + 473, + 478, + 490, + 493, + 522, + 529, + 530, + 535, + 536, + 546, + 554, + 556, + 568, + 595, + 598, + 600, + 605, + 606, + 609, + 613, + 618, + 627, + 630, + 641, + 650, + 655, + 656, + 668, + 670, + 676, + 695, + 698, + 699, + 711, + 712, + 730, + 734, + 735, + 738, + 740, + 741, + 746, + 752, + 754, + 757, + 776, + 790, + 792, + 793, + 795, + 806, + 825, + 827, + 830, + 840, + 844, + 845, + 846, + 847, + 849, + 865, + 870, + 873, + 881, + 884, + 887, + 891, + 895, + 896, + 910, + 911, + 919, + 929, + 931, + 933, + 937, + 941, + 943, + 945, + 950, + 958, + 959, + 963, + 964, + 977, + 993, + 994, + 1003, + 1021, + 1024, + 1029, + 1030, + 1032, + 1035, + 1042, + 1044, + 1048, + 1050, + 1059, + 1060, + 1062, + 1066, + 1087, + 1098, + 1101, + 1118, + 1119, + 1133, + 1140, + 1143, + 1151, + 1167, + 1168, + 1171, + 1172, + 1176, + 1177, + 1185, + 1188, + 1189, + 1193, + 1199, + 1201, + 1202, + 1206, + 1208, + 1217, + 1220, + 1229, + 1231, + 1236, + 1256, + 1276, + 1279, + 1285, + 1295, + 1301, + 1303, + 1304, + 1324, + 1325, + 1342, + 1349, + 1355, + 1357, + 1359, + 1377, + 1380, + 1387, + 1389, + 1406, + 1407, + 1410, + 1411, + 1412, + 1673, + 1674, + 1690, + 1715, + 1729, + 1735, + 1798, + 1802, + 1803, + 1806, + 1810, + 1811, + 1814, + 1815, + 1820, + 1824, + 1825, + 1839 + ], + "count": 240, + "digest": "d9dcc2bc35594d16408fe53437eaf163e696514468da6cfd371f12f48fad71ab", + "eligible": 940 + }, + "protocols": { + "final": { + "num_scenarios": 500, + "num_stages": 24, + "seed": 1990704154, + "sha256": "b992e8024e6ac66abd9958122d30d976387014df47160359971f4eec69c6db1a" + }, + "screening": { + "excludes": "final", + "num_scenarios": 32, + "num_stages": 24, + "seed": 473149067, + "sha256": "90dc129583053c842b7c3318431ada4b5c5d5fdfe231d7b0a634fb7696a192cb" + } + }, + "regions": { + "corridors": [ + 38, + 40, + 81, + 85, + 89, + 133, + 150, + 156, + 158, + 258, + 264, + 356, + 395, + 398, + 404, + 455, + 498, + 500, + 554, + 569, + 574, + 577, + 578, + 587, + 590, + 639, + 640, + 659, + 753, + 756, + 802, + 834, + 850, + 938, + 1015, + 1187, + 1193, + 1257, + 1259, + 1319, + 1322, + 1331, + 1349, + 1370, + 1373, + 1440, + 1473, + 1486, + 1491, + 1500, + 1591, + 1675, + 1686, + 1772, + 1777, + 1785, + 1799, + 1837, + 1849, + 1850, + 1851, + 1853, + 1856, + 1877 + ], + "count": 6, + "demand_pu": [ + 236.39500000000007, + 211.47599999999994, + 135.718, + 117.94400000000003, + 75.19400000000002, + 67.54399999999997 + ], + "demand_share": [ + 0.27999895768065, + 0.25048355326666427, + 0.16075170176400702, + 0.1396992198002774, + 0.08906381955556926, + 0.08000274793283195 + ], + "digest": "b48b567bf11e345470d0ad431e7d49f698e466d773594601c4f85d31f55cb506", + "dispersion": 0.22034708427351785, + "effective_regions": 4.97930570615907, + "share_cap": 0.28, + "single_bus_exception": false, + "sizes": [ + 181, + 241, + 185, + 142, + 106, + 85 + ], + "sweeps": 15, + "unconstrained_dispersion": 0.2637537213697582, + "unconstrained_share": [ + 0.8078188164700668, + 0.10197436605071122, + 0.0613665517351656, + 0.013091767927596705, + 0.008261565303083963, + 0.007486932513375445 + ] + }, + "support": { + "sha256": "163482369eb7aaad69d16ec5eff2e7663417bf03de49b57fd8c2491a2affe440" + }, + "verification": { + "retreats": 0, + "stage_atom_checked": 144, + "stage_atom_failures": 0 + } + }, + { + "artifacts": { + "batteries.json": "cc3eb1a7f10d500fcd5879ddf05ceecb545d04229ef9b9eefb3224a147e2881d", + "demand.json": "b09166feb2a70673dd2a4df609480f696440a8b5f2d17518fe26a21293b0e4cd", + "network.json": "2e52a8b42b8cd063c38534de5c4899683e5476f1b98f4838d1cc4bdf93edc32f" + }, + "battery": { + "digest": "0dfa35debf7111a5ec5de7bf254a300f0c2a480eb62dc3cdbb3c381d1f9935be", + "energy_budget_puh": 164.47975005000006, + "power_budget_pu": 20.559968756250008, + "power_max_pu": 0.22023386506918285, + "power_min_pu": 0.012910414568437821 + }, + "case": "pglib_opf_case2383wp_k", + "counts": { + "battery": 240, + "branch": 2896, + "bus": 2383, + "eligible_bus": 1817, + "gen": 327, + "load": 1826 + }, + "kappa_case": 0.8371875, + "kappa_evaluations": 15, + "kappa_max": 0.88125, + "kappa_retreats": 0, + "network_sha256": "42681376fd85e80db3f6bc799c0c8fc4d2de5d5725a8b3d40b125ad15d8f260d", + "nominal_demand_pu": 245.58380000000008, + "peak_demand_pu": 205.59968756250007, + "placement": { + "algorithm": "sha256-exponential-key weighted sampling without replacement", + "buses": [ + 18, + 30, + 43, + 57, + 63, + 104, + 111, + 126, + 127, + 131, + 180, + 183, + 184, + 185, + 187, + 189, + 192, + 212, + 215, + 224, + 239, + 267, + 271, + 282, + 285, + 286, + 290, + 294, + 299, + 313, + 323, + 326, + 330, + 331, + 345, + 350, + 365, + 366, + 376, + 381, + 388, + 396, + 407, + 413, + 415, + 418, + 434, + 440, + 455, + 457, + 460, + 471, + 482, + 485, + 491, + 493, + 497, + 501, + 506, + 515, + 525, + 529, + 538, + 544, + 553, + 567, + 575, + 583, + 603, + 637, + 638, + 639, + 642, + 649, + 664, + 665, + 678, + 696, + 711, + 713, + 715, + 745, + 746, + 749, + 754, + 766, + 788, + 798, + 803, + 804, + 814, + 831, + 861, + 878, + 892, + 908, + 911, + 912, + 929, + 942, + 943, + 945, + 959, + 963, + 980, + 1004, + 1029, + 1037, + 1053, + 1066, + 1117, + 1118, + 1136, + 1155, + 1175, + 1190, + 1195, + 1198, + 1201, + 1229, + 1240, + 1287, + 1289, + 1290, + 1301, + 1309, + 1311, + 1322, + 1369, + 1381, + 1386, + 1393, + 1416, + 1429, + 1434, + 1438, + 1452, + 1476, + 1478, + 1481, + 1487, + 1505, + 1516, + 1518, + 1533, + 1536, + 1537, + 1538, + 1544, + 1565, + 1580, + 1587, + 1596, + 1602, + 1614, + 1639, + 1684, + 1688, + 1699, + 1701, + 1707, + 1709, + 1710, + 1712, + 1718, + 1744, + 1749, + 1750, + 1758, + 1767, + 1785, + 1795, + 1799, + 1800, + 1815, + 1829, + 1845, + 1875, + 1895, + 1901, + 1904, + 1916, + 1947, + 1962, + 1968, + 1977, + 1981, + 1998, + 2007, + 2010, + 2011, + 2012, + 2018, + 2027, + 2047, + 2050, + 2051, + 2055, + 2058, + 2097, + 2107, + 2110, + 2122, + 2131, + 2136, + 2137, + 2141, + 2160, + 2185, + 2197, + 2202, + 2206, + 2211, + 2225, + 2234, + 2235, + 2237, + 2264, + 2270, + 2279, + 2284, + 2292, + 2293, + 2295, + 2304, + 2306, + 2307, + 2308, + 2316, + 2336, + 2348, + 2351, + 2352, + 2354, + 2360, + 2370, + 2372, + 2373, + 2374, + 2383 + ], + "count": 240, + "digest": "d0f4be32910813a50224c3b46bf9cd32b13ea0ad4cf7b0fed14d66ed49bc15d3", + "eligible": 1817 + }, + "protocols": { + "final": { + "num_scenarios": 500, + "num_stages": 24, + "seed": 1271152573, + "sha256": "dfda1f1f8173f636ebcddcaa406fc4d6dbc28d2fcea8e14cd311b725d6c1ba49" + }, + "screening": { + "excludes": "final", + "num_scenarios": 32, + "num_stages": 24, + "seed": 1553436292, + "sha256": "727280e639f0a0122a93893d58951c12e2d99adac9ddfc32fe4583dc09d4af9b" + } + }, + "regions": { + "corridors": [ + 10, + 11, + 37, + 38, + 48, + 51, + 52, + 60, + 61, + 148, + 187, + 189, + 264, + 344, + 460, + 462, + 509, + 549, + 558, + 566, + 667, + 680, + 688, + 698, + 720, + 723, + 731, + 758, + 839, + 956, + 1085, + 1095, + 1155, + 1187, + 1188, + 1250, + 1265, + 1323, + 1377, + 1483, + 1574, + 1648, + 1666, + 1728, + 1746, + 1834, + 1925, + 2109, + 2279, + 2303, + 2309, + 2321, + 2413, + 2428, + 2488, + 2497, + 2577, + 2614, + 2654, + 2673, + 2797, + 2815, + 2828, + 2868 + ], + "count": 6, + "demand_pu": [ + 60.747499999999974, + 57.23100000000002, + 41.34199999999997, + 39.39930000000003, + 25.859099999999998, + 21.2254 + ], + "demand_share": [ + 0.24713766195302508, + 0.23283156559913726, + 0.16819071106567282, + 0.16028726918121458, + 0.10520198385463556, + 0.08635080834631452 + ], + "digest": "6985ce2ddcc9be1e285c08de20a919889836c8c7229c92f8c235fe0bf48f048e", + "dispersion": 0.5150656743339495, + "effective_regions": 5.325051660926274, + "share_cap": 0.28, + "single_bus_exception": false, + "sizes": [ + 487, + 464, + 267, + 301, + 178, + 120 + ], + "sweeps": 14, + "unconstrained_dispersion": 0.5970870672653886, + "unconstrained_share": [ + 0.34926240102390427, + 0.2366569665380142, + 0.22720635887980783, + 0.1715352416536243, + 0.011132026575613197, + 0.004207005329036147 + ] + }, + "support": { + "sha256": "03fbb135e6b7413526c8a23662d604a44cdbf7661ea87f607e7ce7fa9ebfd472" + }, + "verification": { + "retreats": 0, + "stage_atom_checked": 144, + "stage_atom_failures": 0 + } + }, + { + "artifacts": { + "batteries.json": "3c7b333f557b8c4c02eaa6c2e0f3819dfbe2c6d6e7b5fcc0a01a3b919bbd9a55", + "demand.json": "2c39bc6cfdeaf340d1dc17811b487a35fe0a1ca9d975190c53102fd8c9db9210", + "network.json": "836e69e4ff0ec961b0fae3f985c4f3b36ec0ba29388d43146e9cf002268ef87b" + }, + "battery": { + "digest": "39befc63f04b5c0da314c7c62f14a78d7b98189ac747b3ba2d19cc01be4a36ec", + "energy_budget_puh": 1017.0077577907491, + "power_budget_pu": 127.12596972384364, + "power_max_pu": 6.3786433128420565, + "power_min_pu": 0.20186778110742165 + }, + "case": "pglib_opf_case240_pserc", + "counts": { + "battery": 48, + "branch": 448, + "bus": 240, + "eligible_bus": 137, + "gen": 143, + "load": 139 + }, + "kappa_case": 0.8817187499999999, + "kappa_evaluations": 14, + "kappa_max": 0.9281249999999999, + "kappa_retreats": 0, + "network_sha256": "55ac4949ffc21d1b02b502dd4731ba0cb28a0be8c3842e31c69bc6d755ba242e", + "nominal_demand_pu": 1441.7972819999989, + "peak_demand_pu": 1271.2596972384363, + "placement": { + "algorithm": "sha256-exponential-key weighted sampling without replacement", + "buses": [ + 1003, + 1004, + 1101, + 1301, + 1303, + 1401, + 1402, + 2000, + 2203, + 2403, + 2405, + 2406, + 2407, + 2409, + 2410, + 2411, + 2502, + 2503, + 2604, + 2615, + 2618, + 3103, + 3202, + 3204, + 3302, + 3304, + 3401, + 3806, + 3907, + 3917, + 3921, + 3922, + 4010, + 4102, + 4104, + 4201, + 4202, + 4203, + 5001, + 5002, + 6104, + 6201, + 6502, + 6510, + 7001, + 8003, + 8004, + 8005 + ], + "count": 48, + "digest": "f8a1353e84b14b8e572b4a2f471c97c7209856cfff78b9a36f68e3809e1949ce", + "eligible": 137 + }, + "protocols": { + "final": { + "num_scenarios": 500, + "num_stages": 24, + "seed": 504861337, + "sha256": "ec32b041c8a785dd0bf7bfe15eeff67145553002b0fe40a4bcaa7ed7b016a451" + }, + "screening": { + "excludes": "final", + "num_scenarios": 32, + "num_stages": 24, + "seed": 2046122276, + "sha256": "f38a9069cb79e792f5436485bf257195edb145db1e24bfaab72fd3eecbb41af5" + } + }, + "regions": { + "corridors": [ + 7, + 8, + 21, + 43, + 125, + 126, + 131, + 132, + 145, + 152, + 153, + 160, + 177, + 180, + 181, + 182, + 184, + 186, + 187, + 188, + 191, + 193, + 194, + 196, + 201, + 205, + 216, + 217, + 219, + 220, + 223, + 224, + 225, + 228, + 229, + 231, + 234, + 235, + 236, + 237, + 246, + 250, + 253, + 270, + 272, + 275, + 282, + 283, + 284, + 291, + 293, + 298, + 299, + 300, + 301, + 302, + 317, + 323, + 325, + 329, + 363, + 364, + 375, + 376 + ], + "count": 6, + "demand_pu": [ + 416.67905699999994, + 415.110027, + 222.62192499999998, + 163.013217, + 147.10294299999998, + 123.64749600000002 + ], + "demand_share": [ + 0.2799933816908514, + 0.2789390498056893, + 0.1495939490409212, + 0.10953903519114136, + 0.09884790170110844, + 0.08308668257028823 + ], + "digest": "afa9f80a6b67b9f5a3feefe330a3544f28124a00082c592a036b9c13f1877747", + "dispersion": 0.2892330289299731, + "effective_regions": 4.824980081218391, + "share_cap": 0.28, + "single_bus_exception": false, + "sizes": [ + 33, + 30, + 20, + 27, + 18, + 9 + ], + "sweeps": 9, + "unconstrained_dispersion": 0.2857898172400244, + "unconstrained_share": [ + 0.4471658271376366, + 0.32134620232901223, + 0.152275501209396, + 0.029838992051379933, + 0.026509396328152118, + 0.02286408094442328 + ] + }, + "support": { + "sha256": "7872541579c70ebeccd2b018c588a2398655ace45232e9d918bc7a5097ad2429" + }, + "verification": { + "retreats": 0, + "stage_atom_checked": 144, + "stage_atom_failures": 0 + } + }, + { + "artifacts": { + "batteries.json": "16308b24534cd04fadf235ea2eed8859bfb7ed500c0bfb8a47999839975ed546", + "demand.json": "f71894aec0405e06947e4cd315f90a9ae1bd45905b7d26a829fd0cb2d4aab0f1", + "network.json": "ddb075fe8f20a1f10fc8c76d352f1c8acc3e2e59af42e800979746646a5bcc07" + }, + "battery": { + "digest": "f0d49ebeac80f02e030b7f7031c58e55a6b17d36d9c79373c10ef1029193c296", + "energy_budget_puh": 155.05005515624998, + "power_budget_pu": 19.381256894531248, + "power_max_pu": 0.7930265380738565, + "power_min_pu": 0.03356726087085107 + }, + "case": "pglib_opf_case300_ieee", + "counts": { + "battery": 60, + "branch": 411, + "bus": 300, + "eligible_bus": 191, + "gen": 69, + "load": 201 + }, + "kappa_case": 0.8238281249999999, + "kappa_evaluations": 15, + "kappa_max": 0.8671875, + "kappa_retreats": 0, + "network_sha256": "e4520e1d6b832f34ffc68bb23fe34b1544e1aa5ddc9a2a1da50f35d5de8c7ecd", + "nominal_demand_pu": 235.25849999999997, + "peak_demand_pu": 193.81256894531245, + "placement": { + "algorithm": "sha256-exponential-key weighted sampling without replacement", + "buses": [ + 1, + 2, + 5, + 6, + 9, + 10, + 11, + 13, + 17, + 20, + 21, + 33, + 47, + 49, + 59, + 61, + 71, + 73, + 76, + 80, + 90, + 97, + 99, + 102, + 108, + 120, + 121, + 122, + 124, + 125, + 127, + 135, + 137, + 138, + 139, + 155, + 167, + 170, + 171, + 173, + 182, + 188, + 191, + 192, + 209, + 211, + 220, + 222, + 223, + 224, + 225, + 227, + 228, + 231, + 232, + 233, + 234, + 235, + 319, + 526 + ], + "count": 60, + "digest": "3a61af8bf3bb026222e35de070ab25eafd3d026182f175fd5f1868c529fdaee2", + "eligible": 191 + }, + "protocols": { + "final": { + "num_scenarios": 500, + "num_stages": 24, + "seed": 907620839, + "sha256": "580c71f45283942545e6162f01f9d0dae789cb9c4910ef47f4755fcc876d475a" + }, + "screening": { + "excludes": "final", + "num_scenarios": 32, + "num_stages": 24, + "seed": 131546072, + "sha256": "c2517d8eb172e2e9135ea002b48d541be4fc4cf25975be08521fa3e95169f7ac" + } + }, + "regions": { + "corridors": [ + 49, + 71, + 80, + 81, + 82, + 84, + 85, + 87, + 89, + 90, + 93, + 94, + 97, + 99, + 100, + 101, + 102, + 104, + 105, + 106, + 107, + 110, + 111, + 112, + 115, + 119, + 128, + 133, + 138, + 139, + 140, + 141, + 143, + 145, + 146, + 147, + 149, + 159, + 160, + 167, + 170, + 171, + 172, + 178, + 179, + 191, + 192, + 203, + 204, + 205, + 220, + 221, + 227, + 232, + 233, + 249, + 272, + 274, + 291, + 292, + 307, + 356, + 379, + 403 + ], + "count": 6, + "demand_pu": [ + 66.77000000000001, + 66.69700000000002, + 33.789, + 32.95100000000001, + 19.167400000000008, + 19.102099999999997 + ], + "demand_share": [ + 0.27998565896430044, + 0.2796795491379654, + 0.14168691674022385, + 0.13817294366530872, + 0.08037437651089312, + 0.08010055498130839 + ], + "digest": "2280ae1068bd7b1136e23d0214e3c8bc86f87034c20e4da41546d397a37f502a", + "dispersion": 0.18696201321127232, + "effective_regions": 4.79258406715442, + "share_cap": 0.28, + "single_bus_exception": false, + "sizes": [ + 34, + 27, + 37, + 38, + 44, + 11 + ], + "sweeps": 6, + "unconstrained_dispersion": 0.23664724983767427, + "unconstrained_share": [ + 0.6152660744350074, + 0.3471704759169142, + 0.02808662488756753, + 0.0038578224688805805, + 0.003530746216084183, + 0.002088256075546227 + ] + }, + "support": { + "sha256": "41f04864f56746b44487fab397db8f6960e6f9c975add20ba92df35468acf6bb" + }, + "verification": { + "retreats": 0, + "stage_atom_checked": 144, + "stage_atom_failures": 0 + } + }, + { + "artifacts": { + "batteries.json": "fb4a2f8e20ac4376f3a03e503a21aaf8c3f30dd291758fcc747dae7727c3d5e8", + "demand.json": "247001f91eacacc7b855bc77eae81f1a73c033b7c0de37cfab55ce78079bf21b", + "network.json": "296796946dfb593a678cde3e88f877f998ee3883949f9400bb2631450fe2469e" + }, + "battery": { + "digest": "bca6b83944879a2c5effd45d997e6939cc8707ff5d0d01dbb82b02b16023de43", + "energy_budget_puh": 153.22479287654912, + "power_budget_pu": 19.15309910956864, + "power_max_pu": 0.5283031751539662, + "power_min_pu": 0.047436788572256555 + }, + "case": "pglib_opf_case500_goc", + "counts": { + "battery": 100, + "branch": 733, + "bus": 500, + "eligible_bus": 281, + "gen": 224, + "load": 281 + }, + "kappa_case": 1.0776562499999998, + "kappa_evaluations": 10, + "kappa_max": 1.134375, + "kappa_retreats": 0, + "network_sha256": "4698ac3c5b4e68b759d16205a9e545ab4ce4979cba6dd217bdcceba0089afca5", + "nominal_demand_pu": 177.72920733832004, + "peak_demand_pu": 191.5309910956864, + "placement": { + "algorithm": "sha256-exponential-key weighted sampling without replacement", + "buses": [ + 4, + 5, + 12, + 13, + 15, + 18, + 24, + 31, + 32, + 35, + 37, + 38, + 42, + 45, + 46, + 47, + 48, + 53, + 57, + 65, + 67, + 74, + 76, + 86, + 89, + 92, + 99, + 101, + 105, + 107, + 109, + 110, + 111, + 112, + 113, + 114, + 120, + 122, + 123, + 125, + 132, + 134, + 138, + 139, + 143, + 148, + 150, + 153, + 155, + 157, + 158, + 163, + 165, + 166, + 170, + 171, + 172, + 174, + 175, + 177, + 181, + 183, + 186, + 187, + 189, + 192, + 194, + 195, + 196, + 197, + 199, + 200, + 207, + 209, + 213, + 219, + 221, + 224, + 233, + 235, + 236, + 238, + 242, + 246, + 248, + 251, + 254, + 255, + 256, + 257, + 258, + 260, + 271, + 279, + 294, + 296, + 315, + 333, + 384, + 397 + ], + "count": 100, + "digest": "d80591d10fe0e3e51914b4a216c28b6cf27f42cf51b52bdbce39e11bf77fca53", + "eligible": 281 + }, + "protocols": { + "final": { + "num_scenarios": 500, + "num_stages": 24, + "seed": 2140516806, + "sha256": "29c9ee37467a3c0a17554951dd8801dbcbf15fa43d405759a873c4ebdf3560b2" + }, + "screening": { + "excludes": "final", + "num_scenarios": 32, + "num_stages": 24, + "seed": 866645167, + "sha256": "fb7185b9abd31c5a16212afafaede2a248709ba5d8779989b876ecd0baafab3a" + } + }, + "regions": { + "corridors": [ + 21, + 22, + 29, + 33, + 43, + 60, + 61, + 95, + 107, + 114, + 116, + 119, + 132, + 138, + 173, + 188, + 194, + 203, + 209, + 212, + 254, + 262, + 285, + 290, + 293, + 303, + 309, + 310, + 313, + 315, + 341, + 342, + 377, + 378, + 383, + 386, + 390, + 400, + 401, + 402, + 409, + 410, + 418, + 425, + 426, + 427, + 434, + 435, + 447, + 452, + 456, + 462, + 463, + 466, + 498, + 502, + 505, + 511, + 512, + 516, + 524, + 538, + 539, + 733 + ], + "count": 6, + "demand_pu": [ + 49.70586711132, + 41.595525289, + 31.717166730000006, + 21.740756624, + 18.75049066, + 14.219400924000002 + ], + "demand_share": [ + 0.2796719113066284, + 0.23403877118418698, + 0.1784578190889253, + 0.12232517631508334, + 0.105500333573801, + 0.08000598853137501 + ], + "digest": "88e9d5dc24589f9a4c5a1e00c22aa9037221e63903cc8772cee4f469948e7ab5", + "dispersion": 0.3843555352629458, + "effective_regions": 5.067590382059831, + "share_cap": 0.28, + "single_bus_exception": false, + "sizes": [ + 78, + 57, + 52, + 43, + 22, + 29 + ], + "sweeps": 7, + "unconstrained_dispersion": 0.42873561408078653, + "unconstrained_share": [ + 0.6176080297560258, + 0.10865741550987183, + 0.1043210898066185, + 0.0920725698666399, + 0.06283096904125064, + 0.0145099260195934 + ] + }, + "support": { + "sha256": "2ed9da2e3837a8e7c0571e02f7e87a7f43957fa3071746447cf302e17ddbe1aa" + }, + "verification": { + "retreats": 0, + "stage_atom_checked": 144, + "stage_atom_failures": 0 + } + }, + { + "artifacts": { + "batteries.json": "c5ac6ec9a0e51f3def6b0586301980f2d4fa1850e167ae11e9cf3c988e4a7cc1", + "demand.json": "a42db78a0e2ba22dbd456dc1bd255860f5e7df3b9cc61de0dc560069980269dc", + "network.json": "f90f01476c09c44a05e74e23aab520eb03f7c50cdf71431fbd8a1fd92c6fc871" + }, + "battery": { + "digest": "b960978584853da92024e3e516378e7c09c5671c2d7333c9fe0f084cdbdb7817", + "energy_budget_puh": 101.28054499999996, + "power_budget_pu": 12.660068124999995, + "power_max_pu": 0.2625462535413682, + "power_min_pu": 0.0042210779778277885 + }, + "case": "pglib_opf_case588_sdet", + "counts": { + "battery": 118, + "branch": 686, + "bus": 588, + "eligible_bus": 371, + "gen": 167, + "load": 379 + }, + "kappa_case": 1.1875, + "kappa_evaluations": 1, + "kappa_max": 1.25, + "kappa_retreats": 0, + "network_sha256": "a4c991f3f18b9cda681974cd7e0b29a47fa88006ae799b58c780ac5f33a332be", + "nominal_demand_pu": 106.61109999999996, + "peak_demand_pu": 126.60068124999995, + "placement": { + "algorithm": "sha256-exponential-key weighted sampling without replacement", + "buses": [ + 1, + 5, + 15, + 17, + 20, + 26, + 28, + 32, + 40, + 53, + 57, + 58, + 59, + 76, + 82, + 88, + 91, + 93, + 96, + 97, + 102, + 110, + 111, + 118, + 123, + 124, + 125, + 131, + 136, + 140, + 142, + 150, + 153, + 159, + 160, + 161, + 162, + 163, + 176, + 179, + 181, + 183, + 184, + 185, + 189, + 191, + 193, + 194, + 198, + 199, + 208, + 210, + 214, + 217, + 219, + 221, + 224, + 230, + 249, + 260, + 268, + 276, + 286, + 293, + 296, + 298, + 336, + 354, + 360, + 362, + 365, + 366, + 374, + 376, + 380, + 386, + 389, + 392, + 393, + 406, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 416, + 420, + 423, + 425, + 430, + 436, + 455, + 456, + 458, + 459, + 469, + 474, + 519, + 529, + 537, + 538, + 540, + 544, + 546, + 548, + 550, + 551, + 552, + 555, + 559, + 560, + 566, + 572, + 573, + 577, + 578 + ], + "count": 118, + "digest": "a4aaf68efd837c7b2a872bc6a9778d94a0be4866e9ec50c8a7e8e3b394be2f54", + "eligible": 371 + }, + "protocols": { + "final": { + "num_scenarios": 500, + "num_stages": 24, + "seed": 655790276, + "sha256": "9115e685ce631ba604998ef5e5bf1ef9a43dd5abc68d7eba0de4447db534ef50" + }, + "screening": { + "excludes": "final", + "num_scenarios": 32, + "num_stages": 24, + "seed": 1711235758, + "sha256": "f43c020c30fbd5b68c3032cff22851f8b9787f3557f2fdb06a9edea28daa78ff" + } + }, + "regions": { + "corridors": [ + 42, + 44, + 46, + 102, + 122, + 126, + 133, + 170, + 178, + 183, + 185, + 186, + 190, + 205, + 217, + 219, + 226, + 227, + 229, + 247, + 250, + 252, + 253, + 254, + 272, + 274, + 279, + 284, + 285, + 312, + 321, + 357, + 390, + 397, + 427, + 440, + 504, + 517, + 538, + 539, + 543, + 550, + 556, + 561, + 567, + 568, + 569, + 571, + 581, + 583, + 584, + 589, + 593, + 595, + 596, + 597, + 620, + 649, + 650, + 652, + 660, + 673, + 681, + 683 + ], + "count": 6, + "demand_pu": [ + 25.2746, + 22.8033, + 17.981800000000003, + 16.7106, + 16.2694, + 8.6169 + ], + "demand_share": [ + 0.23477055749484935, + 0.21181516042676435, + 0.1670292392663339, + 0.1552213240990334, + 0.15112310810484447, + 0.0800406106081745 + ], + "digest": "1a2b47ccfb37ac23178b437526dca82c4ccb9da5400696d3038b0466bb2bf57d", + "dispersion": 0.3399688979111372, + "effective_regions": 5.518154843922105, + "share_cap": 0.28, + "single_bus_exception": false, + "sizes": [ + 19, + 47, + 59, + 92, + 131, + 23 + ], + "sweeps": 5, + "unconstrained_dispersion": 0.338898010986567, + "unconstrained_share": [ + 0.23477055749484935, + 0.21181516042676435, + 0.1670292392663339, + 0.1552213240990334, + 0.15434074641034548, + 0.07682297230267347 + ] + }, + "support": { + "sha256": "512ecad47f26aeae812c61a4d89c76cad86c2897d216ae9426977dbf75115616" + }, + "verification": { + "retreats": 0, + "stage_atom_checked": 144, + "stage_atom_failures": 0 + } + }, + { + "artifacts": { + "batteries.json": "def93afec18fbb5c745101af8e47917846d79eb2a9101343f043961424c6f77a", + "demand.json": "a970e77c3d2868d2146bb537a7a5eb267c2a7a5abc008cb8935fd53834fc863e", + "network.json": "1e6fa2d22de83f964ff323584a4019c3757bc0c9655cf12a47f32c4a9718f7c8" + }, + "battery": { + "digest": "080ec58e2f864495bb0413626fd8f2bb30e9c4b5814d14dbc5c128d6eb8af28a", + "energy_budget_puh": 112.14001091250003, + "power_budget_pu": 14.017501364062504, + "power_max_pu": 0.19289651199232657, + "power_min_pu": 0.003993007133630577 + }, + "case": "pglib_opf_case793_goc", + "counts": { + "battery": 159, + "branch": 913, + "bus": 793, + "eligible_bus": 503, + "gen": 214, + "load": 507 + }, + "kappa_case": 1.0620703125, + "kappa_evaluations": 10, + "kappa_max": 1.11796875, + "kappa_retreats": 0, + "network_sha256": "bb070579a5095543a2afb5534f1d3b294a12749a505aa934dec776eb19c02bd0", + "nominal_demand_pu": 131.98280000000003, + "peak_demand_pu": 140.17501364062502, + "placement": { + "algorithm": "sha256-exponential-key weighted sampling without replacement", + "buses": [ + 20, + 28, + 36, + 54, + 58, + 64, + 75, + 77, + 81, + 89, + 97, + 99, + 101, + 108, + 112, + 118, + 129, + 131, + 132, + 134, + 137, + 142, + 145, + 152, + 153, + 168, + 171, + 173, + 191, + 195, + 208, + 214, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 229, + 231, + 235, + 236, + 243, + 245, + 248, + 249, + 251, + 252, + 253, + 254, + 257, + 274, + 277, + 279, + 294, + 298, + 303, + 304, + 308, + 313, + 331, + 336, + 342, + 345, + 348, + 351, + 355, + 363, + 367, + 377, + 378, + 381, + 387, + 392, + 398, + 401, + 405, + 413, + 417, + 441, + 443, + 447, + 454, + 457, + 460, + 477, + 480, + 482, + 484, + 486, + 487, + 488, + 489, + 494, + 497, + 498, + 500, + 501, + 503, + 506, + 511, + 512, + 514, + 520, + 521, + 523, + 524, + 527, + 534, + 541, + 542, + 547, + 550, + 553, + 579, + 581, + 586, + 593, + 599, + 600, + 604, + 610, + 617, + 626, + 627, + 630, + 631, + 635, + 641, + 642, + 645, + 647, + 649, + 657, + 667, + 674, + 675, + 676, + 678, + 679, + 683, + 684, + 688, + 689, + 690, + 698, + 700, + 702, + 707, + 711, + 720, + 740, + 749, + 751, + 762, + 780, + 781 + ], + "count": 159, + "digest": "ff0736165167fc129018428e7950345c68d0c242bb1bbc863bb9efea83e5e409", + "eligible": 503 + }, + "protocols": { + "final": { + "num_scenarios": 500, + "num_stages": 24, + "seed": 10432006, + "sha256": "afb76107757abcda55945ea17dce03c06aea7e6d733134cfaf995c70a3db3eb5" + }, + "screening": { + "excludes": "final", + "num_scenarios": 32, + "num_stages": 24, + "seed": 1605791441, + "sha256": "bc830a6b6304b7c35bf6b93499f5b880c926765c3dac77310cb1b445864c74dc" + } + }, + "regions": { + "corridors": [ + 23, + 96, + 104, + 125, + 130, + 140, + 144, + 150, + 169, + 188, + 203, + 209, + 215, + 218, + 219, + 220, + 222, + 224, + 231, + 238, + 251, + 253, + 330, + 340, + 348, + 349, + 356, + 362, + 367, + 376, + 382, + 391, + 395, + 398, + 399, + 402, + 405, + 412, + 415, + 418, + 423, + 431, + 434, + 436, + 437, + 439, + 442, + 485, + 562, + 573, + 670, + 673, + 677, + 680, + 686, + 690, + 694, + 698, + 705, + 716, + 735, + 744, + 755, + 852 + ], + "count": 6, + "demand_pu": [ + 36.59234999999999, + 26.579069999999984, + 23.4803, + 16.432589999999983, + 15.272560000000004, + 13.823730000000005 + ], + "demand_share": [ + 0.27683601073077285, + 0.20108147489117154, + 0.17763801949756625, + 0.12431922687595598, + 0.1155431281141106, + 0.10458213989042271 + ], + "digest": "c048efd9fb8d90fc320c8fd389b370fd725e981bbfe9258848502f01dd10cf1a", + "dispersion": 0.4575205489371816, + "effective_regions": 5.308697833704804, + "share_cap": 0.28, + "single_bus_exception": false, + "sizes": [ + 96, + 173, + 28, + 125, + 53, + 28 + ], + "sweeps": 6, + "unconstrained_dispersion": 0.4971951036700998, + "unconstrained_share": [ + 0.41967467238006173, + 0.20336516856482714, + 0.17763801949756625, + 0.12431922687595598, + 0.05460521438093034, + 0.020397698300658344 + ] + }, + "support": { + "sha256": "cfe6aac5690e50b524d5856291993365cbb11a73acfe8000b8561f9b5601b0fd" + }, + "verification": { + "retreats": 0, + "stage_atom_checked": 144, + "stage_atom_failures": 0 + } + }, + { + "artifacts": { + "batteries.json": "c3855131854e197fd36638a1ec43d13af4b9d651e346c0748f2f9b7590bdb5e2", + "demand.json": "462cd27c15dd09bb3949b125a7efecc41677b7b9c102356583b9e258e72f6de1", + "network.json": "3008f8f8453c37e28efdc3deb12cbb5595f1663ed49fc3b90403d4b7a1f90d0a" + }, + "battery": { + "digest": "76dafcaed3eb9f31a7a69a0b8ad2a30d4a7f6b410f5420973ebb89c9bea45dd7", + "energy_budget_puh": 537.9018203750002, + "power_budget_pu": 67.23772754687502, + "power_max_pu": 0.6893939841181314, + "power_min_pu": 0.02521692280586277 + }, + "case": "pglib_opf_case1354_pegase", + "counts": { + "battery": 240, + "branch": 1991, + "bus": 1354, + "eligible_bus": 621, + "gen": 260, + "load": 673 + }, + "kappa_case": 0.9203125, + "kappa_evaluations": 13, + "kappa_max": 0.96875, + "kappa_retreats": 0, + "network_sha256": "0d83e4fec87e37090ce671ba0e5452c1c52de4243074283eee682b3b4a708c9b", + "nominal_demand_pu": 730.5967000000003, + "peak_demand_pu": 672.3772754687502, + "placement": { + "algorithm": "sha256-exponential-key weighted sampling without replacement", + "buses": [ + 10, + 21, + 26, + 59, + 118, + 171, + 174, + 207, + 216, + 280, + 305, + 333, + 346, + 408, + 455, + 513, + 594, + 608, + 641, + 658, + 678, + 707, + 747, + 772, + 883, + 905, + 908, + 953, + 980, + 1026, + 1035, + 1081, + 1129, + 1156, + 1159, + 1179, + 1183, + 1249, + 1265, + 1301, + 1380, + 1398, + 1486, + 1541, + 1562, + 1592, + 1607, + 1625, + 1767, + 1813, + 1860, + 1866, + 1917, + 1923, + 1965, + 2019, + 2057, + 2089, + 2128, + 2166, + 2252, + 2286, + 2288, + 2327, + 2360, + 2377, + 2432, + 2526, + 2535, + 2558, + 2563, + 2591, + 2598, + 2629, + 2654, + 2676, + 2732, + 2848, + 2898, + 2940, + 2968, + 3021, + 3037, + 3072, + 3083, + 3121, + 3145, + 3200, + 3221, + 3344, + 3377, + 3391, + 3499, + 3526, + 3613, + 3643, + 3645, + 3657, + 3707, + 3718, + 3758, + 3760, + 3834, + 3866, + 3929, + 3975, + 3994, + 4032, + 4103, + 4144, + 4185, + 4189, + 4239, + 4300, + 4313, + 4314, + 4324, + 4353, + 4426, + 4505, + 4511, + 4562, + 4656, + 4674, + 4683, + 4710, + 4738, + 4747, + 4787, + 4829, + 4831, + 4867, + 4885, + 4936, + 4939, + 4942, + 4950, + 4951, + 5003, + 5049, + 5106, + 5213, + 5256, + 5286, + 5317, + 5400, + 5410, + 5419, + 5420, + 5441, + 5469, + 5525, + 5574, + 5691, + 5720, + 5735, + 5764, + 5853, + 5957, + 6053, + 6071, + 6101, + 6110, + 6151, + 6178, + 6246, + 6357, + 6382, + 6416, + 6478, + 6495, + 6510, + 6630, + 6639, + 6691, + 6791, + 6828, + 6846, + 6891, + 6909, + 6926, + 6952, + 7050, + 7069, + 7070, + 7129, + 7132, + 7226, + 7256, + 7273, + 7342, + 7380, + 7473, + 7579, + 7624, + 7626, + 7640, + 7700, + 7752, + 7770, + 7775, + 7809, + 7895, + 7905, + 7937, + 7955, + 7972, + 7982, + 8057, + 8180, + 8293, + 8439, + 8448, + 8494, + 8497, + 8651, + 8653, + 8669, + 8689, + 8691, + 8704, + 8707, + 8732, + 8748, + 8788, + 8808, + 8809, + 8825, + 8843, + 8853, + 8854, + 8874, + 8893, + 9018, + 9019, + 9021, + 9045, + 9130, + 9203, + 9231 + ], + "count": 240, + "digest": "8c41eb42454a6db54ceaa6aa2a6a29e7a06a7c24fe2081333a6a2ca6e780dc84", + "eligible": 621 + }, + "protocols": { + "final": { + "num_scenarios": 500, + "num_stages": 24, + "seed": 795941078, + "sha256": "064233cdf608bf555c4c6482141a235d3a358d7f50b0178b61fa8b68d7f9515f" + }, + "screening": { + "excludes": "final", + "num_scenarios": 32, + "num_stages": 24, + "seed": 1023704750, + "sha256": "486b17892ce1a515945e353e6442b18eb96d3a4be4879ae696d4585e73c1a5a3" + } + }, + "regions": { + "corridors": [ + 212, + 247, + 250, + 251, + 253, + 255, + 299, + 491, + 492, + 494, + 498, + 499, + 523, + 524, + 527, + 536, + 615, + 672, + 694, + 695, + 696, + 697, + 706, + 713, + 716, + 721, + 733, + 736, + 787, + 855, + 856, + 857, + 872, + 897, + 914, + 942, + 1005, + 1006, + 1009, + 1018, + 1019, + 1020, + 1038, + 1086, + 1147, + 1148, + 1150, + 1213, + 1226, + 1248, + 1264, + 1287, + 1288, + 1361, + 1421, + 1438, + 1451, + 1579, + 1638, + 1678, + 1754, + 1825, + 1826, + 1868 + ], + "count": 6, + "demand_pu": [ + 207.60800000000015, + 207.36730000000003, + 116.14960000000006, + 88.70859999999999, + 62.263900000000014, + 59.36269999999998 + ], + "demand_share": [ + 0.27999888328448164, + 0.2796742535437847, + 0.1566498318655313, + 0.11964042299781197, + 0.08397471421590992, + 0.08006189409248045 + ], + "digest": "3dd1185169fc4ee8e5b06d4007dc40c10f355c76f2dcbebea8af9479dc0317e1", + "dispersion": 0.25191808777795155, + "effective_regions": 4.786252590935108, + "share_cap": 0.28, + "single_bus_exception": false, + "sizes": [ + 222, + 116, + 113, + 70, + 47, + 53 + ], + "sweeps": 7, + "unconstrained_dispersion": 0.2902143589632836, + "unconstrained_share": [ + 0.6700181169559898, + 0.1202670244831785, + 0.11964042299781197, + 0.07370592159982713, + 0.009687372253746355, + 0.006681141709445996 + ] + }, + "support": { + "sha256": "dadc8cb29b2ef9ad0db9e04caf6908ba07a88c1991b0ee032f473bf1b83d45f2" + }, + "verification": { + "retreats": 0, + "stage_atom_checked": 144, + "stage_atom_failures": 0 + } + }, + { + "artifacts": { + "batteries.json": "ab965cf8052d351958eb6168792cf13a0ad7e3cd96651154c7cc64ff50b2abe5", + "demand.json": "dfee214e7eef1f9d25afa303bb81973b20faeb48ee15355d472267e157bb2dd7", + "network.json": "ccd0c31ba1e6cd862f2600c8cf29ba0041b5cef6a53fdecfd0a8904e354b3253" + }, + "battery": { + "digest": "a399dcb321cc1eaa3a85aaed3c8417529e68cf80e41709fa36c07fd7c1635ad9", + "energy_budget_puh": 287.98747422024024, + "power_budget_pu": 35.99843427753003, + "power_max_pu": 0.410437406270644, + "power_min_pu": 0.02670081554590348 + }, + "case": "pglib_opf_case2000_goc", + "counts": { + "battery": 240, + "branch": 3639, + "bus": 2000, + "eligible_bus": 1010, + "gen": 384, + "load": 1010 + }, + "kappa_case": 1.0917578124999998, + "kappa_evaluations": 10, + "kappa_max": 1.14921875, + "kappa_retreats": 0, + "network_sha256": "3bdcedc723e36b8bd1711265704c38e19027120a13e40e5c1b7a77069f54bade", + "nominal_demand_pu": 329.72912000599985, + "peak_demand_pu": 359.9843427753003, + "placement": { + "algorithm": "sha256-exponential-key weighted sampling without replacement", + "buses": [ + 3, + 5, + 6, + 7, + 11, + 13, + 16, + 21, + 27, + 30, + 45, + 75, + 92, + 95, + 99, + 108, + 128, + 140, + 145, + 162, + 163, + 167, + 169, + 179, + 199, + 204, + 206, + 213, + 225, + 226, + 229, + 236, + 237, + 241, + 245, + 246, + 247, + 255, + 259, + 262, + 267, + 270, + 282, + 285, + 286, + 287, + 290, + 295, + 298, + 302, + 303, + 315, + 320, + 330, + 336, + 339, + 346, + 356, + 357, + 378, + 384, + 388, + 402, + 414, + 423, + 432, + 436, + 455, + 463, + 468, + 475, + 480, + 484, + 490, + 491, + 507, + 607, + 624, + 627, + 629, + 633, + 634, + 635, + 649, + 652, + 654, + 656, + 659, + 664, + 667, + 672, + 681, + 697, + 701, + 704, + 707, + 709, + 738, + 742, + 754, + 756, + 761, + 762, + 764, + 767, + 774, + 777, + 778, + 788, + 789, + 792, + 805, + 807, + 820, + 822, + 826, + 828, + 831, + 832, + 834, + 838, + 852, + 854, + 859, + 862, + 865, + 869, + 870, + 871, + 884, + 898, + 899, + 908, + 914, + 927, + 935, + 947, + 959, + 976, + 978, + 980, + 992, + 1033, + 1041, + 1047, + 1062, + 1064, + 1071, + 1084, + 1085, + 1088, + 1096, + 1097, + 1113, + 1123, + 1138, + 1140, + 1143, + 1148, + 1149, + 1156, + 1159, + 1163, + 1176, + 1186, + 1193, + 1199, + 1204, + 1491, + 1500, + 1503, + 1518, + 1523, + 1524, + 1526, + 1530, + 1538, + 1552, + 1555, + 1556, + 1561, + 1565, + 1569, + 1576, + 1577, + 1578, + 1583, + 1584, + 1586, + 1588, + 1592, + 1599, + 1604, + 1605, + 1606, + 1608, + 1610, + 1628, + 1629, + 1640, + 1641, + 1642, + 1648, + 1651, + 1653, + 1656, + 1664, + 1667, + 1669, + 1677, + 1678, + 1687, + 1696, + 1697, + 1703, + 1710, + 1717, + 1720, + 1730, + 1733, + 1742, + 1746, + 1748, + 1755, + 1767, + 1772, + 1778, + 1779, + 1780, + 1781, + 1784, + 1785, + 1787, + 1790, + 1829, + 1833, + 1837, + 1841, + 1846, + 1854 + ], + "count": 240, + "digest": "8d3b59e4c862d424acf88d23d879624322fafb07b8860fbd30d97420be9b6302", + "eligible": 1010 + }, + "protocols": { + "final": { + "num_scenarios": 500, + "num_stages": 24, + "seed": 130404710, + "sha256": "1e0c76896159f6dc875d037f4beda627a1ef267b53a2f8a974d0f1ddae579e1b" + }, + "screening": { + "excludes": "final", + "num_scenarios": 32, + "num_stages": 24, + "seed": 99727030, + "sha256": "5628d0fb58fbb94a38e0a3712dc1e1eeb934959c057a53b64a2a9d51d364faf3" + } + }, + "regions": { + "corridors": [ + 558, + 566, + 696, + 737, + 739, + 740, + 755, + 797, + 855, + 891, + 892, + 893, + 951, + 991, + 992, + 1042, + 1072, + 1099, + 1103, + 1104, + 1123, + 1124, + 1144, + 1147, + 1168, + 1169, + 1175, + 1178, + 1180, + 1183, + 1185, + 1186, + 1496, + 1505, + 1507, + 1509, + 1511, + 1515, + 1517, + 1518, + 1752, + 1763, + 1793, + 1800, + 1812, + 1814, + 1816, + 1820, + 1822, + 1827, + 1838, + 1865, + 1878, + 1927, + 1932, + 1943, + 1979, + 1992, + 1995, + 2015, + 2048, + 2072, + 2074, + 2075 + ], + "count": 6, + "demand_pu": [ + 92.32391831009997, + 92.250487199, + 65.82313664889998, + 26.568426782999996, + 26.382714007999997, + 26.380437057000005 + ], + "demand_share": [ + 0.2799992864094624, + 0.27977658508693853, + 0.199627914718913, + 0.08057652530815763, + 0.08001329699821455, + 0.080006391478314 + ], + "digest": "c90cf8d0471875eba5d2ef849df2c533491f40ed8a5e2a131b992f582bd4a701", + "dispersion": 0.1389876666613602, + "effective_regions": 4.633457185050667, + "share_cap": 0.28, + "single_bus_exception": false, + "sizes": [ + 338, + 197, + 188, + 105, + 105, + 77 + ], + "sweeps": 10, + "unconstrained_dispersion": 0.12661299284635213, + "unconstrained_share": [ + 0.5492041578120396, + 0.2903165222933848, + 0.05470939320637421, + 0.04754150857441636, + 0.0429705435896659, + 0.015257874524119842 + ] + }, + "support": { + "sha256": "c6ec8360619dcdd0074196da0fc8fcb09248990b5da2f8f6dae851aca519c06d" + }, + "verification": { + "retreats": 0, + "stage_atom_checked": 144, + "stage_atom_failures": 0 + } + } + ], + "command": "julia --project=. battery_portfolio.jl --case --out ", + "digest": "11c2f58fabbac489812d86fa2aef95833c54acaac12347337e54940a29ccb1d9", + "horizon": 24, + "panel": { + "accepted": [ + "pglib_opf_case118_ieee", + "pglib_opf_case1951_rte", + "pglib_opf_case2383wp_k", + "pglib_opf_case240_pserc", + "pglib_opf_case300_ieee", + "pglib_opf_case500_goc", + "pglib_opf_case588_sdet", + "pglib_opf_case793_goc", + "pglib_opf_case1354_pegase", + "pglib_opf_case2000_goc" + ], + "primary": [ + "pglib_opf_case118_ieee", + "pglib_opf_case162_ieee_dtc", + "pglib_opf_case179_goc", + "pglib_opf_case200_activ", + "pglib_opf_case240_pserc", + "pglib_opf_case300_ieee", + "pglib_opf_case500_goc", + "pglib_opf_case588_sdet", + "pglib_opf_case793_goc", + "pglib_opf_case1354_pegase", + "pglib_opf_case1888_rte", + "pglib_opf_case2000_goc" + ], + "replacement_rule": "a primary case is replaced only when it is unavailable from the pinned PGLib version or when its unmodified base ACP fails at demand level 0.5; a SOC, DC or method failure is never a reason to replace a case", + "replacements": [ + { + "dropped": "pglib_opf_case162_ieee_dtc", + "promoted": "pglib_opf_case1951_rte", + "reason": "the unmodified base ACP clears the gate at no level of [0.5, 1.25]; the conservative level 0.5 fails" + }, + { + "dropped": "pglib_opf_case179_goc", + "promoted": "pglib_opf_case2312_goc", + "reason": "the unmodified base ACP clears the gate at no level of [0.5, 1.25]; the conservative level 0.5 fails" + }, + { + "dropped": "pglib_opf_case2312_goc", + "promoted": "pglib_opf_case2383wp_k", + "reason": "the unmodified base ACP clears the gate at no level of [0.5, 1.25]; the conservative level 0.5 fails" + }, + { + "dropped": "pglib_opf_case200_activ", + "promoted": "pglib_opf_case2736sp_k", + "reason": "the unmodified base ACP clears the gate at no level of [0.5, 1.25]; the conservative level 0.5 fails" + }, + { + "dropped": "pglib_opf_case2736sp_k", + "promoted": "", + "reason": "the unmodified base ACP clears the gate at no level of [0.5, 1.25]; the conservative level 0.5 fails" + }, + { + "dropped": "pglib_opf_case1888_rte", + "promoted": "", + "reason": "the unmodified base ACP clears the gate at no level of [0.5, 1.25]; the conservative level 0.5 fails" + } + ], + "reserve": [ + "pglib_opf_case1951_rte", + "pglib_opf_case2312_goc", + "pglib_opf_case2383wp_k", + "pglib_opf_case2736sp_k" + ], + "unfilled": [ + { + "last_candidate": "pglib_opf_case2736sp_k", + "reason": "the preregistered reserve list is exhausted", + "slot": "pglib_opf_case200_activ" + }, + { + "last_candidate": "pglib_opf_case1888_rte", + "reason": "the preregistered reserve list is exhausted", + "slot": "pglib_opf_case1888_rte" + } + ] + }, + "profile": [ + 0.72, + 0.68, + 0.65, + 0.64, + 0.66, + 0.72, + 0.8, + 0.88, + 0.94, + 0.98, + 1.0, + 0.99, + 0.97, + 0.95, + 0.94, + 0.96, + 1.0, + 1.0, + 0.98, + 0.94, + 0.9, + 0.84, + 0.79, + 0.75 + ], + "protocols": { + "algorithm": "stage-major StableRNG draw from the frozen support; the screening protocol is drawn from an independent seed and repaired against the final protocol's columns, so the two panels are disjoint by construction", + "final_scenarios": 500, + "num_stages": 24, + "screening_scenarios": 32, + "seed_rule": "SHA256(\"battery_storage_opf/portfolio/1\\nprotocol/\\n20260814\\n\\nseed\\n\") reduced into 1:2^31-1" + }, + "regions": { + "algorithm": "demand-BALANCED assignment over unit-norm PTDF sensitivity signatures on the highest-reach rated corridors: farthest-point initialization, then Lloyd sweeps whose assignment step is a HiGHS integer program minimizing demand-weighted signature distance subject to per-region demand-share bounds; regions relabelled by descending demand", + "balance_iterations": 20, + "count": 6, + "high": 1.15, + "low": 0.97, + "max_corridors": 64, + "min_effective_regions": 4.5, + "modes": [ + 1.15, + 0.97 + ], + "probabilities": [ + 0.16666666666666666, + 0.16666666666666666, + 0.16666666666666666, + 0.16666666666666666, + 0.16666666666666666, + 0.16666666666666666 + ], + "share_max": 0.28, + "share_min": 0.08, + "support_mean": 1.0 + }, + "schema": "battery_storage_opf/portfolio/1", + "seed": 20260814, + "stage_hours": 1.0, + "versions": { + "Clarabel": "0.11.1", + "Ipopt": "1.15.0", + "JuMP": "1.31.1", + "PGLib": "0.2.2", + "PowerModels": "0.21.6", + "SDDP": "1.14.0", + "StableRNGs": "1.0.4", + "julia": "1.12.5" + } +} \ No newline at end of file diff --git a/examples/BatteryStorageOPF/battery_powermodels.jl b/examples/BatteryStorageOPF/battery_powermodels.jl new file mode 100644 index 0000000..208763d --- /dev/null +++ b/examples/BatteryStorageOPF/battery_powermodels.jl @@ -0,0 +1,1332 @@ +# battery_powermodels.jl +# +# The battery layer on top of PowerModels.jl, and nothing else. +# +# THE BOUNDARY, stated once and enforced by the tests: +# +# PowerModels.jl owns buses, generators, branches, voltage variables, the +# reference angle, Ohm's law at both branch ends, +# transformer taps and phase shifts, shunts, angle- +# difference limits, apparent-power limits at both ends, +# the nodal active and reactive balances, generator +# bounds and generator cost. +# this file owns batteries: their energy state, charge/discharge +# controls, state transition, unity-power-factor active +# injection, throughput cost, the strict outgoing-energy +# target equality, the two-sided nodal active-power +# recourse, and the demand parameterization. +# +# There is no handwritten AC trigonometry and no handwritten SOC-WR lifted +# branch equation anywhere below. Selecting `PowerModels.ACPPowerModel` or +# `PowerModels.SOCWRConicPowerModel` selects the network formulation and nothing +# in this file changes. +# +# HOW THE BATTERY REACHES THE NODAL BALANCE. +# PowerModels' `constraint_power_balance` is form-specific and closed: it writes +# the whole balance in one `@constraint`. Its ONLY documented extension point +# for an additional nodal injection is the `storage` component, which enters +# every form's balance as `- sum(ps[s] for s in bus_storage)` on the injection +# side and `- sum(qs[s] ...)` on the reactive side. This file therefore installs +# exactly ONE storage element per bus as an INJECTION CARRIER and defines what +# flows through it: +# +# ps[i] = Δp^d_i - p^{bat}_i - d_i + s_i qs[i] = Δq^d_i +# +# so the stock balance PowerModels writes reads, at bus i, +# +# Σ p_arcs = Σ pg + p^{bat}_i + d_i - s_i - (p^d_i + Δp^d_i) - g^s_i |V_i|² +# Σ q_arcs = Σ qg - (q^d_i + Δq^d_i) + b^s_i |V_i|² +# +# which is the model in the plan: the battery injects at unity power factor, the +# recourse pair enters the ACTIVE balance with opposite signs, the reactive +# balance is HARD, and demand is parameterized by the fixed deviations Δp^d, Δq^d +# that carry the realized stage/atom demand. `ps` and `qs` are JuMP EXPRESSIONS, +# not variables, so this adds no variable and no equality row of its own. + +using JuMP +using PowerModels +using Ipopt +using Clarabel +using HiGHS +using LinearAlgebra +using Printf +import MathOptInterface as MOI + +# Include guards: several public files include the same shared sources, and a +# second `include` of a file that defines a struct is an error, not a no-op. +@isdefined(BatterySpec) || include(joinpath(@__DIR__, "battery_case.jl")) +@isdefined(SolutionRecorder) || include(joinpath(@__DIR__, "battery_solution_schema.jl")) + +PowerModels.silence() + +# ───────────────────────────────────────────────────────────────────────────── +# Solvers +# +# One definition each, used by the diagnostics, by the SDDP baseline and by the +# regression suite. Two engines that disagree because they were solved at +# different tolerances is a failure mode this study has already paid for once. +# ───────────────────────────────────────────────────────────────────────────── + +""" + acp_optimizer(; tol=1e-10, constr_viol_tol=1e-10, max_iter=3000) + -> JuMP optimizer factory + +Interior-point NLP solver for the true-ACP model. + +# Notes +Tolerance is tightened well below the physical tolerances the study reports at, +because a variable parked a solver tolerance below its bound shifts a +positively-priced objective by a near-constant amount every stage, which then +looks like a systematic model difference between two engines. + +`constr_viol_tol` is set explicitly and is NOT implied by `tol`. Ipopt's `tol` +governs the SCALED NLP error, while `constr_viol_tol` — default `1e-4` — is the +absolute cap on constraint violation, and the gap between them is visible in the +physical residuals: measured on `case240_pserc`, `case179_goc` and +`case162_ieee_dtc` at two demand levels each, `tol = 1e-10` alone leaves the +apparent-power limits violated by `8e-7` to `2e-6` pu and the voltage bounds by +`1.1e-8` (Ipopt's `bound_relax_factor`), while every equation the network is +actually built from — Ohm's law, both nodal balances, the battery transition — +already sits at `1e-11` or below. Adding `constr_viol_tol = 1e-10` moves the +limit violations to `1e-11` and the bound violations to `1e-10`, and on the +slowest of those solves it was eighteen times FASTER rather than slower. + +One setting, measured once, used for every case and every demand level of the +portfolio. It is not a per-case adjustment and there is no retry ladder: a solve +that does not converge under it is reported as it stands. Turning scaling off +entirely and zeroing `bound_relax_factor` drives the residuals to exact zeros but +made `case179_goc` fail outright at a demand level it otherwise solves, so it was +rejected. +""" +acp_optimizer(; tol::Real = 1e-10, constr_viol_tol::Real = 1e-10, + bound_relax_factor::Real = ACP_BOUND_RELAX_FACTOR, + max_iter::Integer = 3000) = + JuMP.optimizer_with_attributes(Ipopt.Optimizer, + "print_level" => 0, + "tol" => tol, + "constr_viol_tol" => constr_viol_tol, + # Stated explicitly, and shared with the Exa + # engine, so both engines' ACP solves relax + # bounds by the SAME amount. Ipopt's own + # default happens to equal it; MadNLP's does + # not, and an inherited default is not a + # cross-engine agreement. + "bound_relax_factor" => Float64(bound_relax_factor), + "max_iter" => Int(max_iter), + "sb" => "yes") + +""" + socwr_optimizer(; tol=1e-8, equilibrate=true, max_iter=10_000) + -> JuMP optimizer factory + +Conic solver for the SOC-WR relaxation. + +# Notes +Nothing here retries a failed solve at a different setting: a retry ladder +chooses which duals become cuts, and cut generation must stay stock. The +settings are therefore chosen ONCE, by measurement, to be the ones under which +the solver does not fail in the first place. Two independent measurements fix +them, and they pull in opposite directions: + +**Tolerance and iteration cap, measured 2026-08-06.** These are ONE frozen +correctness configuration, chosen once and applied to every candidate. They +correct an invalid inherited setting; they are not tuned per case. + +The earlier default of `tol = 1e-6, max_iter = 500` did not merely lose +precision — it returned the WRONG ANSWER while reporting `OPTIMAL`. Solves are +bit-reproducible (five repeats, spread exactly 0.00), so this is not noise. On +one cycle-module case with the storage target pinned: + +| tolerance | max_iter | status | objective | +|---|---|---|---| +| `1e-6` | 500 … 50,000 | OPTIMAL | 296,951 | +| `1e-8` | 500 | ALMOST_OPTIMAL | 323,913 | +| `1e-8` | 2,000 | ALMOST_OPTIMAL | 350,788 | +| **`1e-8`** | **10,000** | **OPTIMAL** | **351,411** | +| `1e-8` | 50,000 | OPTIMAL | 351,411 | + +The true optimum is 351,411, against ACP's 358,806 — a 2.1 % relaxation gap. At +`1e-6` the reported value is 15.5 % BELOW it, and no iteration budget helps, +because termination is on the loose tolerance itself. `max_iter = 500` was a +budget inherited from a 57-bus case; a several-hundred-bus conic program needs +far more. + +Why this matters beyond precision: SDDP consumes these solves' DUALS as cuts. +Objectives wrong by percent-level amounts, varying with the pinned state, make +successive cuts mutually inconsistent — which is how individually-healthy +subproblems become a collectively infeasible SDDP. + +`max_iter` is a CAP, not a target; record the iterations and time actually used. + +**Objective convergence is necessary but NOT sufficient.** A production +subproblem must satisfy all three: `OPTIMAL` status, a stable objective, and +stable STATE DUALS validated against finite differences (or one-sided brackets +at a kink). The duals can still move after the objective has settled, and the +duals are what become cuts. `ALMOST_OPTIMAL` may be kept as diagnostic evidence +but can never generate an accepted cut or a headline number. + +**Equilibration is ON, and the objective is PHYSICAL, measured 2026-08-10.** +The two settings were tested together, as a 2×2×2 sweep over objective scaling, +equilibration and tolerance, at two BLAS thread counts, on `case500_goc` bare +and with one cycle module added to it. Only one corner solves the problem that +was posed: + +| objective | equilibration | status | objective | max violation | +|---|---|---|---|---| +| **physical** | **on** | **OPTIMAL**, 27–30 iterations | **453 838.4553 / 617 807.7916** | **≤6.3e-9** | +| physical | off | OPTIMAL, 97–208 iterations | 453 590.70 / 617 387.25 | 3.5e-4 | +| divided by the recourse price | on | NUMERICAL_ERROR / SLOW_PROGRESS | — | — | +| divided by the recourse price | off | ALMOST_OPTIMAL / NUMERICAL_ERROR | — | — | + +Read the second row before the third: with equilibration off, the lower +objective is not a better optimum, it is a point that violates the conic model +by 2.4e-4–3.5e-4 and is not feasible. Only the first row returns a converged +value at a residual worth quoting, and it is the only one that is `OPTIMAL` at +both thread counts on both arms. Tightening to `1e-10` reproduces its objective +to four decimals in one extra iteration. + +Rescaling the objective is therefore rejected as a numerical device: a solver +whose own equilibration already normalizes rows and columns gains nothing from +a second, cruder rescaling of one block, and Clarabel measurably fails on the +rescaled program. Everything this file builds, solves and reports is in one +physical unit system — the objective units of the PGLib case — with no +conversion anywhere. + +The lesson stands, only pointed the other way: a setting that makes an isolated +probe stop failing is not thereby the setting that makes the whole method run. +""" +socwr_optimizer(; tol::Real = 1e-8, equilibrate::Bool = true, + max_iter::Integer = 10_000) = + JuMP.optimizer_with_attributes(Clarabel.Optimizer, + "verbose" => false, + "tol_gap_abs" => tol, + "tol_gap_rel" => tol, + "tol_feas" => tol, + "equilibrate_enable" => equilibrate, + "max_iter" => Int(max_iter)) + +""" + dc_optimizer() -> JuMP optimizer factory + +Solver for the DC-approximation backward model: HiGHS, at its own defaults, +with output suppressed. + +# Notes +**What the DC backward subproblem actually is.** Under `DCPPowerModel` the +network equations are linear, the battery transition and its bounds are linear, +every state and control carries a finite bound, the emergency active recourse +pair is unbounded above at a positive price, and the generator cost is the +convex polynomial PGLib supplies. The subproblem is therefore a convex +quadratic program with a linear feasible set — a mature QP/LP solver's home +territory, not a conic one's. + +**Why this is no longer `socwr_optimizer`.** Sharing one solver across both arms +was chosen so the arms would differ only by FORMULATION. That reasoning bought +nothing once measured: an interior-point conic solver run on this LP returned +`INFEASIBLE`, `DUAL_INFEASIBLE`, `LOCALLY_INFEASIBLE` and `SLOW_PROGRESS` on +nine of the ten portfolio cases, at stage nodes between 3 and 22, on subproblems +whose own primal feasibility is not in question. The conditioning that provokes +it is a right-hand-side range spanning roughly ten orders of magnitude — the +reactive-demand deviations sit near `5e-4` while the recourse prices sit near +`7e5` — which presolve and a simplex/QP basis absorb and an unpreconditioned +conic IPM does not. + +The portfolio compares PRACTICAL, STRONG baselines. A DC-SDDP baseline that +cannot finish an iteration is not the DC formulation's result, it is the +solver's, and reporting it as the former would be wrong. So the DC arm gets the +solver its problem class calls for, and the SOC arm keeps +[`socwr_optimizer`](@ref) — the formulations still differ by exactly one thing, +and now each is solved by something that can solve it. + +**Defaults, deliberately, and the same ones for every case.** Nothing here is +tuned, and nothing here may be tuned per case: a solver setting chosen in +response to one case's result is a free parameter fitted to that case. The only +non-default is `output_flag`, which is presentation, not numerics. +""" +dc_optimizer() = JuMP.optimizer_with_attributes(HiGHS.Optimizer, + "output_flag" => false) + +const ACCEPTED_STATUSES = (MOI.OPTIMAL, MOI.LOCALLY_SOLVED) + +""" + worst_recourse(sol) -> Float64 + +The largest USE of physical active-power recourse in a solved stage, in pu. + +# Notes +Clamped at zero on purpose. An interior-point solver parks a nonnegative variable +a tolerance BELOW its zero bound — this study routinely sees `-1e-8` — and a +"worst recourse" of `-1e-8` is not a negative amount of unserved load, it is +zero. Reporting the raw minimum would also make the admissibility rule compare a +negative number against a positive tolerance and pass for the wrong reason. +""" +worst_recourse(sol) = max(0.0, + maximum(values(sol.deficit); init = 0.0), + maximum(values(sol.surplus); init = 0.0)) + +# ───────────────────────────────────────────────────────────────────────────── +# Injection carriers +# ───────────────────────────────────────────────────────────────────────────── + +""" + network_with_carriers(case::BatteryCase) -> Dict{String,Any} + +Return a copy of the frozen network augmented with one PowerModels `storage` +element per bus. + +# Arguments +- `case::BatteryCase`: the frozen case. + +# Returns +- A deep copy of `case.network` whose `"storage"` table has exactly one entry + per bus, keyed by the bus identifier as a string. + +# Notes +The storage elements carry no dynamics of their own: this file never calls +`PowerModels.variable_storage_power`, `constraint_storage_state`, +`constraint_storage_losses`, `constraint_storage_complementarity_*` or +`constraint_storage_thermal_limit`. Their sole role is to make `ref[:bus_storage]` +nonempty so that the stock nodal balance contains a `ps`/`qs` term this file can +bind to an expression (see the file header). + +Their numeric fields are filled with neutral, valid values only so that +PowerModels' own data checks accept the table; they are never read by any +constraint that this problem specification builds. The battery ratings that +matter live in `case.batteries` and are enforced by this file's own bounds. + +The network is COPIED because PowerModels mutates the dictionaries it is given; +sharing one parse across two formulations is how a case export once stopped +being byte-reproducible. + +**Generator costs come through UNCHANGED.** The polynomial coefficients this +returns are the case's own, so `PowerModels.objective_min_fuel_and_flow_cost` +builds the physical generation cost and every stage model this file assembles is +in the case's physical objective units. A stock `PowerModels.solve_opf` on +`case.network` is then a valid external reference for any of them, comparable +without conversion. +""" +function network_with_carriers(case::BatteryCase) + net = deepcopy(case.network) + # Fail closed on component classes this study's cost accounting does not + # decompose. PowerModels would happily build them and their cost would then + # sit inside the objective but outside `cost_generation`, so a cross-engine + # objective comparison would disagree with the sum of its own parts. + isempty(get(net, "dcline", Dict())) || + error("network_with_carriers: HVDC lines are not supported by this study's cost decomposition") + isempty(get(net, "switch", Dict())) || + error("network_with_carriers: switches are not supported by this study") + haskey(net, "storage") && !isempty(net["storage"]) && + error("network_with_carriers: the case already declares storage; the injection carriers would collide with it") + + # The cost decomposition in `extract_stage_solution` re-evaluates each + # generator's polynomial at the reported dispatch, so a piecewise-linear cost + # model would land inside PowerModels' objective but outside that sum. + for (_, gen) in net["gen"] + haskey(gen, "cost") || continue + Int(get(gen, "model", 2)) == 2 || + error("network_with_carriers: only polynomial (model 2) generator costs are supported") + end + + storage = Dict{String,Any}() + for (_, bus) in net["bus"] + i = Int(bus["index"]) + storage[string(i)] = Dict{String,Any}( + "index" => i, + "storage_bus" => i, + "status" => 1, + # Neutral, valid, and unused: no constraint built here reads them. + "energy" => 0.0, "energy_rating" => 0.0, + "charge_rating" => 0.0, "discharge_rating" => 0.0, + "charge_efficiency" => 1.0, "discharge_efficiency" => 1.0, + "thermal_rating" => 0.0, "qmin" => 0.0, "qmax" => 0.0, + "r" => 0.0, "x" => 0.0, "p_loss" => 0.0, "q_loss" => 0.0, + "ps" => 0.0, "qs" => 0.0, + ) + end + net["storage"] = storage + return net +end + +# ───────────────────────────────────────────────────────────────────────────── +# Problem specification +# ───────────────────────────────────────────────────────────────────────────── + +""" + BatteryStateMode + +How the battery energy state is carried by a stage model. + +- `:sddp` the outgoing energy is an `SDDP.State` variable supplied by the + caller; the stage model has no target and no target multiplier. + This is the TARGETLESS formulation stock SDDP trains on. +- `:strict` the incoming energy is data, and the outgoing energy is pinned by + the HARD equality ``e_{b} = \\hat e_b`` whose multiplier is the + actor signal. There is no target slack and no target penalty. +- `:free` the outgoing energy is a free decision inside its own bounds and + there is no target at all. The incoming energy is data when + `state_in` is supplied, and a free variable — to be linked + externally, as a multiperiod deterministic equivalent does — when it + is not. This is the DIAGNOSTIC mode: a myopic one-stage solve, or one + stage of a perfect-foresight solve. +- `:none` the case's batteries are not installed at all. Used to show that + this problem specification reduces to ordinary PowerModels OPF. + +`:free` is not a third scientific formulation. It carries no target and +therefore no target multiplier, so no policy can be trained on it; it exists so +that the physics of a stage, and the value of the energy a stage inherits, can be +probed with the SAME builders the study's two formulations use, instead of with a +second model that would have to be validated all over again. +""" +const BATTERY_STATE_MODES = (:sddp, :strict, :free, :none) + +""" + BatterySpecification + +Everything the battery layer needs in order to build one stage model. + +# Fields +- `case::BatteryCase`: the frozen case. +- `stage::Int`: stage index ``t``, used only to select the stage's entry of the + frozen demand support. +- `atom::Int`: index of the realized demand atom at this stage. In an SDDP node + it is a placeholder overwritten by `SDDP.parameterize`. +- `mode::Symbol`: one of [`BATTERY_STATE_MODES`](@ref). +- `state_in`: `nothing`, or a `Dict{Int,Float64}` of incoming energies + (`:strict`, and optionally `:free`), or a `Dict{Int,Any}` of JuMP + incoming-state references (`:sddp`). +- `state_out`: `nothing`, or a `Dict{Int,Any}` of JuMP outgoing-state + references (`:sddp`). +- `target`: `nothing`, or a `Dict{Int,Float64}` of outgoing-energy targets + (`:strict`). + +# Notes +One specification builds either formulation of the network: the model +constructor passed to `PowerModels.instantiate_model` decides whether the +network equations are `ACPPowerModel`'s or `SOCWRConicPowerModel`'s, and this +struct is unchanged by that choice. +""" +struct BatterySpecification + case::BatteryCase + stage::Int + atom::Int + mode::Symbol + state_in::Any + state_out::Any + target::Any +end + +function BatterySpecification(case::BatteryCase; + stage::Integer = 1, + atom::Integer = 1, + mode::Symbol = :strict, + state_in = nothing, + state_out = nothing, + target = nothing) + mode in BATTERY_STATE_MODES || + throw(ArgumentError("mode must be one of $BATTERY_STATE_MODES, got :$mode")) + if mode === :strict + state_in === nothing && throw(ArgumentError(":strict requires incoming energies")) + target === nothing && throw(ArgumentError(":strict requires outgoing targets")) + elseif mode === :sddp + (state_in === nothing || state_out === nothing) && + throw(ArgumentError(":sddp requires the SDDP state references")) + elseif mode === :free + # A target in a targetless mode would be silently ignored, which is the + # one outcome worse than an error: the caller would believe a target was + # imposed and read a value that was never constrained. + target === nothing || throw(ArgumentError(":free is targetless; pass mode = :strict to impose a target")) + end + return BatterySpecification(case, Int(stage), Int(atom), mode, state_in, state_out, target) +end + +""" + build_battery_opf(pm::PowerModels.AbstractPowerModel, spec::BatterySpecification) + +The shared PowerModels problem specification for the battery-storage study. + +# Arguments +- `pm`: a model instantiated by `PowerModels.instantiate_model`; its concrete + type — `ACPPowerModel` or `SOCWRConicPowerModel` — selects the network + formulation and is not inspected here. +- `spec::BatterySpecification`: the battery layer's data for this stage. + +# Notes +The body is `PowerModels.build_opf` with the storage block replaced. Everything +electrical is a call into PowerModels; everything battery-related is a call into +this file. That split is the point of the whole design and is asserted by the +provenance test: `pm` must be an actual PowerModels model type, and no AC +trigonometric or SOC-WR lifted branch equation is written here. + +Build ORDER matters and is not cosmetic. The nodal balance is closed over the +`ps`/`qs` entries that exist when it is written, so the battery layer's +variables and its injection expressions are installed BEFORE +`constraint_power_balance` runs. Writing them afterwards would silently produce +a network with no battery in it and no error anywhere. +""" +function build_battery_opf(pm::PowerModels.AbstractPowerModel, spec::BatterySpecification) + # ── 1. Stock PowerModels variables ─────────────────────────────────────── + PowerModels.variable_bus_voltage(pm) + PowerModels.variable_gen_power(pm) + PowerModels.variable_branch_power(pm) + PowerModels.variable_dcline_power(pm) + + # ── 2. Battery layer variables and nodal injection carriers ────────────── + _battery_variables!(pm, spec) + + # ── 3. Stock generator/dcline cost, then the battery layer's own costs ─── + PowerModels.objective_min_fuel_and_flow_cost(pm) + _battery_objective!(pm, spec) + + # ── 4. Stock PowerModels constraints ───────────────────────────────────── + PowerModels.constraint_model_voltage(pm) + + for i in PowerModels.ids(pm, :ref_buses) + PowerModels.constraint_theta_ref(pm, i) + end + + for i in PowerModels.ids(pm, :bus) + PowerModels.constraint_power_balance(pm, i) + end + + for i in PowerModels.ids(pm, :branch) + PowerModels.constraint_ohms_yt_from(pm, i) + PowerModels.constraint_ohms_yt_to(pm, i) + PowerModels.constraint_voltage_angle_difference(pm, i) + PowerModels.constraint_thermal_limit_from(pm, i) + PowerModels.constraint_thermal_limit_to(pm, i) + end + + for i in PowerModels.ids(pm, :dcline) + PowerModels.constraint_dcline_power_losses(pm, i) + end + + # ── 5. Battery dynamics and the strict target equality ─────────────────── + _battery_constraints!(pm, spec) + return nothing +end + +""" + _battery_variables!(pm, spec) + +Declare the battery layer's variables and bind the nodal injection carriers. + +# Notes +Declared here, in this order: + +1. `dpd[i]`, `dqd[i]` — FIXED variables carrying the realized demand deviation + at bus `i`. Fixed rather than baked in as constants because the realized + demand is the stage's noise: `SDDP.parameterize` and the strict evaluator + both move them with `JuMP.fix`, which is the only way a formulation whose + balance is written once can see a new demand. +2. `d[i] ≥ 0`, `s[i] ≥ 0` — the two-sided physical active recourse. Neither has + an upper bound of any kind. Capping `d` by the realized demand — natural if + `d` were only curtailed load — would destroy relatively complete recourse for + strict charging targets, so it is deliberately absent. +3. `p_ch[b] ∈ [0, \\overline p^{ch}_b]`, `p_dis[b] ∈ [0, \\overline p^{dis}_b]`, + and the energy state, whose form depends on `spec.mode`. +4. `ps[i]`, `qs[i]` — EXPRESSIONS, registered into `PowerModels.var` so the + stock balance picks them up. +""" +function _battery_variables!(pm::PowerModels.AbstractPowerModel, spec::BatterySpecification) + model = pm.model + case = spec.case + Δt = stage_hours(case) + buses = sort!(collect(PowerModels.ids(pm, :bus))) + + # ── Demand parameterization ────────────────────────────────────────────── + # `pd_nom` and `qd_nom` are already inside the stock balance (PowerModels + # reads them from `ref`), so the deviation carries the realized minus the + # nominal demand. With every multiplier equal to 1 every deviation is 0 and + # the model is exactly ordinary PowerModels OPF on the untouched case. + # + # The realized demand is computed per LOAD from the frozen support and then + # aggregated to the bus, so a per-load or regional multiplier is carried + # exactly; a bus-level factor would already have averaged it away. + pd_nom, qd_nom = nominal_bus_demand(case) + pd_real, qd_real = realized_bus_demand(case, spec.stage, spec.atom) + dpd = JuMP.@variable(model, [i in buses], base_name = "dpd") + dqd = JuMP.@variable(model, [i in buses], base_name = "dqd") + for i in buses + JuMP.fix(dpd[i], pd_real[i] - pd_nom[i]; force = true) + JuMP.fix(dqd[i], qd_real[i] - qd_nom[i]; force = true) + end + + # ── Two-sided uncapped physical active recourse ────────────────────────── + d = JuMP.@variable(model, [i in buses], lower_bound = 0.0, base_name = "d") + s = JuMP.@variable(model, [i in buses], lower_bound = 0.0, base_name = "s") + + # ── Battery controls and state ─────────────────────────────────────────── + batteries = spec.mode === :none ? BatterySpec[] : case.batteries + ids = [b.index for b in batteries] + byid = Dict(b.index => b for b in batteries) + p_ch = JuMP.@variable(model, [k in ids], lower_bound = 0.0, + upper_bound = byid[k].charge_max, base_name = "p_ch") + p_dis = JuMP.@variable(model, [k in ids], lower_bound = 0.0, + upper_bound = byid[k].discharge_max, base_name = "p_dis") + + e_in = Dict{Int,Any}() + e_out = Dict{Int,Any}() + if spec.mode === :sddp + # The caller owns the SDDP.State pair; the battery layer only reads it. + for k in ids + e_in[k] = spec.state_in[k] + e_out[k] = spec.state_out[k] + end + elseif spec.mode === :strict || spec.mode === :free + # Incoming energy is a free variable here and is pinned by an equality in + # `_battery_constraints!` when data was supplied, so that its multiplier + # is available: that multiplier is ∂Q_t/∂e_{t-1}, the second half of the + # actor signal for the PREVIOUS stage's target, and — in `:free` mode — + # the marginal value of the energy a stage inherits. + ein = JuMP.@variable(model, [k in ids], base_name = "e_in") + eout = JuMP.@variable(model, [k in ids], + lower_bound = byid[k].energy_min, + upper_bound = byid[k].energy_max, base_name = "e_out") + for k in ids + e_in[k] = ein[k] + e_out[k] = eout[k] + end + end + + # ── Nodal injection carriers, as expressions ───────────────────────────── + # p^bat_i aggregates every battery at bus i; unity power factor means the + # reactive carrier holds only the demand deviation. + pbat = Dict{Int,Any}(i => JuMP.AffExpr(0.0) for i in buses) + for b in batteries + JuMP.add_to_expression!(pbat[b.bus], 1.0, p_dis[b.index]) + JuMP.add_to_expression!(pbat[b.bus], -1.0, p_ch[b.index]) + end + ps = Dict{Int,Any}() + qs = Dict{Int,Any}() + for i in buses + ps[i] = dpd[i] - pbat[i] - d[i] + s[i] + qs[i] = JuMP.AffExpr(0.0) + dqd[i] + end + PowerModels.var(pm)[:ps] = ps + PowerModels.var(pm)[:qs] = qs + + # Keep everything reachable for constraints, objective and extraction. The + # battery layer's own bookkeeping lives in `pm.ext`, never in `ref`, so it + # cannot collide with a PowerModels key. + pm.ext[:battery] = Dict{Symbol,Any}( + :spec => spec, :buses => buses, :ids => ids, :byid => byid, + :dpd => dpd, :dqd => dqd, :d => d, :s => s, + :p_ch => p_ch, :p_dis => p_dis, :e_in => e_in, :e_out => e_out, + :pbat => pbat, :Δt => Δt, + :pd_nom => pd_nom, :qd_nom => qd_nom, + # The REALIZED demand of this stage/atom, kept so extraction reports what + # the model was actually solved at rather than re-deriving it. + :pd_real => pd_real, :qd_real => qd_real, + ) + return nothing +end + +""" + _battery_objective!(pm, spec) + +Add the battery layer's cost terms to the stock PowerModels objective. + +# Notes +The added term is + +```math +\\sum_b c^{deg}_b \\Delta t\\,(p^{ch}_b + p^{dis}_b) ++ C^{def}\\sum_i d_i + C^{sur}\\sum_i s_i, +``` + +every coefficient taken from the frozen case, so the ACP model, the SOC-WR model +and both SDDP passes charge for the same thing at the same price. The recourse +prices are far above any generator's marginal cost: they guarantee the strict +target is attainable, they do not make attaining it economical. + +**Units.** The generator cost already in the objective is the case's own +physical polynomial ([`network_with_carriers`](@ref) does not touch it), and the +prices added here are the case's own, so the assembled objective IS the physical +stage cost. Nothing in it is rescaled and nothing mixes unit systems, which is +why every quantity `extract_stage_solution` reads off the solved model — cost, +price, multiplier alike — needs no conversion. +""" +function _battery_objective!(pm::PowerModels.AbstractPowerModel, spec::BatterySpecification) + bat = pm.ext[:battery] + case = spec.case + Δt = bat[:Δt] + extra = JuMP.AffExpr(0.0) + for k in bat[:ids] + b = bat[:byid][k] + JuMP.add_to_expression!(extra, b.throughput_cost * Δt, bat[:p_ch][k]) + JuMP.add_to_expression!(extra, b.throughput_cost * Δt, bat[:p_dis][k]) + end + for i in bat[:buses] + JuMP.add_to_expression!(extra, case.recourse.deficit, bat[:d][i]) + JuMP.add_to_expression!(extra, case.recourse.surplus, bat[:s][i]) + end + JuMP.set_objective_function(pm.model, JuMP.objective_function(pm.model) + extra) + return nothing +end + +""" + _battery_constraints!(pm, spec) + +Add the battery state transition and, in `:strict` mode, the target equality. + +# Notes +The transition is + +```math +e_{b} - \\alpha_b e_{b}^{in} + - \\eta^{ch}_b \\Delta t\\, p^{ch}_b + + \\frac{\\Delta t}{\\eta^{dis}_b} p^{dis}_b = 0, +``` + +with ``e_b`` the END-of-stage energy. Neither ``d`` nor ``s`` appears in it: +the recourse is a property of the NETWORK balance, not of the battery, and a +recourse term inside the state equation would be target slack wearing a +physical name. + +In `:strict` mode two further equalities are added, and both are recorded so +their duals can be read: + +- `energy_in[b]`: pins the incoming energy to data. Its dual is + ``\\partial Q_t / \\partial e_{b,t-1}``. +- `target[b]`: pins the outgoing energy to the policy's target. Its dual is + ``\\lambda_{b,t} = \\partial Q_t / \\partial \\hat e_{b,t}``. + +The actor signal for target ``\\hat e_{b,t}`` is the SUM of the `target` dual at +stage ``t`` and the `energy_in` dual at stage ``t+1``, because in strict mode +the emitted target IS the next stage's incoming state. Nothing else couples the +stages: with both ends of every battery pinned, the strict trajectory separates +into `T` independent stage problems. That separation is what lets the JuMP +engine replay an Exa trajectory stage by stage. +""" +function _battery_constraints!(pm::PowerModels.AbstractPowerModel, spec::BatterySpecification) + bat = pm.ext[:battery] + model = pm.model + Δt = bat[:Δt] + + transition = Dict{Int,Any}() + for k in bat[:ids] + b = bat[:byid][k] + transition[k] = JuMP.@constraint(model, + bat[:e_out][k] - b.self_discharge * bat[:e_in][k] + - b.charge_efficiency * Δt * bat[:p_ch][k] + + (Δt / b.discharge_efficiency) * bat[:p_dis][k] == 0.0) + end + bat[:transition] = transition + + if spec.mode === :strict || spec.mode === :free + # The incoming-energy equality is written whenever incoming energy is + # DATA. In `:free` mode with `state_in === nothing` the incoming energy + # stays a free variable that the caller links to the previous stage. + if spec.state_in !== nothing + energy_in_con = Dict{Int,Any}() + for k in bat[:ids] + energy_in_con[k] = JuMP.@constraint(model, + bat[:e_in][k] == Float64(spec.state_in[k])) + end + bat[:energy_in_con] = energy_in_con + end + if spec.mode === :strict + target_con = Dict{Int,Any}() + for k in bat[:ids] + target_con[k] = JuMP.@constraint(model, + bat[:e_out][k] == Float64(spec.target[k])) + end + bat[:target_con] = target_con + end + end + return nothing +end + +# ───────────────────────────────────────────────────────────────────────────── +# Stage model construction and solution +# ───────────────────────────────────────────────────────────────────────────── + +""" + apply_stage_availability!(net, stage) -> Int + +Scale the limits of every scheduled generator of `net` by its own availability at +`stage`, IN PLACE, and return how many generators were scaled. + +# Arguments +- `net::AbstractDict`: a parsed network, about to be handed to PowerModels. +- `stage::Integer`: the stage being built, `1`-based. + +# Notes +`pmin`, `pmax`, `qmin` and `qmax` are all scaled by the same multiplier, so an +availability of `0` takes the unit out of service completely — active AND +reactive — rather than leaving a generator that cannot generate but can still +hold up a voltage for free. The multiplier form generalises past that one case: a +resource whose capacity varies through the day is the same statement with +fractions in it. + +The generator is scaled rather than deleted, and `gen_status` is deliberately not +touched. PowerModels drops out-of-service generators from `ref`, so flipping the +status would change the variable set — and therefore the model's shape — from one +stage to the next. Scaling keeps every stage's model structurally identical, with +`pg` pinned at zero where the unit is unavailable, and a polynomial cost +evaluated at zero contributes exactly zero to the objective. + +Fail-closed on a schedule that does not cover the requested stage: a case that +declares a two-stage schedule and is then solved at stage 3 has been mixed up +with a different case, and silently reusing the last entry would hide that. +""" +function apply_stage_availability!(net::AbstractDict, stage::Integer) + stage >= 1 || throw(ArgumentError("stage must be 1-based, got $stage")) + scaled = 0 + for (id, gen) in net["gen"] + haskey(gen, STAGE_AVAILABILITY_KEY) || continue + sched = gen[STAGE_AVAILABILITY_KEY] + (sched isa AbstractVector && !isempty(sched)) || + error("generator $id: \"$STAGE_AVAILABILITY_KEY\" must be a non-empty vector of multipliers") + stage <= length(sched) || + error("generator $id: \"$STAGE_AVAILABILITY_KEY\" covers $(length(sched)) stages but stage $stage was requested") + a = Float64(sched[stage]) + (isfinite(a) && a >= 0) || + error("generator $id: availability $a at stage $stage is not a finite nonnegative multiplier") + for k in ("pmin", "pmax", "qmin", "qmax") + haskey(gen, k) && (gen[k] = Float64(gen[k]) * a) + end + scaled += 1 + end + return scaled +end + +""" + battery_stage_model(case, model_type; optimizer, kwargs...) + -> PowerModels.AbstractPowerModel + +Instantiate one stage model of the battery study. + +# Arguments +- `case::BatteryCase`: frozen case. +- `model_type::Type`: `PowerModels.ACPPowerModel` (the TRUE model) or + `PowerModels.SOCWRConicPowerModel` (the relaxation SDDP's backward pass uses). + +# Keywords +- `optimizer`: JuMP optimizer factory; attached to the model that is created. +- `jump_model`: an existing JuMP model to build into (SDDP passes its + subproblem here). Defaults to a fresh model. +- everything else is forwarded to [`BatterySpecification`](@ref). + +# Returns +- The instantiated `pm`. `pm.model` is the JuMP model; `pm.ext[:battery]` holds + the battery layer's variable references. + +# Notes +`model_type` is passed straight to `PowerModels.instantiate_model`, so the +network formulation actually instantiated is PowerModels' own. Nothing in this +function branches on it. + +Every stage model in the study is built here — the strict evaluator, the +diagnostics, and each SDDP subproblem — so the per-stage generator availability +applied below is applied once, for all of them. A consumer that assembled a stage +from `network_with_carriers` on its own would silently ignore it, which is why +nothing else in this study does that. + +Dual reporting is requested through PowerModels' own `setting`, which makes it +store the nodal balance CONSTRAINT REFERENCES in `sol(pm, :bus, i)` under +`:lam_kcl_r` and `:lam_kcl_i`. That is the only supported way to reach the +balance rows: the balance is written by a single closed PowerModels function +that returns nothing, so a diagnostic that wants nodal prices either asks +PowerModels to keep the reference or rewrites the balance — and rewriting it is +exactly what this file does not do. +""" +function battery_stage_model(case::BatteryCase, model_type::Type; + optimizer = nothing, + jump_model = nothing, + kwargs...) + spec = BatterySpecification(case; kwargs...) + net = network_with_carriers(case) + apply_stage_availability!(net, spec.stage) + model = jump_model === nothing ? + (optimizer === nothing ? JuMP.Model() : JuMP.Model(optimizer)) : jump_model + if jump_model !== nothing && optimizer !== nothing + JuMP.set_optimizer(model, optimizer) + end + return PowerModels.instantiate_model(net, model_type, + pm -> build_battery_opf(pm, spec); + jump_model = model, + setting = Dict("output" => Dict("duals" => true))) +end + +""" + solve_strict_stage(case, model_type; stage, atom, energy_in, target, + optimizer, silent=true) -> NamedTuple + +Build and solve one STRICT stage problem, returning its full physical solution. + +# Arguments / Keywords +- `stage::Integer`, `atom::Integer`: which demand realization to impose. +- `energy_in::AbstractDict{Int,<:Real}`: incoming energy per battery identifier. +- `target::AbstractDict{Int,<:Real}`: outgoing-energy target per battery. +- `optimizer`: JuMP optimizer factory (Ipopt for ACP). +- `silent::Bool`: suppress solver output. + +# Returns +A `NamedTuple` with the solver status, the objective decomposed into its +physical components, every physical variable keyed by NETWORK identifier, the +strict target multipliers, and the incoming-energy multipliers. + +# Notes +The returned `target_dual` is the multiplier of ``e_b = \\hat e_b`` as JuMP +reports it for a minimization problem, i.e. the sensitivity of the stage +objective to the target. `energy_in_dual` is the corresponding sensitivity to +the incoming state. Together they give the full actor signal — see +[`_battery_constraints!`](@ref). + +The solve is accepted only when the solver returns a status in +`(OPTIMAL, LOCALLY_SOLVED)`; anything else is reported as-is and the caller must +reject it. A solver status alone is still not sufficient, which is why the +physical residuals are computed separately from the returned solution. +""" +function solve_strict_stage(case::BatteryCase, model_type::Type; + stage::Integer, + atom::Integer, + energy_in::AbstractDict, + target::AbstractDict, + optimizer, + silent::Bool = true, + active_batteries::Bool = true) + mode = active_batteries ? :strict : :none + pm = battery_stage_model(case, model_type; + optimizer = optimizer, + stage = stage, atom = atom, mode = mode, + state_in = active_batteries ? Dict(Int(k) => Float64(v) for (k, v) in energy_in) : nothing, + target = active_batteries ? Dict(Int(k) => Float64(v) for (k, v) in target) : nothing) + silent && JuMP.set_silent(pm.model) + JuMP.optimize!(pm.model) + return extract_stage_solution(pm) +end + +""" + stage_cost_decomposition(pm) -> NamedTuple + +The five quantities a stage objective decomposes into, read off a solved model: +`(cost_generation, cost_throughput, deficit, surplus, cost_deficit, +cost_surplus, objective)`. + +# Returns +`deficit` and `surplus` are the PER-BUS raw recourse values keyed by bus id, in +the network's own ordering — not their sums. The sums are returned beside them +as `cost_deficit`/`cost_surplus` for the callers that only need the aggregate, +but the per-bus maps are the load-bearing part: the shared cost contract +projects and validates recourse ELEMENT BY ELEMENT, and an aggregate cannot be +un-summed. A thousand buses each `+1e-7` and a thousand each `−1e-7` cancel to +exactly zero in a total while every one of them is a real, if tiny, violation of +nothing — and one bus genuinely short by `1` pu can hide inside a total that +other buses' surpluses cancel. + +# Notes +The generator polynomial is the case's own, through `_polynomial_cost`, and the +three remaining components are built from the case's own prices, so all four are +in the units of the objective they must sum to. Nothing is rescaled anywhere. + +This function exists so that `extract_stage_solution` and the SDDP simulation +recorders decompose a cost the SAME way. It is the only decomposition in this +repository; a second one would be a second cost convention. +""" +function stage_cost_decomposition(pm::PowerModels.AbstractPowerModel) + model = pm.model + bat = pm.ext[:battery] + case = bat[:spec].case + Δt = bat[:Δt] + buses = bat[:buses] + ids = bat[:ids] + ok = JuMP.termination_status(model) in (MOI.OPTIMAL, MOI.LOCALLY_SOLVED) + + cost_generation = 0.0 + for g in sort!(collect(PowerModels.ids(pm, :gen))) + gen = PowerModels.ref(pm, :gen, g) + cost_generation += _polynomial_cost(gen, JuMP.value(PowerModels.var(pm, :pg, g))) + end + cost_throughput = sum(bat[:byid][k].throughput_cost * Δt * + (JuMP.value(bat[:p_ch][k]) + JuMP.value(bat[:p_dis][k])) + for k in ids; init = 0.0) + d = Dict{Int,Float64}(i => JuMP.value(bat[:d][i]) for i in buses) + s = Dict{Int,Float64}(i => JuMP.value(bat[:s][i]) for i in buses) + return (cost_generation = cost_generation, + cost_throughput = cost_throughput, + deficit = d, surplus = s, + cost_deficit = case.recourse.deficit * sum(values(d); init = 0.0), + cost_surplus = case.recourse.surplus * sum(values(s); init = 0.0), + objective = ok ? JuMP.objective_value(model) : NaN) +end + +""" + extract_stage_solution(pm) -> NamedTuple + +Read every physical quantity of a solved stage model, keyed by network id. + +# Notes +Generator reactive power is reported BOTH per generator and aggregated to the +bus. Several generators may share a bus and the nodal balance constrains only +their sum, so per-generator values live in a null space that two engines are +free to split differently; the nodal aggregate is the physically determined +quantity, and comparing only the per-generator values would report a difference +that the model does not determine. + +**Nothing here is converted, because nothing was rescaled.** The model that was +solved carries the case's physical stage cost, so its objective value, its +generation-cost polynomial, its nodal prices and its state multipliers are all +already in the case's objective units and are read off as they stand. A residual +compared against the solver's own KKT residuals, or a cost compared against a +stock `PowerModels.solve_opf`, therefore compares like with like — there is no +second unit system anywhere in this file for one of them to slip into. +""" +function extract_stage_solution(pm::PowerModels.AbstractPowerModel) + model = pm.model + bat = pm.ext[:battery] + spec = bat[:spec] + case = spec.case + Δt = bat[:Δt] + status = JuMP.termination_status(model) + ok = status in (MOI.OPTIMAL, MOI.LOCALLY_SOLVED) + + val(x) = JuMP.value(x) + buses = bat[:buses] + gen_ids = sort!(collect(PowerModels.ids(pm, :gen))) + branch_ids = sort!(collect(PowerModels.ids(pm, :branch))) + + # Which voltage quantities EXIST is a property of the formulation, and a + # quantity a formulation does not model is reported as absent rather than + # invented. Polar forms carry the magnitude and the angle; the W-space + # relaxation carries |V|² and no angle; the DC approximation carries the + # angle and no magnitude at all. + vm = Dict{Int,Float64}() + va = Dict{Int,Float64}() + vars = PowerModels.var(pm) + if haskey(vars, :vm) + for i in buses + vm[i] = val(PowerModels.var(pm, :vm, i)) + va[i] = val(PowerModels.var(pm, :va, i)) + end + elseif haskey(vars, :w) + for i in buses + vm[i] = sqrt(max(val(PowerModels.var(pm, :w, i)), 0.0)) + va[i] = NaN + end + else + for i in buses + vm[i] = NaN + va[i] = val(PowerModels.var(pm, :va, i)) + end + end + + # Active-power-only formulations declare no reactive variable at all: + # `variable_gen_power_imaginary` and `variable_branch_power_imaginary` are + # no-ops there. The reactive maps come back EMPTY rather than zero-filled, + # because "this formulation does not model it" and "this formulation says it + # is zero" are different statements and only the first one is true. + reactive = haskey(vars, :qg) + pg = Dict{Int,Float64}(g => val(PowerModels.var(pm, :pg, g)) for g in gen_ids) + qg = reactive ? Dict{Int,Float64}(g => val(PowerModels.var(pm, :qg, g)) for g in gen_ids) : + Dict{Int,Float64}() + pg_bus = Dict{Int,Float64}(i => 0.0 for i in buses) + qg_bus = reactive ? Dict{Int,Float64}(i => 0.0 for i in buses) : Dict{Int,Float64}() + for g in gen_ids + b = Int(PowerModels.ref(pm, :gen, g)["gen_bus"]) + pg_bus[b] += pg[g] + reactive && (qg_bus[b] += qg[g]) + end + + p_fr = Dict{Int,Float64}(); q_fr = Dict{Int,Float64}() + p_to = Dict{Int,Float64}(); q_to = Dict{Int,Float64}() + branch_reactive = haskey(vars, :q) + for l in branch_ids + br = PowerModels.ref(pm, :branch, l) + f = (l, Int(br["f_bus"]), Int(br["t_bus"])) + t = (l, Int(br["t_bus"]), Int(br["f_bus"])) + p_fr[l] = val(PowerModels.var(pm, :p, f)) + p_to[l] = val(PowerModels.var(pm, :p, t)) + if branch_reactive + q_fr[l] = val(PowerModels.var(pm, :q, f)) + q_to[l] = val(PowerModels.var(pm, :q, t)) + end + end + + # The per-bus recourse maps and the cost decomposition come from the SAME + # call, so this function and the SDDP recorders cannot drift apart. + dec = stage_cost_decomposition(pm) + d = dec.deficit + s = dec.surplus + pd = Dict{Int,Float64}(i => bat[:pd_real][i] for i in buses) + qd = Dict{Int,Float64}(i => bat[:qd_real][i] for i in buses) + + ids = bat[:ids] + p_ch = Dict{Int,Float64}(k => val(bat[:p_ch][k]) for k in ids) + p_dis = Dict{Int,Float64}(k => val(bat[:p_dis][k]) for k in ids) + p_bat = Dict{Int,Float64}(k => p_dis[k] - p_ch[k] for k in ids) + e_in = Dict{Int,Float64}(k => val(bat[:e_in][k]) for k in ids) + e_out = Dict{Int,Float64}(k => val(bat[:e_out][k]) for k in ids) + + # Multipliers of the physical objective: objective units per pu-hour of + # stored energy, read off with no conversion. + target_dual = Dict{Int,Float64}() + energy_in_dual = Dict{Int,Float64}() + if JuMP.has_duals(model) + if haskey(bat, :target_con) + for k in ids + target_dual[k] = JuMP.dual(bat[:target_con][k]) + end + end + if haskey(bat, :energy_in_con) + for k in ids + energy_in_dual[k] = JuMP.dual(bat[:energy_in_con][k]) + end + end + end + + # ── Nodal prices ───────────────────────────────────────────────────────── + # PowerModels writes the active balance at bus i as + # + # Σ p_arcs − Σ pg + Σ ps + p^d_i + g^s_i |V_i|² = 0, + # + # i.e. `h(x) = −p^d_i`. JuMP's multiplier λ of `h(x) == b` in a minimization + # is ∂obj/∂b, so the price of demand — the quantity anyone means by a nodal + # price — is ∂obj/∂p^d_i = −λ. The sign is derived here once rather than + # guessed, and it is checked in the regression suite against a finite + # difference of the realized demand. + # + # An active-power-only formulation writes no reactive balance, and + # PowerModels records `NaN` under `:lam_kcl_i` there rather than omitting + # the key — so the key's PRESENCE is not evidence that a row exists, and the + # stored value is checked to be an actual constraint reference before a dual + # is asked for. Reading it as if it were one raises far from its cause. + price_active = Dict{Int,Float64}() + price_reactive = Dict{Int,Float64}() + if JuMP.has_duals(model) + for i in buses + bus_sol = PowerModels.sol(pm, :bus, i) + r = get(bus_sol, :lam_kcl_r, nothing) + im = get(bus_sol, :lam_kcl_i, nothing) + r isa JuMP.ConstraintRef && (price_active[i] = -JuMP.dual(r)) + im isa JuMP.ConstraintRef && (price_reactive[i] = -JuMP.dual(im)) + end + end + + # ── Cost decomposition ─────────────────────────────────────────────────── + # Computed by `stage_cost_decomposition`, the ONE place this repository + # decomposes a stage objective. It is factored out because the SDDP + # simulation recorders need exactly these five quantities per stage and must + # not obtain them a second way. + cost_generation = dec.cost_generation + cost_throughput = dec.cost_throughput + cost_deficit = dec.cost_deficit + cost_surplus = dec.cost_surplus + + return ( + status = status, solved = ok, + objective = ok ? JuMP.objective_value(model) : NaN, + cost_generation = cost_generation, + cost_throughput = cost_throughput, + cost_deficit = cost_deficit, + cost_surplus = cost_surplus, + cost_stage = cost_generation + cost_throughput + cost_deficit + cost_surplus, + vm = vm, va = va, pg = pg, qg = qg, pg_bus = pg_bus, qg_bus = qg_bus, + p_fr = p_fr, q_fr = q_fr, p_to = p_to, q_to = q_to, + deficit = d, surplus = s, pd = pd, qd = qd, + p_ch = p_ch, p_dis = p_dis, p_bat = p_bat, + energy_in = e_in, energy_out = e_out, + target_dual = target_dual, energy_in_dual = energy_in_dual, + price_active = price_active, price_reactive = price_reactive, + simultaneous = Dict{Int,Float64}(k => min(p_ch[k], p_dis[k]) for k in ids), + pm = pm, + ) +end + +""" + binding_constraints(pm, sol; tol=1e-6) -> Vector{NamedTuple} + +Every physical limit of a solved stage that is binding or nearly binding. + +# Arguments +- `pm`: the solved model (used for its network reference). +- `sol`: the named tuple returned by [`extract_stage_solution`](@ref). + +# Keywords +- `tol::Real`: absolute slack, in the natural unit of each limit, below which a + limit counts as binding. + +# Returns +A vector of `(kind, index, side, value, limit, slack)` rows, sorted by slack, for + +- generator active and reactive bounds (pu); +- bus voltage magnitude bounds (pu); +- branch apparent-power limits at BOTH ends (pu, compared on ``|S|`` rather than + on ``|S|^2`` so the tolerance means the same thing on every branch). Where the + formulation carries no reactive flow the reactive term is absent and ``|S|`` + reduces to ``|p|``, which is exactly the quantity that formulation's own + thermal limit constrains; +- branch angle-difference limits (rad), where the formulation has angles. + +A voltage-magnitude row is skipped where the formulation has no magnitude: `vm` +comes back `NaN` there, and a `NaN` slack is never within tolerance, so the row +drops out on its own rather than through a special case. + +# Notes +This answers "what stopped the network from delivering the energy" — the question +that separates a real relaxation-driven storage-value difference from a numerical +one. It reads the reported solution rather than the solver's active set, so it +says the same thing about a solution produced by either engine. +""" +function binding_constraints(pm::PowerModels.AbstractPowerModel, sol; tol::Real = 1e-6) + rows = NamedTuple[] + push_row!(kind, index, side, value, limit) = begin + slack = abs(limit - value) + slack <= tol && push!(rows, (kind = kind, index = index, side = side, + value = float(value), limit = float(limit), + slack = float(slack))) + end + + for (g, val) in sol.pg + gen = PowerModels.ref(pm, :gen, g) + push_row!("pg", g, "min", val, Float64(gen["pmin"])) + push_row!("pg", g, "max", val, Float64(gen["pmax"])) + end + for (g, val) in sol.qg + gen = PowerModels.ref(pm, :gen, g) + push_row!("qg", g, "min", val, Float64(gen["qmin"])) + push_row!("qg", g, "max", val, Float64(gen["qmax"])) + end + for (i, val) in sol.vm + bus = PowerModels.ref(pm, :bus, i) + push_row!("vm", i, "min", val, Float64(bus["vmin"])) + push_row!("vm", i, "max", val, Float64(bus["vmax"])) + end + for l in keys(sol.p_fr) + br = PowerModels.ref(pm, :branch, l) + rate = Float64(get(br, "rate_a", Inf)) + isfinite(rate) || continue + push_row!("thermal", l, "from", hypot(sol.p_fr[l], get(sol.q_fr, l, 0.0)), rate) + push_row!("thermal", l, "to", hypot(sol.p_to[l], get(sol.q_to, l, 0.0)), rate) + if !isnan(sol.va[Int(br["f_bus"])]) + θ = sol.va[Int(br["f_bus"])] - sol.va[Int(br["t_bus"])] + push_row!("angle", l, "min", θ, Float64(get(br, "angmin", -pi))) + push_row!("angle", l, "max", θ, Float64(get(br, "angmax", pi))) + end + end + sort!(rows; by = r -> r.slack) + return rows +end + +""" + _polynomial_cost(gen, pg) -> Float64 + +Evaluate a PowerModels polynomial cost model at `pg`. + +# Notes +PowerModels stores the polynomial highest-order-first with `ncost` coefficients, +so the value is ``\\sum_{j} c_j\\, pg^{ncost-j}``. Only `model == 2` (polynomial) +is supported; a piecewise-linear cost model would need its own evaluation and +none of the benchmarks used here carries one. + +The result is in the units of the coefficients it was handed. Every model this +file builds carries the case's own coefficients, so calling it on `case.network` +and calling it on a model's `ref` return the same number for the same dispatch. +""" +function _polynomial_cost(gen::AbstractDict, pg::Real) + Int(get(gen, "model", 2)) == 2 || + error("only polynomial (model 2) generator costs are supported") + cost = Float64.(gen["cost"]) + n = Int(gen["ncost"]) + total = 0.0 + for (j, c) in enumerate(cost[(end - n + 1):end]) + total += c * pg^(n - j) + end + return total +end + +""" + record_stage_solution!(rec, sol, case; scenario, stage) + +Write one solved stage into a [`SolutionRecorder`](@ref) in the shared schema. +""" +function record_stage_solution!(rec::SolutionRecorder, sol, case::BatteryCase; + scenario::Integer, stage::Integer) + record_map!(rec, scenario, stage, "vm", sol.vm) + record_map!(rec, scenario, stage, "va", sol.va) + record_map!(rec, scenario, stage, "pg", sol.pg) + record_map!(rec, scenario, stage, "qg", sol.qg) + record_map!(rec, scenario, stage, "pg_bus", sol.pg_bus) + record_map!(rec, scenario, stage, "qg_bus", sol.qg_bus) + record_map!(rec, scenario, stage, "p_fr", sol.p_fr) + record_map!(rec, scenario, stage, "q_fr", sol.q_fr) + record_map!(rec, scenario, stage, "p_to", sol.p_to) + record_map!(rec, scenario, stage, "q_to", sol.q_to) + record_map!(rec, scenario, stage, "deficit", sol.deficit) + record_map!(rec, scenario, stage, "surplus", sol.surplus) + record_map!(rec, scenario, stage, "pd", sol.pd) + record_map!(rec, scenario, stage, "qd", sol.qd) + record_map!(rec, scenario, stage, "p_ch", sol.p_ch) + record_map!(rec, scenario, stage, "p_dis", sol.p_dis) + record_map!(rec, scenario, stage, "p_bat", sol.p_bat) + record_map!(rec, scenario, stage, "energy_in", sol.energy_in) + record_map!(rec, scenario, stage, "energy_out", sol.energy_out) + isempty(sol.target_dual) || record_map!(rec, scenario, stage, "target_dual", sol.target_dual) + isempty(sol.price_active) || record_map!(rec, scenario, stage, "price_active", sol.price_active) + isempty(sol.price_reactive) || record_map!(rec, scenario, stage, "price_reactive", sol.price_reactive) + record!(rec, scenario, stage, "cost_generation", 0, sol.cost_generation) + record!(rec, scenario, stage, "cost_throughput", 0, sol.cost_throughput) + record!(rec, scenario, stage, "cost_deficit", 0, sol.cost_deficit) + record!(rec, scenario, stage, "cost_surplus", 0, sol.cost_surplus) + record!(rec, scenario, stage, "cost_stage", 0, sol.cost_stage) + record!(rec, scenario, stage, "objective", 0, sol.objective) + record!(rec, scenario, stage, "solved", 0, sol.solved ? 1.0 : 0.0) + return rec +end + +# ───────────────────────────────────────────────────────────────────────────── +# Provenance +# ───────────────────────────────────────────────────────────────────────────── + +""" + assert_powermodels_provenance(pm, expected::Type) -> Nothing + +Assert, from the LIVE object, that the network formulation actually built is the +intended PowerModels one. + +# Notes +This is the runtime half of the provenance gate. Numerical agreement with a +handwritten model is explicitly NOT an acceptable substitute: the claim the +study makes is that SDDP's backward pass is an actual +`PowerModels.SOCWRConicPowerModel` or an actual `PowerModels.DCPPowerModel`, and +its forward pass an actual `PowerModels.ACPPowerModel`, and only the type of the +instantiated object can establish that. + +Beyond the type, the assertion checks that the model carries the variables the +intended formulation is DEFINED by, so that a type that had been made to alias +something else would still be caught: + +| formulation | must carry | must NOT carry | +|---|---|---| +| `ACPPowerModel` | `vm`, `va` | — | +| `SOCWRConicPowerModel` | `w`, `wr`, `wi` | — | +| `DCPPowerModel` | `va`, `p` | `vm`, `w`, `q` | + +The DC row is stated in both directions on purpose. What distinguishes the DC +approximation is as much what it LACKS — a voltage magnitude and any reactive +quantity — as what it has, and a positive-only check would pass on a model that +had quietly acquired them. +""" +function assert_powermodels_provenance(pm::PowerModels.AbstractPowerModel, expected::Type) + typeof(pm) === expected || + error("expected an actual $expected, got $(typeof(pm))") + pm isa PowerModels.AbstractPowerModel || + error("$(typeof(pm)) is not a PowerModels.AbstractPowerModel") + vars = keys(PowerModels.var(pm)) + if expected === PowerModels.ACPPowerModel + (:vm in vars && :va in vars) || + error("ACPPowerModel is missing its polar voltage variables") + elseif expected === PowerModels.SOCWRConicPowerModel + (:w in vars && :wr in vars && :wi in vars) || + error("SOCWRConicPowerModel is missing its lifted voltage variables") + elseif expected === PowerModels.DCPPowerModel + (:va in vars && :p in vars) || + error("DCPPowerModel is missing its angle or active-flow variables") + (:vm in vars || :w in vars) && + error("DCPPowerModel carries a voltage-magnitude variable; it is not the DC approximation") + (:q in vars || :qg in vars) && + error("DCPPowerModel carries a reactive variable; it is not the DC approximation") + end + return nothing +end diff --git a/examples/BatteryStorageOPF/battery_sddp.jl b/examples/BatteryStorageOPF/battery_sddp.jl new file mode 100644 index 0000000..61e70fc --- /dev/null +++ b/examples/BatteryStorageOPF/battery_sddp.jl @@ -0,0 +1,1019 @@ +# battery_sddp.jl +# +# The SDDP baseline: stock SDDP.jl over the shared PowerModels battery problem +# specification, with a SELECTABLE convex backward pass and a true-ACP forward +# pass. +# +# WHAT IS STOCK AND WHY IT MATTERS. +# SDDP.jl owns the policy graph, the state variables, the sampling scheme, cut +# generation, training and simulation. This file constructs two policy graphs +# from the SAME `build_battery_opf` specification — one instantiated as an +# actual convex PowerModels formulation, one as an actual +# `PowerModels.ACPPowerModel` — and hands them to SDDP's own +# `AlternativeForwardPass`/`AlternativePostIterationCallback` pair, which is +# SDDP.jl's documented mechanism for training against a convex approximation +# while operating a nonconvex model. No `duality_handler` is overridden, no cut +# is filtered, retried or reweighted: which duals become cuts is SDDP's decision. +# +# TWO BACKWARD FORMULATIONS, AND WHAT THEIR SCALARS MEAN. +# `:soc` builds the backward nodes as `PowerModels.SOCWRConicPowerModel` and +# `:dc` as `PowerModels.DCPPowerModel`. Both are PowerModels' own formulations: +# nothing in this study writes a DC network equation, exactly as nothing in it +# writes an AC one. The generator data and the generator cost polynomials are +# the PGLib case's own in both, unrepriced and unlinearized, and the battery +# layer added on top is the same one — state, transition, charge/discharge, +# unity-power-factor injection, demand parameterization, uncapped physical +# recourse — in both. +# +# The SCALARS they produce are NOT the same kind of object, and this file never +# lets them be printed as if they were: +# +# * the SOC-WR arm's scalar is a lower bound on the SOC-WR RELAXATION over the +# horizon it was trained on. The relaxation lower-bounds the true ACP +# problem, so the scalar does too; +# * the DC arm's scalar is the DC-APPROXIMATION TRAINING BOUND. The DC +# approximation is neither a relaxation nor a restriction of the nonconvex +# ACP problem — it drops the reactive balance and fixes voltage magnitudes — +# so its value bounds NOTHING about ACP, in either direction. It is the +# internal convergence scalar of the approximation the cuts came from, and +# it is reported to say whether that training converged. +# +# Neither is comparable to a forward objective over a different horizon, and +# nothing in this file invites that comparison. +# +# Usage +# julia --project=. battery_sddp.jl # short construction smoke +# DR_BAT_SDDP_BACKWARD=dc DR_BAT_SDDP_STAGES=3 julia --project=. battery_sddp.jl + +using SDDP +using JuMP +using JSON +using PowerModels +using Ipopt +using Clarabel +using Printf +using Random +using Statistics + +@isdefined(BatterySpecification) || include(joinpath(@__DIR__, "battery_powermodels.jl")) +@isdefined(perfect_foresight_panel) || include(joinpath(@__DIR__, "battery_diagnostics.jl")) + +"The true model SDDP's forward pass operates. Not selectable: the study compares methods, not physics." +const FORWARD_FORMULATION = PowerModels.ACPPowerModel + +""" + BACKWARD_SPECS + +The selectable backward formulations, keyed by the identifier that names them +everywhere — in a keyword, in a result field, in a printed line and in a cut +file's name. + +# Fields of each row +- `formulation::Type`: the PowerModels model type the backward nodes really are. +- `optimizer`: the solver factory that matches it. +- `bounds_acp::Bool`: whether the scalar SDDP converges to is a valid lower + bound on the true ACP problem. TRUE for the SOC-WR relaxation, FALSE for the + DC approximation. +- `bound_name::String`: what that scalar may be CALLED. Every place this file + prints or labels it reads this field, so the DC arm cannot acquire the word + "bound" on its own anywhere. +- `tag::String`: the token a serialized artifact must carry. +""" +const BACKWARD_SPECS = Dict{Symbol,NamedTuple}( + :soc => (formulation = PowerModels.SOCWRConicPowerModel, + optimizer = socwr_optimizer, + bounds_acp = true, + bound_name = "SOC-WR relaxation bound", + tag = "socwr"), + :dc => (formulation = PowerModels.DCPPowerModel, + optimizer = dc_optimizer, + bounds_acp = false, + bound_name = "DC-approximation training bound", + tag = "dc"), +) + +""" + backward_spec(backward::Symbol) -> NamedTuple + +The [`BACKWARD_SPECS`](@ref) row for `backward`, or an error naming the ones +that exist. +""" +function backward_spec(backward::Symbol) + haskey(BACKWARD_SPECS, backward) || throw(ArgumentError( + "backward formulation must be one of $(sort!(collect(keys(BACKWARD_SPECS)))), got :$backward")) + return BACKWARD_SPECS[backward] +end + +""" + sddp_cut_path(dir, case, backward) -> String + +The path a cut file for this case and this backward formulation is written to. + +# Notes +The formulation's tag is in the FILE NAME, not only in a sibling record. A cut +set is the one artifact of a run that outlives the session that made it, and a +directory holding `case118.cuts.json` twice — once from SOC and once from DC — +is a directory in which the two can be confused by nothing worse than a +`readdir`. [`train_battery_sddp`](@ref) refuses a `cut_path` whose name does not +carry the tag as a dot-delimited component, so the convention is enforced rather +than merely offered. +""" +sddp_cut_path(dir::AbstractString, case::BatteryCase, backward::Symbol) = + joinpath(dir, "$(case.name).$(backward_spec(backward).tag).cuts.json") + +""" + sddp_path_cost(case, path, T) -> Float64 + +The cost of one simulated path: its `T` stage objectives summed. + +# Notes +Both policy graphs are built by the shared specification, so every stage +objective SDDP sees — and therefore every cut, every `stage_objective` a +simulation records and the bound `SDDP.calculate_bound` returns — is the physical +stage cost of the case, in the objective units of the underlying PGLib network. +Stage costs, cuts and bounds live in that one unit system on both graphs, so a +path cost is a plain sum and a number leaving this file needs no conversion. + +`case` is taken for the signature every consumer already calls and to keep the +path bound to the case it was simulated on; the sum itself does not need it. +""" +sddp_path_cost(case::BatteryCase, path, T::Integer) = + sum(path[t][:stage_objective] for t in 1:Int(T)) + +""" + battery_recorders(ids) -> Dict{Symbol,Function} + +The `custom_recorders` every simulation in this file uses. + +# What is recorded, and why each one +| key | value | +|---|---| +| `:energy_out`, `:energy_in` | the battery state, in `ids` order | +| `:deficit`, `:surplus` | the recourse SUMMED over buses, kept for the existing convergence log | +| `:deficit_by_bus`, `:surplus_by_bus` | the recourse PER BUS, raw and unprojected | +| `:bus_ids` | the bus ordering the two per-bus vectors use | +| `:cost_generation`, `:cost_throughput` | the stage cost decomposition | +| `:p_ch`, `:p_dis` | throughput | + +# Notes +**The per-bus vectors are what make an element-wise cost contract possible.** +The shared `physical_stage_cost` projects and validates recourse element by +element; a recorder that returned only the totals would leave a simulation +record from which the contract cannot be evaluated at all, and the aggregate is +not a conservative stand-in for it — positive and negative tolerance-scale +residues at different buses CANCEL in a sum, so a total of exactly zero is +consistent with every bus being off, and one bus off by a lot can hide behind +other buses' surpluses. `sddp_physical_path_cost` is the consumer. + +Everything is read off the live subproblem after it is solved, through +`stage_cost_decomposition` — the same decomposition `extract_stage_solution` +uses — so what is stored is the value the solver actually returned, decomposed +once, in one place. + +The per-bus values are stored RAW. Projection belongs to the cost contract and +happens there; a recorder that pre-projected would destroy the evidence the +contract has to judge. +""" +function battery_recorders(ids::AbstractVector{Int}) + dec(sp::JuMP.Model) = stage_cost_decomposition(sp.ext[:pm]) + order(sp::JuMP.Model) = sp.ext[:pm].ext[:battery][:buses] + return Dict{Symbol,Function}( + :energy_out => (sp::JuMP.Model) -> [JuMP.value(sp[:e][k].out) for k in ids], + :energy_in => (sp::JuMP.Model) -> [JuMP.value(sp[:e][k].in) for k in ids], + :deficit => (sp::JuMP.Model) -> max(0.0, sum(JuMP.value.(sp.ext[:battery][:d]))), + :surplus => (sp::JuMP.Model) -> max(0.0, sum(JuMP.value.(sp.ext[:battery][:s]))), + :bus_ids => (sp::JuMP.Model) -> collect(Int, order(sp)), + :deficit_by_bus => (sp::JuMP.Model) -> (D = dec(sp).deficit; [D[i] for i in order(sp)]), + :surplus_by_bus => (sp::JuMP.Model) -> (S = dec(sp).surplus; [S[i] for i in order(sp)]), + :cost_generation => (sp::JuMP.Model) -> dec(sp).cost_generation, + :cost_throughput => (sp::JuMP.Model) -> dec(sp).cost_throughput, + :p_ch => (sp::JuMP.Model) -> sum(JuMP.value(sp.ext[:battery][:p_ch][k]) for k in ids; init = 0.0), + :p_dis => (sp::JuMP.Model) -> sum(JuMP.value(sp.ext[:battery][:p_dis][k]) for k in ids; init = 0.0), + ) +end + +""" + sddp_physical_path_cost(case, path, T; tol=PHYSICAL_RECOURSE_TOL) + -> (cost, worst_recourse, admissible) + +The CORRECTED physical cost of one simulated path, under the shared cost +contract. + +``C = \\sum_{t=1}^{T} \\mathrm{physical\\_stage\\_cost}(\\mathrm{stage}_t)_{\\mathrm{corrected}}`` + +# Returns +- `cost`: the sum of the per-stage CORRECTED costs — generation, throughput and + the recourse actually charged after element-wise projection. +- `worst_recourse`: the largest absolute RAW per-bus recourse over every bus of + every stage, before projection. +- `admissible`: whether every individual element of every stage was inside + `tol`. + +# Notes +Each stage goes through `physical_stage_cost`, the byte-identical contract both +engines carry, fed the PER-BUS recourse the recorders stored. Only after every +element has been validated and projected is anything summed — validate, project, +then sum, in that order. Summing first and testing the total would accept a +stage whose bus-level violations happen to cancel, which is the failure this +ordering exists to exclude. + +`sddp_path_cost` remains the RAW sum of the solver's stage objectives and is +still what the training log and the convergence history report. This is the +quantity a policy is SELECTED and REPORTED on, exactly as on the Exa side. +""" +function sddp_physical_path_cost(case::BatteryCase, path, T::Integer; + tol::Real = PHYSICAL_RECOURSE_TOL) + total = 0.0 + worst = 0.0 + admissible = true + for t in 1:Int(T) + st = path[t] + bus = Int.(st[:bus_ids]) + c = physical_stage_cost( + (cost_generation = Float64(st[:cost_generation]), + cost_throughput = Float64(st[:cost_throughput]), + deficit = Dict(bus[i] => Float64(st[:deficit_by_bus][i]) for i in eachindex(bus)), + surplus = Dict(bus[i] => Float64(st[:surplus_by_bus][i]) for i in eachindex(bus)), + objective = Float64(st[:stage_objective])), case.recourse; tol = tol) + total += c.corrected + worst = max(worst, c.worst_recourse) + admissible &= c.admissible + end + return (cost = total, worst_recourse = worst, admissible = admissible) +end + +""" + battery_policy_graph(case, model_type; num_stages, optimizer) -> SDDP.PolicyGraph + +Build a `num_stages`-stage linear policy graph whose stage problem is the shared +battery problem specification instantiated as `model_type`. + +# Arguments +- `case::BatteryCase`: frozen case. +- `model_type::Type`: `SOCWRConicPowerModel` or `ACPPowerModel`. + +# Keywords +- `num_stages::Integer`: horizon. +- `optimizer`: solver factory matching `model_type`. + +# Returns +- The `SDDP.PolicyGraph`. Each node's JuMP model additionally carries + `sp.ext[:battery]` (the battery layer's variable references) and + `sp.ext[:pm]` (the live PowerModels object, so provenance can be asserted + from the graph itself). + +# Notes +Battery energy is the genuine SDDP state: `@variable(sp, ..., SDDP.State)` +creates the incoming/outgoing pair, SDDP equates one stage's outgoing value to +the next stage's incoming value, and the battery transition — written by the +shared specification, not here — is what ties that state to the dispatch. + +Demand is the node's NOISE. `SDDP.parameterize` moves the fixed demand-deviation +variables the specification installed; the atoms and their probabilities come +from the frozen case, so the forward graph, the backward graph and the Exa +engine all face the same stochastic program. + +Information timing is the plan's: the incoming energy and the realized demand +are both known when the stage decision is taken, and the outgoing energy is the +decision that becomes the next stage's state. +""" +function battery_policy_graph(case::BatteryCase, model_type::Type; + num_stages::Integer, + optimizer) + ids = [b.index for b in case.batteries] + byid = Dict(b.index => b for b in case.batteries) + num_stages <= horizon(case.demand) || + throw(ArgumentError("policy graph asks for $num_stages stages but the frozen support covers $(horizon(case.demand))")) + + graph = SDDP.LinearPolicyGraph( + stages = num_stages, + sense = :Min, + # Every stage cost is a sum of nonnegative terms (generation cost with + # nonnegative coefficients, throughput cost, priced recourse), so 0 is a + # valid and honest lower bound on the value function. + lower_bound = 0.0, + optimizer = optimizer, + ) do sp, t + JuMP.@variable(sp, e[k in ids], SDDP.State, + initial_value = byid[k].energy_initial, + lower_bound = byid[k].energy_min, + upper_bound = byid[k].energy_max) + state_in = Dict{Int,Any}(k => e[k].in for k in ids) + state_out = Dict{Int,Any}(k => e[k].out for k in ids) + + pm = battery_stage_model(case, model_type; + jump_model = sp, + stage = t, atom = 1, mode = :sddp, + state_in = state_in, state_out = state_out) + sp.ext[:pm] = pm + sp.ext[:battery] = pm.ext[:battery] + + # The stage cost is exactly the objective the shared specification + # assembled: stock PowerModels generation cost plus the battery layer's + # throughput and recourse prices. + SDDP.@stageobjective(sp, JuMP.objective_function(sp)) + + bat = pm.ext[:battery] + pd_nom, qd_nom = bat[:pd_nom], bat[:qd_nom] + buses = bat[:buses] + # The stage's atoms come from the FROZEN support, per stage: the support + # is stage-dependent in general, so a graph-wide atom list would be + # wrong on any case whose late stages carry a different support. The + # realized per-bus demand of each atom is materialized ONCE here, so a + # parameterize call is a few `fix`es rather than a re-aggregation of the + # case's loads on every node solve of every iteration. + atoms = collect(1:num_atoms(case.demand, t)) + probs = copy(atom_probabilities(case.demand, t)) + realized = [realized_bus_demand(case, t, k) for k in atoms] + SDDP.parameterize(sp, atoms, probs) do ω + pd, qd = realized[ω] + for i in buses + JuMP.fix(bat[:dpd][i], pd[i] - pd_nom[i]; force = true) + JuMP.fix(bat[:dqd][i], qd[i] - qd_nom[i]; force = true) + end + return + end + end + return graph +end + +""" + assert_graph_provenance(graph, expected::Type) + +Assert that every node of `graph` was instantiated as an actual `expected` +PowerModels model. + +# Notes +Checking the type on one node would not do: a policy graph builds one model per +stage and a formulation switch that only fired on some stages would be a +different stochastic program with the same name. +""" +function assert_graph_provenance(graph::SDDP.PolicyGraph, expected::Type) + for (key, node) in graph.nodes + haskey(node.subproblem.ext, :pm) || + error("node $key carries no PowerModels object") + assert_powermodels_provenance(node.subproblem.ext[:pm], expected) + end + return nothing +end + +""" + canonicalize_cut_file!(path, case; coef_tol=1e-9, weaken_tol=1e-6) -> NamedTuple + +Rewrite an SDDP cut file so no cut carries a denormal state coefficient, without +letting any cut rise anywhere in the state box. + +# The transformation +A cut is `theta >= alpha + sum_i beta_i x_i` over storage states with known +bounds `l_i <= x_i <= u_i`. Let `R = {i : |beta_i| < coef_tol}` and + + W = sum_{i in R} |beta_i| (u_i - l_i). + +`R` is applied only when `W <= weaken_tol`. Then `beta_i := 0` for `i` in `R` and + + alpha' = alpha + min_{x in [l,u]} sum_{i in R} beta_i x_i + = alpha + sum_{i in R} (beta_i > 0 ? beta_i l_i : beta_i u_i). + +# Why this is valid +For every `x` in the box, +`RHS(x) - RHS'(x) = sum_R beta_i x_i - min_y sum_R beta_i y_i >= 0`, so the +canonicalized cut is NEVER above the original: it can only weaken, never cut off +a point the original admitted, and therefore remains a valid lower bound on the +value function. The weakening is bounded above by `W` and recorded per cut. + +# Why it is needed +Accumulated cuts introduce coefficients of order 1e-14 into an otherwise +ordinary LP — measured on `case588_sdet` node 16 at iteration 74: matrix range +ratio 2.64e17, which HiGHS rejects at load with zero simplex iterations. The DC +MODEL, the recourse prices, the solver and its settings are untouched; only the +REPRESENTATION of the cuts changes. + +A cut whose removal set would exceed `weaken_tol` is left exactly as it was and +counted in `refused`, so the allowance can never be silently exceeded. +""" +function canonicalize_cut_file!(path::AbstractString, case::BatteryCase; + coef_tol::Real = 1e-9, weaken_tol::Real = 1e-6) + isfile(path) || return (changed = 0, removed = 0, refused = 0, max_weakening = 0.0) + raw = JSON.parsefile(path) + # Two shapes carry cuts here. `SDDP.write_cuts_to_file` emits a BARE LIST of + # node objects; this study's segment checkpoint wraps that same list under + # `"cuts"` alongside its bound and provenance. Both are canonicalized in + # place, and the wrapper is preserved untouched. + data = raw isa AbstractVector ? raw : + (raw isa AbstractDict && haskey(raw, "cuts") ? raw["cuts"] : + error("unrecognised cut file layout: $path")) + lo = Dict{String,Float64}(); hi = Dict{String,Float64}() + for b in case.batteries, key in (string("e[", b.index, "]"), string(b.index)) + lo[key] = Float64(b.energy_min); hi[key] = Float64(b.energy_max) + end + changed = 0; removed = 0; refused = 0; maxW = 0.0 + for node in data, key in ("single_cuts", "multi_cuts") + haskey(node, key) || continue + for cut in node[key] + haskey(cut, "coefficients") || continue + co = cut["coefficients"] + R = String[]; W = 0.0 + for (k, v) in co + b = Float64(v) + (b != 0 && abs(b) < coef_tol) || continue + W += abs(b) * (get(hi, k, 0.0) - get(lo, k, 0.0)); push!(R, k) + end + isempty(R) && continue + if W > weaken_tol + refused += 1 + continue + end + shift = 0.0 + for k in R + b = Float64(co[k]) + shift += b > 0 ? b * get(lo, k, 0.0) : b * get(hi, k, 0.0) + co[k] = 0.0 + end + cut["intercept"] = Float64(cut["intercept"]) + shift + changed += 1; removed += length(R); maxW = max(maxW, W) + end + end + open(path, "w") do io; JSON.print(io, raw); end + return (changed = changed, removed = removed, refused = refused, max_weakening = maxW) +end + +""" + train_battery_sddp(case; backward=:soc, num_stages, iteration_limit, + time_limit, seed, print_level, backward_optimizer, + forward_optimizer, protocol, evaluate_columns, + evaluate_every, cut_path, resume_cuts) + -> NamedTuple + +Train the SDDP policy: convex backward, true-ACP forward. + +# Keywords +- `backward::Symbol`: `:soc` (default) or `:dc`; see [`BACKWARD_SPECS`](@ref). + It selects the PowerModels formulation the backward nodes are instantiated as, + and nothing else about the stochastic program. +- `num_stages::Integer`: horizon. +- `iteration_limit`, `time_limit`: the training budget. Either may be `nothing`. +- `seed::Integer`: seed of the forward sampling scheme. +- `backward_optimizer`: solver factory, or `nothing` for the backward + formulation's own. `forward_optimizer`: solver factory for the ACP graph. + These are SOLVER settings; nothing here may change the mathematical model. +- `protocol`, `evaluate_columns`, `evaluate_every`: when all three are given, + the TRUE-ACP forward cost is evaluated on those fixed protocol columns every + `evaluate_every` iterations and recorded in `history`. +- `cut_path`: when given, the cuts are written there after training. It must + carry the backward formulation's tag — see [`sddp_cut_path`](@ref). +- `resume_cuts`: when given, the cut file a previous training stage wrote, read + into the freshly built graphs BEFORE training so this call CONTINUES that + policy instead of starting a new one. It must carry the same tag, so a SOC cut + set can never be read into a DC graph. + +# Returns +A `NamedTuple` with the two graphs, the termination status, the backward +formulation's scalar, the elapsed wall time, and `history` — one row per +evaluation with the iteration, the scalar at that point, the true-ACP mean cost +on the fixed columns, the worst recourse and the elapsed time. + +Four fields identify the arm and travel with every downstream consumer: +`backward` (the identifier), `backward_formulation` (the actual PowerModels +type), `bound_name` (what `bound` may be called) and `bound_bounds_acp` (whether +`bound` is a valid lower bound on the true ACP problem — TRUE only for `:soc`). + +# Notes +`SDDP.AlternativeForwardPass(forward)` performs each forward pass on the ACP +graph and copies the resulting states back into the convex graph, and +`AlternativePostIterationCallback(forward)` copies the newly created cuts into +the ACP graph. Both are SDDP.jl's own; the pair is what makes "value with a +convex approximation, operate the true model" a stock configuration rather than +an intervention, and it is SHARED by the two arms rather than reimplemented for +the second one. The periodic evaluation is composed AROUND that callback, never +in place of it. + +**Three signals, never compared in level.** The backward scalar is a property of +the formulation the cuts came from over `num_stages` stages — a bound on the +relaxation for `:soc`, an internal convergence scalar for `:dc`. The true-ACP +forward cost on the fixed columns is a physical cost of the current policy on a +small, fixed sample. A final protocol cost is a third thing again. + +The evaluation is a MEASUREMENT, not a stopping rule: training runs its declared +budget. Choosing when to stop by watching the evaluation would select a policy on +the same numbers used to report it. + +CONTINUATION. `resume_cuts` exists so a long run can survive preemption: the +graphs are REBUILT from the frozen case and SDDP's own `read_cuts_from_file` +restores the policy into them. No solver object is serialized and no SDDP +internal is parsed. The file is read into BOTH graphs, because +`AlternativePostIterationCallback` is what normally puts each iteration's cuts +into the ACP graph, and a forward graph resumed without them would take its +decisions against an empty cost-to-go while the backward graph believed +otherwise. The two graphs carry the same state variables, so the same file is the +right file for both. +""" +function train_battery_sddp(case::BatteryCase; + backward::Symbol = :soc, + num_stages::Integer = 3, + iteration_limit = 10, + time_limit = nothing, + seed::Integer = 20260804, + print_level::Integer = 0, + backward_optimizer = nothing, + forward_optimizer = acp_optimizer(), + protocol = nothing, + evaluate_columns = nothing, + evaluate_every = nothing, + cut_path = nothing, + resume_cuts = nothing, + canonicalize_cuts::Bool = false) + spec = backward_spec(backward) + # The tag must be a dot-delimited COMPONENT of the file name, not merely a + # substring of it: a PGLib case name that happened to contain "dc" would + # otherwise let a DC run write over a path built for the SOC arm. + for (label, p) in (("cut_path", cut_path), ("resume_cuts", resume_cuts)) + p === nothing || spec.tag in split(basename(String(p)), '.') || + throw(ArgumentError( + "$label file $(basename(String(p))) does not carry the \"$(spec.tag)\" tag of the " * + ":$backward backward formulation; use sddp_cut_path to build one")) + end + backward_optimizer === nothing && (backward_optimizer = spec.optimizer()) + Random.seed!(seed) + backward_graph = battery_policy_graph(case, spec.formulation; + num_stages = num_stages, optimizer = backward_optimizer) + forward = battery_policy_graph(case, FORWARD_FORMULATION; + num_stages = num_stages, optimizer = forward_optimizer) + assert_graph_provenance(backward_graph, spec.formulation) + assert_graph_provenance(forward, FORWARD_FORMULATION) + + # CONTINUATION, before the first iteration: SDDP's own reader restores the + # cuts into both graphs. `add_to_existing_cuts` is not needed below — these + # graphs were built a moment ago and carry no training results of their own, + # so `SDDP.train` sees a model it has never trained. + if resume_cuts !== nothing + isfile(String(resume_cuts)) || error("no cut file to resume from: $resume_cuts") + # RESTORED cuts get the same rule as newly generated ones, so a resumed + # lineage cannot reintroduce the denormals it exists to avoid. + canonicalize_cuts && canonicalize_cut_file!(String(resume_cuts), case) + SDDP.read_cuts_from_file(backward_graph, String(resume_cuts)) + SDDP.read_cuts_from_file(forward, String(resume_cuts)) + end + + ids = [b.index for b in case.batteries] + history = NamedTuple[] + stock = SDDP.AlternativePostIterationCallback(forward) + iteration = Ref(0) + t0 = time() + + # The stock callback FIRST — the periodic evaluation is composed around it, + # so the cuts the ACP graph carries are exactly the ones stock SDDP made. + callback = function (result) + stock(result) + iteration[] += 1 + (protocol === nothing || evaluate_every === nothing) && return + iteration[] % Int(evaluate_every) == 0 || return + sims = simulate_battery_sddp_on((forward = forward, num_stages = Int(num_stages)), + protocol; ids = ids, columns = evaluate_columns) + # The CORRECTED physical cost, element-wise: each stage through the + # shared contract, validated and projected per bus before anything is + # summed. `acp_raw_mean` keeps the solver's raw objective sum beside it + # as a diagnostic, never as the headline. + phys = [sddp_physical_path_cost(case, s, num_stages) for s in sims] + costs = [p.cost for p in phys] + raw = [sddp_path_cost(case, s, num_stages) for s in sims] + rec = maximum(p.worst_recourse for p in phys) + push!(history, (iteration = iteration[], backward = backward, + bound = SDDP.calculate_bound(backward_graph), + acp_mean = mean(costs), acp_min = minimum(costs), + acp_max = maximum(costs), worst_recourse = rec, + admissible = all(p.admissible for p in phys), + acp_raw_mean = mean(raw), + elapsed = time() - t0)) + return + end + + kwargs = Dict{Symbol,Any}(:print_level => print_level, + :forward_pass => SDDP.AlternativeForwardPass(forward), + :post_iteration_callback => callback) + iteration_limit === nothing || (kwargs[:iteration_limit] = Int(iteration_limit)) + time_limit === nothing || (kwargs[:time_limit] = Float64(time_limit)) + SDDP.train(backward_graph; kwargs...) + elapsed = time() - t0 + + if cut_path !== nothing + SDDP.write_cuts_to_file(backward_graph, cut_path) + # Canonicalize on the way OUT too: with one-iteration segments this + # guarantees every cut is canonical before the next iteration reads it. + canonicalize_cuts && canonicalize_cut_file!(cut_path, case) + end + + return (backward_graph = backward_graph, forward = forward, + status = SDDP.termination_status(backward_graph), + # Which arm this is, in four fields, so no consumer has to infer it + # and none can print the DC scalar under the SOC one's name. + backward = backward, + backward_formulation = spec.formulation, + bound_name = spec.bound_name, + bound_bounds_acp = spec.bounds_acp, + # In the objective units of the PGLib case, like every stage cost the + # backward graph was built from. Every consumer of `trained.bound` — + # `cost_report`, the smoke and any downstream analysis — reads it as + # it stands, directly comparable with `sddp_path_cost`. + bound = SDDP.calculate_bound(backward_graph), + case = case, + elapsed = elapsed, num_stages = Int(num_stages), + iterations = iteration[], history = history, + cut_path = cut_path) +end + +""" + sddp_method_id(trained) -> Symbol + +The study's method identifier of a trained run: `:sddp_soc` or `:sddp_dc`. + +# Notes +Derived from the run itself rather than from what a caller remembers asking for, +so a result table, a log line and a serialized artifact all name the same arm. +""" +sddp_method_id(trained) = Symbol("sddp_", trained.backward) + +""" + simulate_battery_sddp(trained, num_replications; ids, seed) -> Vector + +Simulate the trained policy on the TRUE-ACP graph. + +# Arguments +- `ids`: battery identifiers, in the order the recorded energy vectors use. + +# Notes +Simulation records, per stage: the stage objective, the outgoing battery energy, +the charge/discharge split and the two recourse totals. The recourse totals are +recorded on every path because a policy that leans on them is rejected, and a +rejection rule that is only evaluated at the end of a campaign is not a rule. + +Everything is recorded through `custom_recorders`, which read the live +subproblem after it is solved, so what is stored is the value the solver +actually returned rather than a re-derivation of it. +""" +function simulate_battery_sddp(trained, num_replications::Integer; + ids::AbstractVector{Int}, + seed::Integer = 20260804) + Random.seed!(seed) + return SDDP.simulate(trained.forward, num_replications; + custom_recorders = battery_recorders(collect(Int, ids))) +end + +""" + simulate_battery_sddp_on(trained, protocol; ids, columns=nothing) -> Vector + +Simulate the trained policy on the true-ACP graph over EXACTLY the demand paths +of a protocol, in the protocol's own order. + +# Arguments +- `protocol::AbstractMatrix{<:Integer}`: the `(stages × scenarios)` atom-index + matrix. + +# Keywords +- `columns`: the global scenario identifiers to simulate; every column by + default. + +# Notes +This is what makes an SDDP cost PAIRED with a perfect-foresight cost or with a +TS-DDR cost: all three are then accumulated over the same demand realizations, +and the difference between two policies is measured scenario by scenario rather +than between two independent samples of a distribution whose spread dwarfs it. + +`SDDP.Historical` is SDDP.jl's own mechanism for replaying a fixed set of +realizations; nothing about the policy or the graph changes, only which noises +the forward pass sees. +""" +function simulate_battery_sddp_on(trained, protocol::AbstractMatrix{<:Integer}; + ids::AbstractVector{Int}, + columns = nothing) + T = trained.num_stages + T <= size(protocol, 1) || + throw(ArgumentError("protocol has $(size(protocol, 1)) stages but the policy has $T")) + cols = columns === nothing ? collect(1:size(protocol, 2)) : collect(Int.(columns)) + scenarios = [[(t, Int(protocol[t, c])) for t in 1:T] for c in cols] + return SDDP.simulate(trained.forward, length(scenarios); + sampling_scheme = SDDP.Historical(scenarios), + custom_recorders = battery_recorders(collect(Int, ids))) +end + +""" + sddp_smoke(; case_dir, backward, num_stages, iteration_limit, replications) + -> NamedTuple + +The SDDP construction smoke, for either backward formulation. + +# Notes +This is a CONSTRUCTION test, not a performance experiment. It runs the smallest +horizon and iteration count that still exercises every mechanism the study +depends on — actual backward nodes of the requested formulation, actual ACP +forward nodes, battery energy as a genuine state that propagates, demand +realized from the frozen atoms, cuts created by stock SDDP, and clean ACP +forward solves — and it reports nothing about policy quality. + +Every line it prints names the arm, and the backward scalar is printed under the +name [`BACKWARD_SPECS`](@ref) gives it, so a DC log can never be read as a +statement about a bound on the ACP problem. +""" +function sddp_smoke(; case_dir::AbstractString = joinpath(@__DIR__, "case", "pglib_opf_case14_ieee"), + backward::Symbol = Symbol(get(ENV, "DR_BAT_SDDP_BACKWARD", "soc")), + num_stages::Integer = parse(Int, get(ENV, "DR_BAT_SDDP_STAGES", "3")), + iteration_limit::Integer = parse(Int, get(ENV, "DR_BAT_SDDP_ITERATIONS", "10")), + replications::Integer = parse(Int, get(ENV, "DR_BAT_SDDP_SIMS", "6"))) + case = read_battery_case(case_dir) + ids = [b.index for b in case.batteries] + trained = train_battery_sddp(case; backward = backward, num_stages = num_stages, + iteration_limit = iteration_limit) + sims = simulate_battery_sddp(trained, replications; ids = ids) + @printf("SDDP smoke [%s]: %d stages, %d iterations, status %s\n", + sddp_method_id(trained), num_stages, iteration_limit, trained.status) + @printf(" backward formulation %s forward formulation %s\n", + trained.backward_formulation, FORWARD_FORMULATION) + # `trained.bound` and `sddp_path_cost` are both in the objective units of the + # PGLib case. Whether the scalar BOUNDS the forward costs printed after it is + # a property of the formulation, and is stated rather than implied. + @printf(" %s over %d stages: %.6f (elapsed %.1f s)\n", + trained.bound_name, trained.num_stages, trained.bound, trained.elapsed) + @printf(" (%s a lower bound on the true ACP problem)\n", + trained.bound_bounds_acp ? "is" : "is NOT") + ncuts = sum(length(node.bellman_function.global_theta.cuts) + for (_, node) in trained.backward_graph.nodes) + @printf(" stock cuts created: %d\n", ncuts) + costs = [sddp_path_cost(case, s, num_stages) for s in sims] + @printf(" ACP forward cost over %d paths: mean %.6f min %.6f max %.6f\n", + length(costs), mean(costs), minimum(costs), maximum(costs)) + worst_d = maximum(sims[i][t][:deficit] for i in eachindex(sims), t in 1:num_stages) + worst_s = maximum(sims[i][t][:surplus] for i in eachindex(sims), t in 1:num_stages) + @printf(" worst recourse on any simulated stage: deficit %.3e surplus %.3e\n", worst_d, worst_s) + for (j, k) in enumerate(ids) + # `energy_out` is recorded in `ids` order, so column j is battery k. + traj = [sims[1][t][:energy_out][j] for t in 1:num_stages] + @printf(" battery %d energy path (path 1): %.5f -> %s\n", k, + sims[1][1][:energy_in][j], + join((@sprintf("%.5f", x) for x in traj), " -> ")) + end + return (trained = trained, sims = sims, cuts = ncuts, + worst_deficit = worst_d, worst_surplus = worst_s) +end + +if abspath(PROGRAM_FILE) == @__FILE__ + sddp_smoke() +end + + +# ───────────────────────────────────────────────────────────────────────────── +# The standard cost report +# ───────────────────────────────────────────────────────────────────────────── + +""" + cost_report(case, trained, protocol; columns, alpha=0.95, io=stdout) -> NamedTuple + +The four quantities this study reports about a case, always together and always +with the same meaning. + +# Returns / prints + +| | | +|---|---| +| `sddp_mean`, `sddp_cvar`, `sddp_max` | the SDDP policy's TRUE-ACP cost over `columns`: mean, the `alpha`-CVaR (mean of the worst `1-alpha` tail) and the worst path | +| `bound` | the backward formulation's own scalar over the SAME horizon, under the name `trained.bound_name` gives it | +| `pf_mean`, `pf_cvar`, `pf_max` | the true-ACP perfect-foresight cost over the SAME paths | +| `gap_bound` | `(sddp_mean − bound) / sddp_mean` | +| `gap_pf` | `(sddp_mean − pf_mean) / pf_mean` | + +# Notes +The two gaps answer different questions and are never added or conflated: + +- `gap_bound` is the SDDP policy's true cost against the backward formulation's + scalar. For the SOC-WR arm that scalar is a LOWER BOUND on the relaxed problem + and the difference is dominated by how loose the relaxation is on this network; + it is not a statement about the policy. For the DC arm it is not a bound at + all, and the difference is the distance between a true-ACP cost and the + internal scalar of a different approximation — a diagnostic of the DC training, + never a headroom. +- `gap_pf` is the policy against CLAIRVOYANCE on the same demand paths. It is + the policy headroom, and it contains the value of perfect information, which no + nonanticipative method can recover. + +Every quantity in the table is in the objective units of the PGLib case, the one +unit system this study has. The policy costs come through +[`sddp_path_cost`](@ref), the bound off the backward graph built from those same +stage costs, and the perfect-foresight and forecast costs are sums of +`cost_stage`. No quantity here is rescaled at any point, so the rows are +comparable as they stand. + +Both are reported on the SAME paths and the SAME horizon as the bound, because a +bound never bounds a quantity accumulated over a different horizon and a paired +comparison is only paired if both sides saw the same scenarios. The tail metric +is reported beside every mean because a storage policy that is good on average +and bad in the tail is a different object from one that is good in both. + +# The supporting metrics, and why they exist +A large `gap_bound` is easy to mistake for room a better policy could take. These +two say how much of each gap is actually available: + +- **`e` — the deterministic-forecast policy** and `VSS = (forecast − a)/a`. If + ignoring the uncertainty entirely costs nothing, the case has no stochastic + content and no two methods can be told apart on it, whatever the other gaps. +- **`f` — the recoverable ceiling**, `(a − max(bound, pf_mean))/a`. The best + nonanticipative cost `RP` is unknown, but it is at least the relaxation bound + and at least the wait-and-see mean, so this is a hard cap on what TS-DDR could + ever take from `a`. It is an upper bound on the prize, not the prize. + + **On the DC arm the bound term is dropped from that maximum**, and this is the + one place where the difference between the two scalars changes arithmetic + rather than only a label. The DC-approximation training bound does not bound + `RP`, so admitting it into `max(·)` could tighten the ceiling with a number + that has no right to tighten it, and could report a prize smaller than the one + that exists. The ceiling then rests on the wait-and-see mean alone, which is a + valid lower bound on `RP` under either arm. + +**The policy is never run in the SOC model.** This SDDP is deliberately +INCONSISTENT — cuts built in the relaxation, states visited in the true model — +so operating the same cuts inside the relaxation gives the cost of a different +trajectory of a different problem, not a decomposition of anything about the +policy we have. +""" +function cost_report(case::BatteryCase, trained, protocol::AbstractMatrix{<:Integer}; + columns, alpha::Real = 0.95, forecast::Bool = true, + io::IO = stdout) + T = trained.num_stages + ids = [b.index for b in case.batteries] + cols = collect(Int.(columns)) + + sims = simulate_battery_sddp_on(trained, protocol; ids = ids, columns = cols) + # Row a is the CORRECTED physical cost under the shared contract, so it is + # the same estimand the Exa arm reports and the same one a checkpoint was + # selected on. The raw objective sum is kept beside it as `sddp_raw_mean`, + # a solver diagnostic; comparing two engines on a raw objective compares + # their barrier parameters. + phys = [sddp_physical_path_cost(case, s, T) for s in sims] + sddp = [p.cost for p in phys] + sddp_raw = [sddp_path_cost(case, s, T) for s in sims] + rec = maximum(p.worst_recourse for p in phys) + all(p.admissible for p in phys) || error( + "the SDDP forward panel is INADMISSIBLE: worst individual per-bus recourse " * + "$(rec) pu exceeds $(PHYSICAL_RECOURSE_TOL); a cost report over a panel the " * + "physical contract rejects would report a policy that does not exist") + + panel = perfect_foresight_panel(case, protocol; ids = cols) + panel.complete || + error("perfect-foresight panel incomplete; a gap against a partial panel is a gap against a different problem") + pf = [r.cost for r in panel.rows] + + fc = forecast ? forecast_policy_cost(case, protocol; columns = cols) : nothing + + tail(v) = (n = max(1, ceil(Int, (1 - alpha) * length(v))); mean(sort(v; rev = true)[1:n])) + a, bnd, cc = mean(sddp), trained.bound, mean(pf) + # RP, the best nonanticipative cost, is unknown but is bounded below by the + # wait-and-see mean always, and by the backward scalar only when that scalar + # is a bound on the true ACP problem. Whichever admissible one is tighter + # caps what ANY better policy — TS-DDR included — could recover from `a`. + rp_lower = trained.bound_bounds_acp ? max(bnd, cc) : cc + + r = (n = length(cols), horizon = T, alpha = alpha, + method = sddp_method_id(trained), backward = trained.backward, + sddp_mean = a, sddp_cvar = tail(sddp), sddp_max = maximum(sddp), + sddp_sem = std(sddp) / sqrt(length(sddp)), sddp_recourse = rec, + sddp_raw_mean = mean(sddp_raw), + bound = bnd, bound_name = trained.bound_name, + bound_bounds_acp = trained.bound_bounds_acp, + pf_mean = cc, pf_cvar = tail(pf), pf_max = maximum(pf), + pf_sem = std(pf) / sqrt(length(pf)), + forecast_mean = fc === nothing ? NaN : fc.mean, + forecast_max = fc === nothing ? NaN : fc.max, + gap_bound = (a - bnd) / a, + gap_pf = (a - cc) / cc, + vss = fc === nothing ? NaN : (fc.mean - a) / a, + recoverable_ceiling = (a - rp_lower) / a) + + @printf(io, "\n%s: %d stages, %d paths, CVaR at %.0f%%\n", r.method, T, r.n, 100alpha) + @printf(io, " a) SDDP true-ACP cost mean %14.2f CVaR %14.2f max %14.2f (sem %.2f)\n", + r.sddp_mean, r.sddp_cvar, r.sddp_max, r.sddp_sem) + @printf(io, " b) %-20s %14.2f (%s a lower bound on the true ACP problem)\n", + r.bound_name, r.bound, r.bound_bounds_acp ? "is" : "is NOT") + @printf(io, " c) perfect foresight mean %14.2f CVaR %14.2f max %14.2f (sem %.2f)\n", + r.pf_mean, r.pf_cvar, r.pf_max, r.pf_sem) + @printf(io, " d) gap a-b %8.4f%% gap a-c %8.4f%% worst recourse %.2e\n", + 100r.gap_bound, 100r.gap_pf, r.sddp_recourse) + @printf(io, "\n supporting metrics\n") + if fc !== nothing + @printf(io, " e) deterministic-forecast policy %14.2f max %14.2f\n", + r.forecast_mean, r.forecast_max) + @printf(io, " VSS = (forecast - a)/a = %7.4f%% — how much the uncertainty is worth\n", + 100r.vss) + end + @printf(io, " f) recoverable ceiling %7.4f%% — (a - %s)/a, the most ANY better\n", + 100r.recoverable_ceiling, r.bound_bounds_acp ? "max(b,c)" : "c") + @printf(io, " nonanticipative policy, TS-DDR included, could win from a\n") + return r +end + +# ───────────────────────────────────────────────────────────────────────────── +# The study's four method identifiers +# +# One table, carried by BOTH public engines with the same four rows and the same +# invariant fields, because the comparison the study makes is between four +# methods and not between two packages. Each engine can RUN the two methods it +# owns and refuses the other two by name, pointing at the engine that owns them — +# neither package loads the other, and a dispatch layer that pretended otherwise +# would fail somewhere less legible than here. The peer copy of this block lives +# in `DecisionRulesExa.jl/examples/BatteryStorageOPF/train_battery_exa_strict.jl` +# and each suite asserts the identifiers and the invariants independently. +# ───────────────────────────────────────────────────────────────────────────── + +""" + BATTERY_METHODS + +The four policies this study compares, keyed by their stable identifier. + +# The rows + +| identifier | family | engine | what varies | +|---|---|---|---| +| `:tsddr_nonlinear` | `:tsddr` | `:exa` | LSTM encoder, nonlinear head | +| `:tsldr_recurrent_linear` | `:tsddr` | `:exa` | affine recurrence, affine head | +| `:sddp_soc` | `:sddp` | `:jump` | `SOCWRConicPowerModel` backward cuts | +| `:sddp_dc` | `:sddp` | `:jump` | `DCPPowerModel` backward cuts | + +# The invariants + +Every row declares the SAME `horizon`, `stage_semantics`, `recourse`, +`cost_contract` and `comparison` fields, and a test in each engine's suite +asserts it. They are recorded rather than assumed because the four methods are +only comparable if they share them: the frozen case and protocol identities, +`T = 24`, strict reachable targets with no target slack, uncapped physical nodal +recourse with the same admissibility rule, `physical_stage_cost` as the only +headline cost, and a true-ACP evaluation on paired protocol columns. + +`protocol` is `"screening"` for every row at this phase. The final 500-column +protocol is not opened by anything in this file. +""" +const BATTERY_METHODS = Dict{Symbol,NamedTuple}( + :tsddr_nonlinear => ( + family = :tsddr, engine = :exa, architecture = :tsddr_nonlinear, + backward = nothing, + summary = "strict TS-DDR with an LSTM encoder and a nonlinear bounded head"), + :tsldr_recurrent_linear => ( + family = :tsddr, engine = :exa, architecture = :tsldr_recurrent_linear, + backward = nothing, + summary = "strict recurrent TSLDR: affine recurrence and affine head, " * + "raw target affine in the observed demand history"), + :sddp_soc => ( + family = :sddp, engine = :jump, architecture = nothing, + backward = :soc, + summary = "SDDP with SOCWRConicPowerModel backward cuts and ACP forward decisions"), + :sddp_dc => ( + family = :sddp, engine = :jump, architecture = nothing, + backward = :dc, + summary = "SDDP with DCPPowerModel backward cuts and ACP forward decisions"), +) + +""" +The properties every one of [`BATTERY_METHODS`](@ref)' four rows shares. +""" +const BATTERY_METHOD_INVARIANTS = ( + horizon = 24, + protocol = "screening", + stage_semantics = "strict reachable outgoing-energy target, no target slack", + recourse = "uncapped two-sided physical nodal active recourse, admissibility at 1e-6 pu", + cost_contract = "physical_stage_cost from battery_solution_schema.jl", + comparison = "true-ACP cost on paired frozen protocol columns", +) + +""" + battery_method(id::Symbol) -> NamedTuple + +The descriptor of one method identifier, with the shared invariants merged in. + +# Notes +An unknown identifier raises and lists the four, rather than returning +`nothing`: a campaign driver that silently skipped a misspelled method would +report three-quarters of a study as a whole one. +""" +function battery_method(id::Symbol) + haskey(BATTERY_METHODS, id) || throw(ArgumentError( + "unknown battery method :$id; the study's methods are $(sort!(collect(keys(BATTERY_METHODS))))")) + return merge(BATTERY_METHODS[id], (id = id,), BATTERY_METHOD_INVARIANTS) +end + +""" + run_battery_method(id::Symbol, case::BatteryCase; kwargs...) -> NamedTuple + +Dispatch a method identifier to this engine's implementation. + +# Notes +This engine owns the two `:sddp` rows and runs them through the ONE +[`train_battery_sddp`](@ref) entry point, differing only in `backward`. The two +`:exa` rows are the ExaModels engine's: this package has neither Flux nor +ExaModels, so they are refused by name here rather than half-implemented. + +This is a dispatch layer, not a campaign runner. It selects an implementation +and forwards keyword arguments; it schedules nothing, resumes nothing and writes +no ledger. +""" +function run_battery_method(id::Symbol, case::BatteryCase; kwargs...) + m = battery_method(id) + m.engine === :jump || error( + "method :$id runs on the $(m.engine) engine (DecisionRulesExa.jl/examples/BatteryStorageOPF), " * + "not on this one; this package loads neither Flux nor ExaModels") + return train_battery_sddp(case; backward = m.backward, kwargs...) +end diff --git a/examples/BatteryStorageOPF/battery_solution_schema.jl b/examples/BatteryStorageOPF/battery_solution_schema.jl new file mode 100644 index 0000000..031f180 --- /dev/null +++ b/examples/BatteryStorageOPF/battery_solution_schema.jl @@ -0,0 +1,540 @@ +# battery_solution_schema.jl +# +# The shared, engine-neutral description of a solved battery-storage AC-OPF +# trajectory. This file is shipped BYTE-IDENTICALLY in both public engines, so +# a solution written by the JuMP/PowerModels engine and a solution written by +# the ExaModels engine are the same object and can be differenced by name +# rather than by position. +# +# Format: one long CSV with header +# +# scenario,stage,class,index,value +# +# `class` names a physical quantity (see `SOLUTION_CLASSES`), `index` is the +# NETWORK identifier of the component it belongs to (bus id, generator id, +# branch id, battery id) — never a positional offset — and `value` is a Float64 +# printed with full round-tripping precision. Scalars per stage use index 0. +# +# Why long format and why identifiers. Objective agreement between two engines +# can hide a different feasible set, a null-space variable, or a solver barrier +# offset; only a per-variable comparison catches those, and a per-variable +# comparison is only trustworthy when both sides agree what "variable 7" means. +# Component identifiers in PGLib cases are arbitrary integers and need not be +# consecutive, so positional indexing is not merely fragile — it is wrong. +# +# This file deliberately has no package dependencies beyond `Printf` and the +# standard library: it must be copyable into either engine without dragging a +# resolver conflict behind it. + +using Printf + +""" +Physical classes a battery-storage solution may record. + +Battery-layer classes (index = battery identifier): + +- `"energy_in"` incoming energy ``e_{b,t-1}`` (pu·h) +- `"energy_out"` outgoing energy ``e_{b,t}`` (pu·h) +- `"target"` the strict target ``\\hat e_{b,t}`` (pu·h); absent in the + targetless SDDP formulation +- `"target_dual"` the multiplier ``\\lambda_{b,t}`` of the strict target + equality, i.e. ``\\partial Q/\\partial \\hat e_{b,t}`` +- `"p_ch"` charging power (pu) +- `"p_dis"` discharging power (pu) +- `"p_bat"` net active injection ``p^{dis}-p^{ch}`` (pu) + +Nodal classes (index = bus identifier): + +- `"deficit"` the uncapped nonnegative recourse injection ``d_{i,t}`` (pu) +- `"surplus"` the uncapped nonnegative recourse sink ``s_{i,t}`` (pu) +- `"pd"`, `"qd"` REALIZED active/reactive demand at the bus (pu) +- `"vm"`, `"va"` voltage magnitude (pu) and angle (rad) +- `"pg_bus"`, `"qg_bus"` generation aggregated to the bus (pu) +- `"price_active"`, `"price_reactive"` nodal duals of the balance, where the + engine has them + +Generator classes (index = generator identifier): `"pg"`, `"qg"` (pu). + +Branch classes (index = branch identifier): `"p_fr"`, `"q_fr"`, `"p_to"`, +`"q_to"` (pu, at the respective ends). + +Scalar classes (index 0): + +- `"cost_generation"`, `"cost_throughput"`, `"cost_deficit"`, `"cost_surplus"` +- `"cost_stage"` their sum for the stage +- `"objective"` the engine's own reported stage objective +- `"residual_equality"` worst absolute equality-constraint residual +- `"solved"` 1.0 when the engine accepted the solve, 0.0 otherwise +""" +const SOLUTION_CLASSES = ( + "energy_in", "energy_out", "target", "target_dual", + "p_ch", "p_dis", "p_bat", + "deficit", "surplus", "pd", "qd", "vm", "va", "pg_bus", "qg_bus", + "price_active", "price_reactive", + "pg", "qg", + "p_fr", "q_fr", "p_to", "q_to", + "cost_generation", "cost_throughput", "cost_deficit", "cost_surplus", + "cost_stage", "objective", "residual_equality", "solved", +) + +"Header line of every solution CSV." +const SOLUTION_HEADER = "scenario,stage,class,index,value" + +""" + SolutionRecorder + +Accumulator for solution records in the shared long format. + +# Fields +- `rows::Vector{Tuple{Int,Int,String,Int,Float64}}`: `(scenario, stage, class, + index, value)` in insertion order. + +# Notes +Rows are appended in whatever order an engine produces them; nothing downstream +depends on the order, because comparison is by `(scenario, stage, class, +index)`. Insertion order IS preserved on write so that a diff of two files from +the same engine stays readable. +""" +struct SolutionRecorder + rows::Vector{Tuple{Int,Int,String,Int,Float64}} +end + +SolutionRecorder() = SolutionRecorder(Tuple{Int,Int,String,Int,Float64}[]) + +""" + record!(rec, scenario, stage, class, index, value) + +Append one record, validating the class name. + +# Notes +An unknown class is an ERROR rather than a silently-written row: a typo in a +class name would make the corresponding quantity vanish from a cross-engine +comparison and the comparison would still report "all classes agree". +""" +function record!(rec::SolutionRecorder, scenario::Integer, stage::Integer, + class::AbstractString, index::Integer, value::Real) + class in SOLUTION_CLASSES || error("unknown solution class \"$class\"") + push!(rec.rows, (Int(scenario), Int(stage), String(class), Int(index), Float64(value))) + return rec +end + +""" + record_map!(rec, scenario, stage, class, values::AbstractDict) + +Append one record per `(identifier => value)` pair, in sorted identifier order. +""" +function record_map!(rec::SolutionRecorder, scenario::Integer, stage::Integer, + class::AbstractString, values::AbstractDict) + for k in sort!(collect(keys(values))) + record!(rec, scenario, stage, class, k, values[k]) + end + return rec +end + +""" + write_solution(path, rec::SolutionRecorder) + +Write the accumulated records to `path` in the shared long format. + +# Notes +Values are printed with `%.17g`, which round-trips every `Float64` exactly, so a +cross-engine difference read back from these files is a difference between the +engines and never a difference introduced by printing. +""" +function write_solution(path::AbstractString, rec::SolutionRecorder) + mkpath(dirname(abspath(path))) + open(path, "w") do io + println(io, SOLUTION_HEADER) + for (s, t, c, i, v) in rec.rows + @printf(io, "%d,%d,%s,%d,%.17g\n", s, t, c, i, v) + end + end + return path +end + +""" + physical_residuals(network, batteries, Δt, sol) -> NamedTuple + +Recompute the physics of one solved stage from its reported values and return +the worst violation in each class. + +# Arguments +- `network::AbstractDict`: the frozen PGLib network (per-unit). +- `batteries`: the case's batteries; each must expose `index`, `bus`, + `self_discharge`, `charge_efficiency`, `discharge_efficiency`. +- `Δt::Real`: stage duration in hours. +- `sol`: a named tuple or dictionary exposing, keyed by NETWORK identifier, + `vm`, `va`, `pg`, `qg`, `p_fr`, `q_fr`, `p_to`, `q_to`, `deficit`, `surplus`, + `pd`, `qd`, `p_ch`, `p_dis`, `energy_in`, `energy_out`. + +# Returns +A `NamedTuple` of worst absolute violations: +`(branch_flow, active_balance, reactive_balance, thermal, angle, voltage, + transition)`. + +# Notes +This is deliberately INDEPENDENT of both engines: it re-derives the AC branch +flows from the reported voltages, re-adds the nodal balances from the reported +injections, and re-applies the battery transition to the reported controls. An +engine can therefore be wrong in a way its own solver is happy with and still be +caught here — which is the only kind of check worth running against a manually +written formulation. + +Angles are only meaningful for a polar solution. A solution whose `va` entries +are `NaN` (a W-space relaxation has no angle variable) yields `NaN` in the +branch-flow and angle classes, which is honest rather than silently zero. +""" +function physical_residuals(network::AbstractDict, batteries, Δt::Real, sol) + vm, va = sol.vm, sol.va + worst_flow = 0.0 + worst_thermal = 0.0 + worst_angle = 0.0 + + inj_p = Dict{Int,Float64}(k => 0.0 for k in keys(vm)) + inj_q = Dict{Int,Float64}(k => 0.0 for k in keys(vm)) + + for (_, br) in network["branch"] + Int(get(br, "br_status", 1)) == 0 && continue + l = Int(br["index"]) + haskey(sol.p_fr, l) || continue + f, t = Int(br["f_bus"]), Int(br["t_bus"]) + r, x = Float64(get(br, "br_r", 0.0)), Float64(br["br_x"]) + r2x2 = r^2 + x^2 + g = r2x2 > 0 ? r / r2x2 : 0.0 + b = r2x2 > 0 ? -x / r2x2 : 0.0 + tap = Float64(get(br, "tap", 1.0)); tap = tap ≈ 0 ? 1.0 : tap + shift = Float64(get(br, "shift", 0.0)) + tr, ti = tap * cos(shift), tap * sin(shift) + ttm = tr^2 + ti^2; ttm = ttm > 0 ? ttm : 1.0 + g_fr, b_fr = Float64(get(br, "g_fr", 0.0)), Float64(get(br, "b_fr", 0.0)) + g_to, b_to = Float64(get(br, "g_to", 0.0)), Float64(get(br, "b_to", 0.0)) + + vf, vt = vm[f], vm[t] + θ = va[f] - va[t] + pfr = (g + g_fr) / ttm * vf^2 + (-g * tr + b * ti) / ttm * vf * vt * cos(θ) + + (-b * tr - g * ti) / ttm * vf * vt * sin(θ) + qfr = -(b + b_fr) / ttm * vf^2 - (-b * tr - g * ti) / ttm * vf * vt * cos(θ) + + (-g * tr + b * ti) / ttm * vf * vt * sin(θ) + pto = (g + g_to) * vt^2 + (-g * tr - b * ti) / ttm * vt * vf * cos(-θ) + + (-b * tr + g * ti) / ttm * vt * vf * sin(-θ) + qto = -(b + b_to) * vt^2 - (-b * tr + g * ti) / ttm * vt * vf * cos(-θ) + + (-g * tr - b * ti) / ttm * vt * vf * sin(-θ) + + worst_flow = max(worst_flow, abs(pfr - sol.p_fr[l]), abs(qfr - sol.q_fr[l]), + abs(pto - sol.p_to[l]), abs(qto - sol.q_to[l])) + + rate = Float64(get(br, "rate_a", Inf)) + if isfinite(rate) + worst_thermal = max(worst_thermal, + sol.p_fr[l]^2 + sol.q_fr[l]^2 - rate^2, + sol.p_to[l]^2 + sol.q_to[l]^2 - rate^2) + end + amin = Float64(get(br, "angmin", -pi)); amax = Float64(get(br, "angmax", pi)) + worst_angle = max(worst_angle, amin - θ, θ - amax) + + inj_p[f] -= sol.p_fr[l]; inj_q[f] -= sol.q_fr[l] + inj_p[t] -= sol.p_to[l]; inj_q[t] -= sol.q_to[l] + end + + for (_, gen) in network["gen"] + Int(get(gen, "gen_status", 1)) == 0 && continue + gi = Int(gen["index"]) + haskey(sol.pg, gi) || continue + bus = Int(gen["gen_bus"]) + inj_p[bus] += sol.pg[gi] + inj_q[bus] += sol.qg[gi] + end + + for (_, sh) in get(network, "shunt", Dict{String,Any}()) + Int(get(sh, "status", 1)) == 0 && continue + bus = Int(sh["shunt_bus"]) + haskey(inj_p, bus) || continue + inj_p[bus] -= Float64(get(sh, "gs", 0.0)) * vm[bus]^2 + inj_q[bus] += Float64(get(sh, "bs", 0.0)) * vm[bus]^2 + end + + worst_transition = 0.0 + for b in batteries + haskey(sol.p_ch, b.index) || continue + inj_p[b.bus] += sol.p_dis[b.index] - sol.p_ch[b.index] + lhs = sol.energy_out[b.index] - b.self_discharge * sol.energy_in[b.index] - + b.charge_efficiency * Δt * sol.p_ch[b.index] + + (Δt / b.discharge_efficiency) * sol.p_dis[b.index] + worst_transition = max(worst_transition, abs(lhs)) + end + + worst_p = 0.0 + worst_q = 0.0 + worst_v = 0.0 + for (_, bus) in network["bus"] + i = Int(bus["index"]) + haskey(inj_p, i) || continue + worst_p = max(worst_p, abs(inj_p[i] + sol.deficit[i] - sol.surplus[i] - sol.pd[i])) + worst_q = max(worst_q, abs(inj_q[i] - sol.qd[i])) + worst_v = max(worst_v, Float64(get(bus, "vmin", 0.0)) - vm[i], + vm[i] - Float64(get(bus, "vmax", Inf))) + end + + return (branch_flow = worst_flow, active_balance = worst_p, + reactive_balance = worst_q, thermal = worst_thermal, + angle = worst_angle, voltage = worst_v, transition = worst_transition) +end + +""" + read_solution(path) -> Dict{Tuple{Int,Int,String,Int},Float64} + +Read a solution file into a lookup keyed by `(scenario, stage, class, index)`. + +# Notes +Duplicate keys are an ERROR. Two rows claiming the same physical quantity mean +the writer lost track of what it was recording, and silently keeping the last +one would make a parity comparison depend on file order. +""" +function read_solution(path::AbstractString) + out = Dict{Tuple{Int,Int,String,Int},Float64}() + open(path, "r") do io + header = readline(io) + header == SOLUTION_HEADER || + error("$path: unexpected header \"$header\"; expected \"$SOLUTION_HEADER\"") + for line in eachline(io) + isempty(strip(line)) && continue + parts = split(line, ',') + length(parts) == 5 || error("$path: malformed row \"$line\"") + key = (parse(Int, parts[1]), parse(Int, parts[2]), String(parts[3]), parse(Int, parts[4])) + haskey(out, key) && error("$path: duplicate record for $key") + out[key] = parse(Float64, parts[5]) + end + end + return out +end + +# ───────────────────────────────────────────────────────────────────────────── +# The physical cost contract +# +# An interior-point method does not leave a nonnegative variable AT zero. It +# leaves it a barrier tolerance away, and the sign of that offset depends on the +# solver's own bound handling — Ipopt parks the two nodal recourse injections at +# exactly zero, MadNLP a bound-relaxation below it. On any one bus the +# difference is about 1e-8 pu and physically nothing at all. +# +# The recourse PRICE, however, is chosen far above any generator, 1e5 to 1e6 per +# pu. Multiply 1e-8 pu by 1e6 and sum over two thousand buses and the two +# engines' reported stage objectives differ by tens of cost units on a problem +# where neither used any recourse. That difference is a barrier artifact of the +# solver, not a difference between two policies, and it must never reach a +# training-selection metric or a paired cost comparison. +# +# So the reported cost is defined here, once, and both engines compute it with +# THIS code: +# +# * the raw solver objective is preserved, untouched, for diagnostics; +# * every recourse element within the declared physical tolerance of zero is +# projected to EXACTLY zero, element by element; +# * an element outside that tolerance is NOT projected. The solve is marked +# inadmissible and the caller rejects it — a policy that genuinely used +# recourse is rejected, never quietly priced. +# +# The projection changes what is REPORTED, never what was solved. The stage +# problem still carries the recourse variables at their full price, which is +# what makes a dynamically reachable target attainable; nothing here relaxes a +# constraint, adds a penalty or rewrites an objective. +# ───────────────────────────────────────────────────────────────────────────── + +""" +The bound relaxation every TRUE-ACP solve of this study runs with. + +# Notes +ONE value, shared by every ACP path on both engines: the JuMP/PowerModels/Ipopt +ACP model, the ExaModels/MadNLP ACP model on CPU **and** on GPU, nonlinear +TS-DDR, recurrent-linear TSLDR, the ACP forward evaluation of both the SOC-WR +and the DC SDDP arms, and every deterministic-equivalent, perfect-foresight, +diagnostic and cross-engine parity solve. It is stated explicitly at each site +rather than inherited from a solver default, because the two solvers do not +agree on what that default is and a study whose two engines relax bounds +differently is not comparing like with like. + +An interior-point method relaxes every variable bound by this factor before +solving, so the point it converges to may sit fractionally outside the declared +box. That is why the study's guarantee is the residual RECOMPUTED from the +reported solution — never the solver's own infeasibility report, which is the +infeasibility of the relaxed problem. + +**It may not be set to zero.** Doing so was measured to break the CUDSS/GPU path +outright: `RESTORATION_FAILED` at iteration 2 on every case and horizon tried +(`case14` T=3 and T=24, `case118` T=24, `case1354_pegase` T=24), while the CPU +path was unaffected — so a CPU-only validation cannot establish this setting. +""" +const ACP_BOUND_RELAX_FACTOR = 1e-8 + +""" +Largest transition-equality residual that still counts as exact, derived from +[`ACP_BOUND_RELAX_FACTOR`](@ref) rather than fitted to an observation. + +# Notes +The transition row carries BOTH controls, + +```math +e_t - \\alpha e_{t-1} - \\eta^{ch}\\Delta t\\, p^{ch}_t + + (\\Delta t/\\eta^{dis})\\, p^{dis}_t = 0, +``` + +and an interior-point method may leave each control a bound relaxation outside +its declared box. Projecting those deviations back onto the ORIGINAL feasible +bounds contributes about + +```math +\\Delta t\\,(\\eta^{ch}\\delta^{ch} + \\delta^{dis}/\\eta^{dis}), +``` + +which at `ACP_BOUND_RELAX_FACTOR = 1e-8` is order `2e-8`. The factor five covers +the known coefficients with margin while staying 100x tighter than the +single-digit `1e-6` original-problem residual scale the study reports at. + +Checked PER TRANSITION ROW, so it does not accumulate with the horizon. The +`max` with `1e-9` preserves the historical gate for any tighter relaxation. +""" +const TRANSITION_RESIDUAL_TOL = max(1e-9, 5 * ACP_BOUND_RELAX_FACTOR) + +""" +Largest recourse injection, in pu, that still counts as none. + +# Notes +The same constant on both sides of the study. It is well above any +interior-point method's distance-to-bound — measured at about `1e-8` pu on both +engines — and far below any quantity the network cares about. +""" +const PHYSICAL_RECOURSE_TOL = 1e-6 + +""" + project_recourse(values, tol=PHYSICAL_RECOURSE_TOL) -> (projected, worst, admissible) + +Project a recourse map element by element. + +# Returns +- `projected::Dict{Int,Float64}`: every element within `tol` of zero replaced by + exactly `0.0`, every other element kept verbatim. +- `worst::Float64`: the largest absolute RAW value, before projection. +- `admissible::Bool`: whether every element was within `tol`. + +# Notes +Element by element, never in aggregate. A thousand buses each `1e-7` pu short sum +to `1e-4` pu, which an aggregate test would wave through and which this rejects +one element at a time — and, conversely, one bus genuinely short by `1` pu is +caught even though the other thousand are clean. +""" +function project_recourse(values::AbstractDict, tol::Real = PHYSICAL_RECOURSE_TOL) + out = Dict{Int,Float64}() + worst = 0.0 + admissible = true + for k in sort!(collect(keys(values))) + v = Float64(values[k]) + worst = max(worst, abs(v)) + if abs(v) <= tol + out[k] = 0.0 + else + out[k] = v + admissible = false + end + end + return out, worst, admissible +end + +""" + physical_stage_cost(sol, recourse; tol=PHYSICAL_RECOURSE_TOL) -> NamedTuple + +The stage cost this study reports, and the raw objective it came from. + +# Arguments +- `sol`: any engine's stage solution, needing `cost_generation`, + `cost_throughput`, `deficit`, `surplus` and (optionally) `objective`. +- `recourse::RecourseCosts`: the case's own frozen recourse prices. + +# Returns +`(raw, corrected, generation, throughput, deficit, surplus, correction, +worst_recourse, admissible)` — every field in the case's own objective units. + +# Notes +`corrected` is the sum of the generation cost, the throughput cost and the +recourse actually charged AFTER projection; `correction = raw - corrected` is +the barrier artifact, reported so it can be inspected rather than discovered. + +This is the ONLY function either engine may use to produce a headline cost. The +raw objective is a solver diagnostic and comparing two engines on it compares +their barrier parameters. +""" +function physical_stage_cost(sol, recourse; tol::Real = PHYSICAL_RECOURSE_TOL) + d, worst_d, ok_d = project_recourse(sol.deficit, tol) + s, worst_s, ok_s = project_recourse(sol.surplus, tol) + cost_d = recourse.deficit * sum(values(d); init = 0.0) + cost_s = recourse.surplus * sum(values(s); init = 0.0) + gen = Float64(sol.cost_generation) + thr = Float64(sol.cost_throughput) + corrected = gen + thr + cost_d + cost_s + raw = hasproperty(sol, :objective) ? Float64(sol.objective) : NaN + return (raw = raw, corrected = corrected, generation = gen, throughput = thr, + deficit = cost_d, surplus = cost_s, + correction = isnan(raw) ? NaN : raw - corrected, + worst_recourse = max(worst_d, worst_s), + admissible = ok_d && ok_s) +end + +# ───────────────────────────────────────────────────────────────────────────── +# Checkpoint selection at machine resolution +# ───────────────────────────────────────────────────────────────────────────── + +""" +Half-width, in Float64 ULPs, of the band in which two panel costs are treated as +EQUIVALENT for checkpoint ordering. + +# Notes +Two runs of the same policy can report panel costs differing in the last bits +without differing scientifically: a resumed segment builds fresh solver handles, +so the identical scenario is solved by a different solver instance. Measured on +the certified parallel gate, that distance is 1 ULP. + +This band governs ORDERING ONLY. It never touches physical admissibility, the +recourse tolerance, or the precision of any stored or reported value — those +remain exactly as computed. +""" +const SELECTION_TIE_ULPS = 2 + +""" + ulp_distance(a, b) -> Int + +The number of representable Float64 values between `a` and `b`. + +# Notes +Computed on the ordered integer encoding, which is exact and has no tolerance of +its own. `Inf` distance is returned when either input is not finite, so a `NaN` +or an uninitialised incumbent can never look "close". +""" +function ulp_distance(a::Real, b::Real) + (isfinite(a) && isfinite(b)) || return typemax(Int) + a == b && return 0 + ia = reinterpret(Int64, Float64(a)); ia < 0 && (ia = typemin(Int64) - ia) + ib = reinterpret(Int64, Float64(b)); ib < 0 && (ib = typemin(Int64) - ib) + d = abs(widen(ia) - widen(ib)) + return d > typemax(Int) ? typemax(Int) : Int(d) +end + +""" + improves(candidate, incumbent; ulps=SELECTION_TIE_ULPS) -> Bool + +Does `candidate` beat `incumbent` by MORE than the tie band? + +# Notes +A bare `candidate < incumbent` is not deterministic at machine resolution: two +segmentations of one run can order the same two policies differently when their +panel costs differ in the last bit, and the selected checkpoint would then depend +on where the run happened to be cut. Requiring strict improvement beyond +`ulps` makes the ordering identical under any segmentation, and the caller keeps +the EARLIER global index on a tie, so the tie-break is deterministic too. + +An unset incumbent (`Inf`) is always improved upon. +""" +function improves(candidate::Real, incumbent::Real; ulps::Integer = SELECTION_TIE_ULPS) + isfinite(candidate) || return false + isfinite(incumbent) || return true + candidate < incumbent || return false + return ulp_distance(candidate, incumbent) > ulps +end diff --git a/examples/BatteryStorageOPF/build_battery_case.jl b/examples/BatteryStorageOPF/build_battery_case.jl new file mode 100644 index 0000000..1bc7f73 --- /dev/null +++ b/examples/BatteryStorageOPF/build_battery_case.jl @@ -0,0 +1,1373 @@ +# build_battery_case.jl +# +# The public case-construction API, and the script that builds the frozen cases +# of this study with it. +# +# This is the only file in the project that reaches out to PGLib.jl in order to +# CREATE case data; everything downstream reads the four frozen files through +# `battery_case.jl`. The user-facing workflow is five calls: +# +# src = acquire_pglib_case("pglib_opf_case30_ieee") # A1 +# buses, prec = select_battery_buses(src.network, SampledPlacement(3; seed = 7)) +# fleet, crec = battery_fleet(src.network, buses; power = ..., energy_hours = ...) +# supp = freeze_demand_support(sampler, src.network, 24; seed = 11, ...) # A3 +# build_case(src; dir = ..., batteries = fleet, support = supp, ...) +# +# and `verify(dir)` re-checks everything the manifest claims. +# +# Usage +# julia --project=. build_battery_case.jl # build (and mirror) +# julia --project=. build_battery_case.jl --verify # re-verify only +# DR_BAT_CASE=pglib_opf_case30_ieee julia ... build_battery_case.jl +# +# Environment overrides (all optional; defaults are the frozen correctness case): +# DR_BAT_CASE PGLib case name +# DR_BAT_DIR output directory (default: case/) +# DR_BAT_NUM number of batteries +# DR_BAT_BUSES comma-separated explicit bus override +# DR_BAT_SEED placement seed +# DR_BAT_HORIZON frozen horizon +# DR_BAT_MIRROR path of the DecisionRulesExa.jl battery example to +# mirror the case and the shared sources into +# +# Reproducibility note. PowerModels MUTATES the network dictionary it is handed +# (per-unit conversion, status propagation, angle-bound defaults). The exporter +# therefore parses the PGLib case FRESH here and writes those bytes before any +# model is instantiated, so the artifact can never depend on which formulation +# happened to be built first in the same session. Running this script twice +# produces byte-identical files. + +using PGLib +using PowerModels +using Pkg +using SHA +using Distributions + +@isdefined(BatterySpec) || include(joinpath(@__DIR__, "battery_case.jl")) +@isdefined(DemandSampler) || include(joinpath(@__DIR__, "battery_demand.jl")) + +PowerModels.silence() + +# ───────────────────────────────────────────────────────────────────────────── +# A1 — PGLib case acquisition +# ───────────────────────────────────────────────────────────────────────────── + +""" + acquire_pglib_case(name; variant=nothing, validate=true) -> NamedTuple + +Obtain a canonical PGLib benchmark and check it is usable as a study case. + +# Arguments +- `name::AbstractString`: PGLib case name, e.g. `"pglib_opf_case30_ieee"` or + `"pglib_opf_case30_ieee__api"`. + +# Keywords +- `variant`: `nothing` (inferred from the name), `"api"`, `"sad"` or `""` for the + typical operating condition. PGLib ships each benchmark in three libraries and + they are DIFFERENT SYSTEMS, not different settings of one: `__api` is the + congested "active power increase" condition and `__sad` the small-angle + difference one. Selecting one is a choice of benchmark, not a modification of a + case, which is why it goes through the acquisition entry point and is recorded + in the manifest. +- `validate::Bool`: run [`validate_network`](@ref) before returning. + +# Returns +A `NamedTuple` with + +- `name::String` — the case name as given; +- `network::Dict{String,Any}` — the parsed, per-unit PowerModels network + dictionary, VERBATIM, with the benchmark's original component identifiers; +- `versions::Dict{String,Any}` — the PGLib, PowerModels and Julia versions the + bytes came from; +- `report::NamedTuple` — the validation summary (component counts, the load and + generation totals, the largest connected component). + +# Notes +Component identifiers are the benchmark's own and are never renumbered. PGLib +cases contain nonconsecutive identifiers, isolated buses and out-of-service +components; every consumer in this study keys on identifiers rather than on +positions, and this entry point is where that is checked rather than assumed. + +Recording the PGLib version matters because a benchmark revised upstream must not +silently become "the same case": the version travels into the manifest and a +rebuild against a different release changes the artifact hashes. +""" +function acquire_pglib_case(name::AbstractString; variant = nothing, + validate::Bool = true) + v = variant === nothing ? _infer_variant(name) : String(variant) + network = isempty(v) ? pglib(String(name)) : pglib(String(name), v) + # `PGLib.pglib` WARNS and returns an empty dictionary for a name it cannot + # find. A study that accepted that would silently build a case out of + # nothing, so an empty result is an error here. + isempty(get(network, "bus", Dict())) && + error("PGLib returned no network for \"$name\"" * + (isempty(v) ? "" : " (variant \"$v\")") * + "; check the name with PGLib.find_pglib_case") + versions = Dict{String,Any}( + "PGLib" => string(_package_version("PGLib")), + "PowerModels" => string(_package_version("PowerModels")), + "julia" => string(VERSION), + "variant" => isempty(v) ? "typical" : v, + ) + report = validate ? validate_network(network) : nothing + return (name = String(name), variant = v, network = network, + versions = versions, report = report) +end + +""" + _infer_variant(name) -> String + +The PGLib library a case name belongs to, from its `__api` / `__sad` suffix. + +# Notes +Inferring rather than requiring the caller to pass it twice removes the one way +this can go silently wrong: asking for `"…__api"` out of the typical-operating- +condition library, getting a name miss, and building the study on whatever the +fuzzy name filter happened to return. +""" +function _infer_variant(name::AbstractString) + endswith(name, "__api") && return "api" + endswith(name, "__sad") && return "sad" + return "" +end + +""" + validate_network(network) -> NamedTuple + +Fail closed on the network properties this study depends on, and summarize what +was found. + +# Checks +1. the mandatory component tables exist and are non-empty; +2. every component identifier equals its own `"index"` field, so the dictionary + key and the identifier can never disagree; +3. every in-service generator, branch and load refers to an existing bus; +4. every in-service bus is reachable from the reference bus over in-service + branches — a second electrical island has its own reference angle and its own + power balance, and a study that quietly spans two of them is not the study it + says it is; +5. at least one reference bus and at least one positively-priced generator exist; +6. total nominal active load is positive. + +# Returns +Counts, totals, and the number of buses in the reference bus's island. + +# Notes +Out-of-service components are reported but not rejected: PGLib ships them and +PowerModels handles them. What is rejected is an in-service component the model +cannot place. +""" +function validate_network(network::AbstractDict) + for tbl in ("bus", "gen", "branch", "load") + haskey(network, tbl) || error("network has no \"$tbl\" table") + isempty(network[tbl]) && error("network table \"$tbl\" is empty") + end + for tbl in ("bus", "gen", "branch", "load", "shunt") + haskey(network, tbl) || continue + for (key, comp) in network[tbl] + parse(Int, string(key)) == Int(comp["index"]) || + error("$tbl entry \"$key\" carries index $(comp["index"])") + end + end + + bus_ids = Set(Int(b["index"]) for (_, b) in network["bus"]) + active_bus = Set(Int(b["index"]) for (_, b) in network["bus"] + if Int(get(b, "bus_type", 1)) != 4) + for (_, g) in network["gen"] + Int(get(g, "gen_status", 1)) == 0 && continue + Int(g["gen_bus"]) in bus_ids || error("generator $(g["index"]) sits at unknown bus $(g["gen_bus"])") + end + for (_, l) in network["load"] + Int(get(l, "status", 1)) == 0 && continue + Int(l["load_bus"]) in bus_ids || error("load $(l["index"]) sits at unknown bus $(l["load_bus"])") + end + adjacency = Dict{Int,Vector{Int}}(i => Int[] for i in bus_ids) + n_branch_off = 0 + for (_, br) in network["branch"] + if Int(get(br, "br_status", 1)) == 0 + n_branch_off += 1 + continue + end + f, t = Int(br["f_bus"]), Int(br["t_bus"]) + (f in bus_ids && t in bus_ids) || + error("branch $(br["index"]) connects unknown buses $f-$t") + push!(adjacency[f], t) + push!(adjacency[t], f) + end + + refs = [Int(b["index"]) for (_, b) in network["bus"] if Int(get(b, "bus_type", 1)) == 3] + isempty(refs) && error("network has no reference (slack) bus") + # Breadth-first search from the first reference bus over in-service branches. + seen = Set{Int}([first(sort!(refs))]) + queue = collect(seen) + while !isempty(queue) + i = pop!(queue) + for j in adjacency[i] + if !(j in seen) + push!(seen, j) + push!(queue, j) + end + end + end + stranded = setdiff(active_bus, seen) + isempty(stranded) || + error("in-service buses $(sort!(collect(stranded))) are not connected to the reference bus") + + priced = 0 + scheduled = 0 + for (_, g) in network["gen"] + # An availability schedule is checked on EVERY generator, in service or + # not: a malformed schedule on a unit that is switched on later is still + # a malformed case, and this is the last point at which the case exists + # as data rather than as a solved stage. + if haskey(g, STAGE_AVAILABILITY_KEY) + sched = g[STAGE_AVAILABILITY_KEY] + (sched isa AbstractVector && !isempty(sched)) || + error("generator $(g["index"]): \"$STAGE_AVAILABILITY_KEY\" must be a non-empty vector of multipliers") + all(x -> isfinite(Float64(x)) && Float64(x) >= 0, sched) || + error("generator $(g["index"]): \"$STAGE_AVAILABILITY_KEY\" must hold finite nonnegative multipliers, got $sched") + scheduled += 1 + end + Int(get(g, "gen_status", 1)) == 0 && continue + c = Float64.(g["cost"]) + any(!=(0), c) && (priced += 1) + end + priced > 0 || error("network has no positively-priced generator") + + total_pd = sum(Float64(l["pd"]) for (_, l) in network["load"] + if Int(get(l, "status", 1)) != 0; init = 0.0) + total_pd > 0 || error("network has no positive nominal active load") + total_pmax = sum(Float64(g["pmax"]) for (_, g) in network["gen"] + if Int(get(g, "gen_status", 1)) != 0; init = 0.0) + + return (bus = length(network["bus"]), gen = length(network["gen"]), + branch = length(network["branch"]), load = length(network["load"]), + shunt = length(get(network, "shunt", Dict())), + branches_out_of_service = n_branch_off, + buses_out_of_service = length(bus_ids) - length(active_bus), + connected_component = length(seen), + total_load_pu = total_pd, total_pmax_pu = total_pmax, + priced_generators = priced, + scheduled_generators = scheduled, + headroom = total_pmax / total_pd) +end + +"Version string of an installed package, by name." +function _package_version(name::AbstractString) + for (_, dep) in Pkg.dependencies() + dep.name == name && return something(dep.version, "unknown") + end + return "unknown" +end + +# ───────────────────────────────────────────────────────────────────────────── +# Recourse prices +# ───────────────────────────────────────────────────────────────────────────── + +""" + recourse_prices(network; factor=50) -> RecourseCosts + +Derive the two-sided recourse prices from the host case. + +# Arguments +- `network::AbstractDict`: parsed PowerModels network (per-unit). + +# Keywords +- `factor::Real`: multiple of the largest generator marginal cost. + +# Returns +- [`RecourseCosts`](@ref) with `deficit == surplus`. + +# Notes +The price is `factor` times the largest generator MARGINAL cost evaluated at that +generator's `pmax`, rounded up to an integer. Deriving it from the case rather +than hard-coding a number is what lets the same API move to another PGLib +benchmark without a hidden re-tuning, and the factor puts recourse far outside +any economic trade-off against dispatch: a policy that uses it is rejected, not +priced. + +**Why the factor is LARGE, measured.** It is tempting to shrink this price on +conditioning grounds: it is the biggest coefficient in the objective by orders of +magnitude, and on `pglib_opf_case300_ieee` a factor of 50 gives 584,698 per pu +against a worst measured NODAL price of 4,612. A failing SOC-WR subproblem duly +produced a `DUAL_INFEASIBLE` certificate whose "unbounded ray" had recourse +components of order `1e-7` — numerical noise that, multiplied by 584,698, moves +the objective by about 2. That reads as a scaling problem, and it is a trap. + +A controlled measurement on `pglib_opf_case118_ieee` says the opposite. Holding +everything else fixed and running ten SDDP iterations: + +| conic tolerance | factor | result | +|---|---|---| +| `1e-8` | 50 | 10 iterations, bound 2,152,035 | +| `1e-6` | 50 | 10 iterations, bound 2,149,769 | +| `1e-8` | **5** | **fails at node 19** | + +The factor, not the tolerance, decides whether SDDP runs, and it is the LOW price +that fails. The reason is admissibility, not scaling: as the price falls the +solution starts to USE recourse, and the base ACP sweep of the research case +shows exactly that — worst recourse 0.00 at factors 50 and 20, then 2.3e-3, +4.5e-3 and 7.5e-3 at 5, 2 and 1. Once recourse is a cheap generator rather than a +last resort, the stage problem's dual structure changes and the conic solves +degrade. + +So the price must beat every real alternative by a wide margin, and the apparent +tension with conditioning is resolved in favour of the large price. + +Deficit and surplus are priced EQUALLY. An asymmetric price would make the +recourse a one-sided economic signal and could bias which side of a reachable +interval a target is pushed towards; the recourse must be neutral, because its +purpose is feasibility, not incentive. +""" +function recourse_prices(network::AbstractDict; factor::Real = 50) + marginal = 0.0 + for (_, gen) in network["gen"] + Int(get(gen, "gen_status", 1)) == 0 && continue + cost = Float64.(gen["cost"]) + ncost = Int(gen["ncost"]) + pmax = Float64(gen["pmax"]) + # PowerModels stores the polynomial highest-order-first. The marginal + # cost of a quadratic model at pmax is 2 c2 pmax + c1; a linear model's + # is just c1; a constant model's is 0. + m = if ncost >= 3 + 2 * cost[end - 2] * pmax + cost[end - 1] + elseif ncost == 2 + cost[end - 1] + else + 0.0 + end + marginal = max(marginal, m) + end + marginal > 0 || error("recourse_prices: case has no positively-priced generator") + price = ceil(factor * marginal) + return RecourseCosts(price, price) +end + +# ───────────────────────────────────────────────────────────────────────────── +# The general case constructor +# ───────────────────────────────────────────────────────────────────────────── + +""" + build_case(source; dir, batteries, support, placement=Dict(), recourse=nothing, + protocol_stages, protocol_scenarios, mirror=nothing) -> BatteryCase + +Write the four frozen artifacts of a case and read them back through the +verifier. + +# Arguments +- `source`: the named tuple returned by [`acquire_pglib_case`](@ref). + +# Keywords +- `dir::AbstractString`: output directory. +- `batteries::Vector{BatterySpec}`: the fleet, from [`battery_fleet`](@ref). +- `support::DemandSupport`: the FROZEN finite support, from + [`freeze_demand_support`](@ref). +- `placement::AbstractDict`: the placement and capacity records, merged into the + battery artifact. +- `recourse`: `nothing` to derive prices from the case, or explicit + [`RecourseCosts`](@ref). +- `protocol_stages`, `protocol_scenarios`: the FINAL paired protocol the manifest + freezes a digest for. +- `screening_seed`, `screening_scenarios`: the INDEPENDENT screening protocol. + Leaving the seed `nothing` records no screening protocol, which is right for a + correctness case that has no selection decision to make. +- `mirror`: an Exa example directory to mirror the case and shared sources into. + +# Notes +Everything the two engines must agree on is decided HERE, once, and hashed. The +function returns the case as READ BACK from disk, not the objects it was handed, +so a case that cannot be re-read is a build failure rather than a later mystery. +""" +function build_case(source; + dir::AbstractString, + batteries::AbstractVector{BatterySpec}, + support::DemandSupport, + placement::AbstractDict = Dict{String,Any}(), + recourse::Union{Nothing,RecourseCosts} = nothing, + protocol_stages::Integer, + protocol_scenarios::Integer, + screening_seed::Union{Nothing,Integer} = nothing, + screening_scenarios::Integer = 0, + mirror::Union{Nothing,AbstractString} = nothing, + quiet::Bool = false) + prices = recourse === nothing ? recourse_prices(source.network) : recourse + record = Dict{String,Any}(placement) + record["versions"] = source.versions + + manifest = write_battery_case(dir; + name = source.name, + network = source.network, + batteries = batteries, + recourse = prices, + demand = support, + source_version = string(source.versions["PGLib"]), + placement = record, + protocol_stages = protocol_stages, + protocol_scenarios = protocol_scenarios, + screening_seed = screening_seed, + screening_scenarios = screening_scenarios) + + built = read_battery_case(dir) # read back through the verifier + if !quiet + print(describe(built)) + println("manifest artifacts:") + for (k, v) in sort!(collect(manifest["artifacts"]); by = first) + println(" ", rpad(k, 18), v) + end + println(" ", rpad("support", 18), manifest["support"]["sha256"]) + println(" ", rpad("protocol", 18), manifest["protocol"]["sha256"]) + manifest["screening"] === nothing || + println(" ", rpad("screening", 18), manifest["screening"]["sha256"]) + end + mirror === nothing || mirror_to_exa(dir, mirror; quiet = quiet) + return built +end + +""" + mirror_to_exa(case_dir, exa_example_dir; quiet=false) + +Copy the frozen case and the byte-shared source files into the ExaModels +example, then assert byte identity of everything copied. + +# Notes +The two engines are independent packages: neither may `include` a file from the +other. They are nonetheless required to agree exactly on the case contract and +on the solution schema, so those files are shipped as COPIES whose identity is +asserted here. The assertion is the point — a copy that has drifted is precisely +the failure this guards against. +""" +function mirror_to_exa(case_dir::AbstractString, exa_example_dir::AbstractString; + quiet::Bool = false) + isdir(exa_example_dir) || error("mirror target $exa_example_dir does not exist") + target_case = joinpath(exa_example_dir, "case", basename(case_dir)) + mkpath(target_case) + for f in ("network.json", "batteries.json", "demand.json", "case_manifest.json") + cp(joinpath(case_dir, f), joinpath(target_case, f); force = true) + sha256_file(joinpath(case_dir, f)) == sha256_file(joinpath(target_case, f)) || + error("mirrored case file $f is not byte-identical") + end + for f in ("battery_case.jl", "battery_solution_schema.jl") + src = joinpath(@__DIR__, f) + dst = joinpath(exa_example_dir, f) + cp(src, dst; force = true) + sha256_file(src) == sha256_file(dst) || + error("mirrored source file $f is not byte-identical") + end + quiet || println("mirrored case + shared sources into ", exa_example_dir) + return nothing +end + +""" + verify(dir) -> BatteryCase + +Re-verify a frozen case in place: hashes, schemas, the stage-duration contract, +the battery bound consistency checks, the support digest and the regenerated +protocol digest. +""" +function verify(dir::AbstractString = get(ENV, "DR_BAT_DIR", + joinpath(@__DIR__, "case", DEFAULT_CASE))) + case = read_battery_case(dir) + print(describe(case)) + println("case verified: ", dir) + return case +end + +# ───────────────────────────────────────────────────────────────────────────── +# The correctness case +# +# These are the constants of the Phase-1 correctness case. They deliberately +# describe a SMALL, comfortably feasible system: the case exists to establish +# that the two engines agree and that the strict formulation is well posed, not +# to expose a scientific mechanism. The research case is constructed from the +# same API by `case_design.jl`, and is not this section's business. +# ───────────────────────────────────────────────────────────────────────────── + +"PGLib benchmark used by the correctness case." +const DEFAULT_CASE = get(ENV, "DR_BAT_CASE", "pglib_opf_case14_ieee") + +"Number of batteries placed when no explicit bus list is given." +const DEFAULT_NUM_BATTERIES = parse(Int, get(ENV, "DR_BAT_NUM", "3")) + +"Seed of the deterministic placement draw." +const DEFAULT_PLACEMENT_SEED = parse(Int, get(ENV, "DR_BAT_SEED", "20260804")) + +"Seed of the paired evaluation protocol." +const DEFAULT_PROTOCOL_SEED = parse(Int, get(ENV, "DR_BAT_PROTOCOL_SEED", "20260804")) + +"Frozen horizon of the correctness case." +const DEFAULT_HORIZON = parse(Int, get(ENV, "DR_BAT_HORIZON", "48")) + +"Shape and scenario dimensions the manifest freezes a protocol digest for." +const PROTOCOL_STAGES = parse(Int, get(ENV, "DR_BAT_PROTOCOL_STAGES", "48")) +const PROTOCOL_SCENARIOS = parse(Int, get(ENV, "DR_BAT_PROTOCOL_SCENARIOS", "200")) + +""" +Fraction of stored energy a battery retains across one one-hour stage. + +A physical 0.1 %/h standing loss. It is deliberately NOT 1: with ``\\alpha = 1`` +the multiplier transform that turns stage duals into the actor signal reduces to +a plain difference and a sign or scaling error in the ``\\alpha`` term would go +unnoticed by every test. +""" +const SELF_DISCHARGE = 0.999 + +""" +Degradation price charged on battery throughput, per pu·h of +``\\Delta t (p^{ch} + p^{dis})``. + +Small against the case's marginal generation cost — about 0.2 % of it — so it +does not distort dispatch, but STRICTLY positive, which is what makes +simultaneous charging and discharging strictly suboptimal in the continuous +relaxation instead of merely unattractive. +""" +const THROUGHPUT_COST = 5.0 + +""" + build(; case, dir, num_batteries, buses, seed, horizon, mirror) -> BatteryCase + +Build the correctness case: a system-wide three-atom demand support on a mild +diurnal profile, with batteries drawn from the load buses. + +# Notes +The demand sampler is written out in full rather than hidden behind a constant, +because this function is also the smallest complete EXAMPLE of the public +workflow: acquire, place, rate, freeze, build. +""" +function build(; case::AbstractString = DEFAULT_CASE, + dir::AbstractString = get(ENV, "DR_BAT_DIR", joinpath(@__DIR__, "case", DEFAULT_CASE)), + num_batteries::Integer = DEFAULT_NUM_BATTERIES, + buses = _env_buses(), + seed::Integer = DEFAULT_PLACEMENT_SEED, + horizon::Integer = DEFAULT_HORIZON, + mirror::Union{Nothing,AbstractString} = get(ENV, "DR_BAT_MIRROR", nothing), + quiet::Bool = false) + + source = acquire_pglib_case(case) + + # ── batteries ──────────────────────────────────────────────────────────── + strategy = buses === nothing ? SampledPlacement(num_batteries; seed = seed) : + ExplicitPlacement(buses) + chosen, placement_record = select_battery_buses(source.network, strategy) + total_load = sum(values(nominal_load_at_bus(source.network))) + fleet, capacity_record = battery_fleet(source.network, chosen; + power = 0.10 * total_load, + energy_hours = 2.0, + self_discharge = SELF_DISCHARGE, + throughput_cost = THROUGHPUT_COST, + initial_fraction = 0.5) + + # ── demand ─────────────────────────────────────────────────────────────── + # Three symmetric system-wide atoms with equal probability. Symmetry means + # the process has mean 1, so the deterministic shape alone describes expected + # demand and the uncertainty is a pure spread around it. Three atoms is small + # enough that an SDDP backward pass enumerates it exactly and large enough + # that a policy must hedge rather than track a single forecast. + sampler = SystemMultiplier(DiscreteNonParametric([0.95, 1.0, 1.05], + [1 / 3, 1 / 3, 1 / 3])) + support = freeze_demand_support(sampler, source.network, horizon; + seed = DEFAULT_PROTOCOL_SEED, + method = :exact, + profile = diurnal_profile(horizon), + protocol_seed = DEFAULT_PROTOCOL_SEED, + stage_hours = 1.0) + + return build_case(source; + dir = dir, + batteries = fleet, + support = support, + placement = Dict{String,Any}("buses" => placement_record, + "capacity" => capacity_record, + "capacity_rule" => + "power = 10% of nominal system load; energy = 2 h"), + protocol_stages = PROTOCOL_STAGES, + protocol_scenarios = PROTOCOL_SCENARIOS, + mirror = mirror, + quiet = quiet) +end + +""" + ensure_case(dir; kwargs...) -> BatteryCase + +Read the frozen case at `dir`, BUILDING it first if it is not there. + +# Notes +Cases are constructed, never committed: the artifacts are a pure function of this +file's builder and its recorded seeds. Anything that needs a case — a test, a +diagnostic, an engine — calls this rather than assuming someone checked one in. +""" +function ensure_case(dir::AbstractString = joinpath(@__DIR__, "case", DEFAULT_CASE); + quiet::Bool = true, kwargs...) + if isfile(joinpath(dir, "case_manifest.json")) + # A case directory left over from an earlier revision of the contract + # cannot be read, and the right response is to rebuild it rather than to + # stop: the artifacts are a pure function of the builder, so there is + # nothing in them to lose. A rebuild that ALSO fails is a real error and + # is not caught here. + try + return read_battery_case(dir) + catch e + quiet || @warn "rebuilding $dir: it could not be read" exception = e + rm(dir; recursive = true, force = true) + end + end + return build(; dir = dir, quiet = quiet, kwargs...) +end + +# ───────────────────────────────────────────────────────────────────────────── +# The research case +# +# Constructed from the same public API as the correctness case, with three +# differences, each of which was MEASURED rather than chosen: +# +# • the benchmark is a congested PGLib operating condition, selected because +# its SOC-WR relaxation misprices stored energy at a decision level; +# • the deterministic profile has a quiet window where charging is cheap and a +# stressed window where stored energy is valuable; +# • the uncertainty is a small JOINT REGIONAL support that stresses the +# constrained pocket and the rest of the system independently, so WHERE +# energy is stored matters and not only HOW MUCH. +# +# Every constant below is part of the frozen benchmark and travels into the +# manifest. The evidence that selected them is in the Phase-2 record. +# ───────────────────────────────────────────────────────────────────────────── + +""" + research_profile(horizon; low, high, peak_hour, period) -> Vector{Float64} + +The deterministic hourly profile of the research case. + +# Notes +A raised cosine on `[low, high]` rather than a mean-1 profile: the LEVEL matters +here, because the case is built on a congested operating condition where the +network's ability to deliver is the binding physics. The quiet hours sit low +enough that charging is cheap and uncongested; the peak sits at the highest level +the base ACP problem still serves with NO recourse under every atom, which is the +admissibility precondition of the whole strict-target construction. +""" +function research_profile(horizon::Integer; low::Real = 0.78, high::Real = 1.00, + peak_hour::Integer = 19, period::Integer = 24) + mid = (low + high) / 2 + amp = (high - low) / 2 + return [mid + amp * cos(2π * (t - peak_hour) / period) for t in 1:horizon] +end + +""" + regional_demand_sampler(network; pocket_buses, quiet_stages, horizon, + pocket_atoms, rest_atoms) -> DemandSampler + +The research case's authoring sampler: deterministic quiet hours, and a joint +two-region support in the stressed hours. + +# Arguments +- `pocket_buses`: the buses of the constrained region, measured from the nodal + prices and the binding limits of the base dispatch. + +# Keywords +- `quiet_stages`: a predicate `t -> Bool` marking the stages with no uncertainty. +- `pocket_atoms`, `rest_atoms`: the finite supports of the two regional + multipliers. + +# Notes +Two regions, two atoms each, is the smallest construction that makes the +LOCATION of stored energy a hedging decision rather than a bookkeeping detail: +the pocket and the rest move independently, so a policy that has put its energy +in the wrong place cannot move it in time. A system-wide sampler with the same +marginal spread would produce a purely temporal problem, and every battery in it +would be interchangeable. + +The quiet stages carry a DEGENERATE support (one atom, probability 1). That is +deliberate and is not the same as having no stages there: the quiet hours are +where the policy charges, and the value of doing so is exactly what the stressed +hours reveal. +""" +function regional_demand_sampler(network::AbstractDict; + pocket_buses, + quiet_stages, + horizon::Integer, + pocket_atoms = ([0.96, 1.04], [0.5, 0.5]), + rest_atoms = ([0.99, 1.01], [0.5, 0.5])) + meta = demand_meta(network) + pocket = Set(Int.(pocket_buses)) + pocket_loads = [meta.load_ids[j] for j in 1:meta.num_loads if meta.load_bus[j] in pocket] + rest_loads = [meta.load_ids[j] for j in 1:meta.num_loads if !(meta.load_bus[j] in pocket)] + isempty(pocket_loads) && error("regional_demand_sampler: the pocket contains no load") + isempty(rest_loads) && error("regional_demand_sampler: every load is in the pocket") + + stressed = GroupMultiplier([pocket_loads, rest_loads], + [DiscreteNonParametric(pocket_atoms...), + DiscreteNonParametric(rest_atoms...)]) + quiet = DeterministicMultiplier(1.0) + return StageMultiplier([quiet_stages(t) ? quiet : stressed for t in 1:horizon]) +end + +""" + nearest_bus_regions(network, centers) -> Vector{Vector{Int}} + +Partition every load bus among `centers` by electrical hop distance: each load +bus joins the center it is closest to over the branch graph. + +# Notes +Automatic, and it applies to any PGLib case and any set of centers. Passing the +battery buses as the centers gives each battery its OWN region, which is the +precondition for the demand process to say anything locational: if two batteries +sit in one region they see the same signal and one of them is redundant. + +Ties go to the lower-numbered center, so the partition is a deterministic +function of the network and the centers. Buses unreachable from every center +(an islanded component) join the first center; a PGLib case is connected, so +this is a guard rather than a case that arises. + +The manual path is to skip this and hand region load lists straight to +[`JointRegionMultiplier`](@ref) — which is what to do when a measured region +(nodal prices, binding limits) is wanted instead of a distance partition. +""" +function nearest_bus_regions(network::AbstractDict, centers) + ctr = Int.(collect(centers)) + isempty(ctr) && throw(ArgumentError("nearest_bus_regions: no centers given")) + + adj = Dict{Int,Vector{Int}}() + for (_, br) in network["branch"] + get(br, "br_status", 1) == 0 && continue + f, t = Int(br["f_bus"]), Int(br["t_bus"]) + push!(get!(adj, f, Int[]), t) + push!(get!(adj, t, Int[]), f) + end + + # One multi-source BFS rather than one BFS per center: the frontier carries + # its owner, so every bus is settled at its true nearest center in one sweep. + owner = Dict{Int,Int}() + for (i, c) in enumerate(ctr) + haskey(owner, c) || (owner[c] = i) + end + frontier = copy(ctr) + while !isempty(frontier) + nxt = Int[] + for b in frontier, nb in get(adj, b, Int[]) + haskey(owner, nb) && continue + owner[nb] = owner[b] + push!(nxt, nb) + end + frontier = nxt + end + + regions = [Int[] for _ in ctr] + for b in sort!(collect(keys(nominal_load_at_bus(network)))) + push!(regions[get(owner, b, 1)], b) + end + return regions +end + +""" + rotating_regime_sampler(network; centers, horizon, low, high, period, + peak_hour, stress_min, stress_max, joint_share, + spread, min_probability, groups, modes, + mode_probabilities) -> DemandSampler + +A demand process whose regional means, tail and inter-region CORRELATION all +change with the stage. + +# Keywords +- `centers`: the region centers, normally the battery buses. Ignored when + `groups` is given. +- `groups`: explicit region bus lists — the manual path past the automatic + [`nearest_bus_regions`](@ref) partition. +- `low`, `high`: the slack and stressed regional multipliers. +- `period`, `peak_hour`: the daily cycle the regime rotates on. +- `stress_min`, `stress_max`: total probability that SOME region is stressed, at + the trough and at the peak of the cycle. +- `joint_share`: at the cycle peak, the share of that stress probability going + to the all-regions-stressed mode. Below the peak it is scaled down, so heavy + hours are also the CORRELATED hours. +- `spread`: `0` makes every region equally likely to be the stressed one at + every stage; `1` makes the stressed region rotate sharply with the cycle. +- `min_probability`: modes below this at a stage are dropped and the rest + renormalized. +- `modes`, `mode_probabilities`: fully manual overrides — an explicit + `(num_regions, K)` mode matrix and a `t -> probabilities` callable. + +# Notes +The modes are the joint outcomes worth naming: everything slack, exactly one +region stressed (one mode per region), and everything stressed. So `R` regions +cost `R + 2` atoms rather than the `2^R` of independent regions. + +What makes the process hard to operate is that the three things a policy would +want to know move independently: + +- the MEAN moves, because the probability of any stress at all follows the daily + cycle between `stress_min` and `stress_max`; +- the TAIL moves, because the all-regions-stressed mode is scaled by the cycle + on top of that, so the heavy outcome is concentrated in a few hours; +- the CORRELATION moves, and it changes SIGN. When mass sits on the single-region + modes the regions are negatively correlated — one is stressed exactly when the + others are slack, and energy stored in the right place is worth much more than + the same energy stored elsewhere. When mass sits on the calm and all-stressed + modes they are positively correlated, and location buys nothing. + +A policy therefore cannot reduce the problem to one storage schedule copied +across sites, nor to a per-site schedule computed independently: the right +charge in region `r` depends on the phase of the cycle and on what the other +regions are expected to do. That is the multi-dimensional structure the study +needs, and it is also precisely what a perfect-foresight solution exploits — +knowing WHICH region gets stressed, not merely how much total demand arrives. + +`spread` is the knob that sets how much of that is locational: at `spread=0` the +identity of the stressed region is pure coin-flip noise with no time structure, +and the process degenerates to a temporal one. +""" +function rotating_regime_sampler(network::AbstractDict; + centers = Int[], + horizon::Integer, + low::Real = 0.85, + high::Real = 1.15, + period::Integer = 24, + peak_hour::Integer = 19, + stress_min::Real = 0.10, + stress_max::Real = 0.90, + joint_share::Real = 0.35, + spread::Real = 0.85, + min_probability::Real = 1e-3, + groups = nothing, + modes = nothing, + mode_probabilities = nothing) + region_buses = groups === nothing ? nearest_bus_regions(network, centers) : + [sort!(collect(Int.(g))) for g in groups] + R = length(region_buses) + R >= 2 || throw(ArgumentError("rotating_regime_sampler needs at least 2 regions")) + any(isempty, region_buses) && + throw(ArgumentError("rotating_regime_sampler: a region contains no load bus")) + + # Columns: calm, then one per region, then all-stressed. + M = modes === nothing ? + hcat(fill(Float64(low), R), + [[r == c ? Float64(high) : Float64(low) for r in 1:R] for c in 1:R]..., + fill(Float64(high), R)) : + (modes isa AbstractMatrix ? Float64.(Matrix(modes)) : + reduce(hcat, [Float64.(collect(m)) for m in modes])) + K = size(M, 2) + + "Mode probabilities at stage `t`: the cycle sets how much stress and how correlated." + function default_probs(t) + # Cycle intensity in [0, 1], peaking at `peak_hour`. + s = (1 + cos(2π * (t - peak_hour) / period)) / 2 + total = stress_min + (stress_max - stress_min) * s + joint = total * joint_share * s # tail concentrates at the peak + local_total = total - joint + # Which region is the stressed one rotates through the cycle: region r's + # turn is offset by r/R of a period. + w = [(1 - spread) + spread * + max(0.0, cos(2π * (t - peak_hour) / period - 2π * (r - 1) / R)) + for r in 1:R] + sw = sum(w) + sw > 0 || (w = fill(1.0, R); sw = R) + return vcat(1 - total, local_total .* (w ./ sw), joint) + end + + probs_at = mode_probabilities === nothing ? default_probs : mode_probabilities + + stages = Vector{DemandSampler}(undef, horizon) + for t in 1:horizon + p = Float64.(collect(probs_at(t))) + length(p) == K || error("mode probabilities at stage $t have length $(length(p)), expected $K") + # Strictly positive as well as above the threshold: at the trough of the + # cycle the all-stressed mode has probability exactly zero, and a + # zero-probability atom is not a mode of the law, it is an absent one. + keep = [k for k in 1:K if p[k] >= min_probability && p[k] > 0] + isempty(keep) && (keep = [argmax(p)]) + q = p[keep] ./ sum(p[keep]) + # A single surviving mode stays a JointRegionMultiplier carrying one + # atom: it keys by BUS, where DeterministicMultiplier's dict form keys + # by LOAD, and mixing the two keyings is a silent way to get it wrong. + stages[t] = JointRegionMultiplier(region_buses, M[:, keep], q; by = :bus) + end + return StageMultiplier(stages) +end + +""" + build_research_case(; case, dir, pocket_buses, battery_buses, horizon, + report_stages, power_fraction, energy_hours, + profile_low, profile_high, pocket_atoms, rest_atoms, + protocol_scenarios, mirror) -> BatteryCase + +Build a multiperiod research case from the public API. + +# Keywords +- `case::AbstractString`: PGLib benchmark, including its `__api` / `__sad` suffix. +- `pocket_buses`: the constrained region, from the base-dispatch measurement. +- `battery_buses`: where the batteries go, from the storage-value measurement. +- `horizon::Integer`: stages FROZEN, i.e. reported window plus look-ahead tail. +- `report_stages::Integer`: the reported window, recorded in the manifest. +- `power_fraction::Real`: each battery's power rating as a fraction of nominal + active load — of the WHOLE SYSTEM under `power_basis = :system`, or of the + battery's OWN region under `:region`; `energy_hours` its duration. +- `power_basis::Symbol`: `:system` or `:region`. Use `:region` on a large network + with concentrated regions, where a system-wide fraction can make one battery + large enough to dominate its own neighbourhood. +- `profile_low`, `profile_high`: the deterministic profile's band. +- `pocket_atoms`, `rest_atoms`: `(values, probabilities)` of the two regional + multipliers in the stressed hours. `:pocket` regime only. +- `demand_regime`: `:pocket` for the two-region product process, or `:rotating` + for the time-varying joint process of [`rotating_regime_sampler`](@ref), whose + regions carry independent means, tails and a correlation that changes sign + through the cycle. +- `region_centers`: the `:rotating` regions' centers; defaults to the battery + buses, which is what gives each battery a region of its own. +- `region_groups`: explicit region bus lists — the manual path past the + automatic partition. +- `regime`: a NamedTuple of [`rotating_regime_sampler`](@ref) keywords + (`low`, `high`, `stress_min`, `stress_max`, `joint_share`, `spread`, …). + +# Notes +Everything the benchmark IS is an argument here, and every argument lands in the +manifest, so the frozen case can be rebuilt from the recorded values alone. + +**Horizon and tail.** The reported window is a whole number of daily cycles, so a +storage decision taken in it is completed inside it. The tail exists for one +reason: with a finite horizon and no terminal value, the optimal thing to do in +the last stages is to empty every battery, and if the reported window ended at +the horizon that dump would be a large part of the reported cost. Reporting a +prefix and letting the tail absorb the terminal effect is what keeps the +comparison about the policy rather than about the boundary condition. + +**Quiet hours.** Stages in the first half of each daily cycle carry a degenerate +one-atom support. That is where charging happens, and it is deterministic on +purpose: the study is about valuing STORED energy under uncertainty about when it +will be needed, not about a policy's ability to forecast the hour it charges in. +""" +function build_research_case(; case::AbstractString, + dir::AbstractString, + pocket_buses, + battery_buses, + horizon::Integer = 36, + report_stages::Integer = 24, + power_fraction::Real = 0.04, + energy_hours::Real = 2.0, + profile_low::Real = 0.78, + profile_high::Real = 1.00, + pocket_atoms = ([0.96, 1.04], [0.5, 0.5]), + rest_atoms = ([0.99, 1.01], [0.5, 0.5]), + pocket_fraction::Real = 0.0, + demand_regime::Symbol = :pocket, + power_basis::Symbol = :system, + region_centers = Int[], + region_groups = nothing, + regime = NamedTuple(), + self_discharge::Real = SELF_DISCHARGE, + throughput_cost::Real = THROUGHPUT_COST, + initial_fraction::Real = 0.5, + reserve_fraction::Real = 0.0, + protocol_seed::Integer = DEFAULT_PROTOCOL_SEED, + protocol_scenarios::Integer = 500, + screening_seed::Integer = DEFAULT_PROTOCOL_SEED + 1, + screening_scenarios::Integer = 64, + recourse_factor::Real = 50, + quiet_stages = t -> mod1(t, 24) <= 12, + mirror::Union{Nothing,AbstractString} = nothing, + quiet::Bool = false) + source = acquire_pglib_case(case) + + # A pocket given as a FRACTION selects that share of the load buses, largest + # demand first. It is the automatic path; an explicit `pocket_buses` list is + # the manual one, for when a measured region is wanted instead. + pocket = pocket_fraction > 0 ? + (l = nominal_load_at_bus(source.network); + r = sort!(collect(keys(l)); by = b -> -max(0.0, l[b])); + r[1:max(1, round(Int, pocket_fraction * length(r)))]) : pocket_buses + + buses, placement_record = select_battery_buses(source.network, + ExplicitPlacement(battery_buses)) + load_at = nominal_load_at_bus(source.network) + total_load = sum(max(0.0, v) for v in values(load_at)) + + # `:system` rates every battery at a fraction of TOTAL system load; `:region` + # rates it at a fraction of its OWN region's load. + # + # `:system` is a trap on a large network with concentrated regions. On + # case300 a 0.04 system fraction is 9.54 pu, and the region around bus 138 + # holds 24 pu of load — so charging that battery is a 40 % local demand + # spike, and there are states where it is the only thing between the region + # and a shortfall. The dual of stored energy is then the RECOURSE price + # rather than an energy price, and SDDP builds cuts whose coefficients span + # ten orders of magnitude, which is what makes later subproblems report false + # infeasibility. Sizing against the battery's own region keeps the machine + # proportionate to the load it serves at every site. + power_rule = if power_basis === :region + regions = region_groups === nothing ? + nearest_bus_regions(source.network, + isempty(region_centers) ? buses : region_centers) : + [sort!(collect(Int.(g))) for g in region_groups] + region_load = Dict{Int,Float64}() + for (i, r) in enumerate(regions) + l = sum(max(0.0, get(load_at, b, 0.0)) for b in r) + for b in r + region_load[b] = l + end + i <= length(buses) || continue + end + bus -> power_fraction * get(region_load, bus, total_load) + elseif power_basis === :system + power_fraction * total_load + else + throw(ArgumentError("power_basis must be :system or :region, got :$power_basis")) + end + + fleet, capacity_record = battery_fleet(source.network, buses; + power = power_rule, + energy_hours = energy_hours, + self_discharge = self_discharge, + throughput_cost = throughput_cost, + initial_fraction = initial_fraction, + reserve_fraction = reserve_fraction) + + # `:pocket` is the two-region product process; `:rotating` is the + # time-varying joint one, whose regions are the batteries' own neighbourhoods. + sampler = if demand_regime === :rotating + rotating_regime_sampler(source.network; + centers = isempty(region_centers) ? buses : region_centers, + horizon = horizon, + groups = region_groups, + regime...) + elseif demand_regime === :pocket + regional_demand_sampler(source.network; + pocket_buses = pocket, + quiet_stages = quiet_stages, + horizon = horizon, + pocket_atoms = pocket_atoms, + rest_atoms = rest_atoms) + else + throw(ArgumentError("demand_regime must be :pocket or :rotating, got :$demand_regime")) + end + support = freeze_demand_support(sampler, source.network, horizon; + seed = protocol_seed, + method = :exact, + profile = research_profile(horizon; + low = profile_low, + high = profile_high), + profile_period = 24, + protocol_seed = protocol_seed, + stage_hours = 1.0) + + return build_case(source; + dir = dir, + batteries = fleet, + support = support, + recourse = recourse_prices(source.network; factor = recourse_factor), + placement = Dict{String,Any}( + "recourse_factor" => Float64(recourse_factor), + "buses" => placement_record, + "capacity" => capacity_record, + "capacity_rule" => "power = $(power_fraction) × nominal $(power_basis === :region ? "region" : "system") load; energy = $(energy_hours) h", + "power_basis" => String(power_basis), + "pocket_buses" => sort!(collect(Int.(pocket))), + "pocket_fraction" => Float64(pocket_fraction), + "profile" => Dict{String,Any}("low" => Float64(profile_low), + "high" => Float64(profile_high), + "peak_hour" => 19, + "period" => 24), + "pocket_atoms" => Dict{String,Any}("values" => Float64.(pocket_atoms[1]), + "probabilities" => Float64.(pocket_atoms[2])), + "rest_atoms" => Dict{String,Any}("values" => Float64.(rest_atoms[1]), + "probabilities" => Float64.(rest_atoms[2])), + "demand_regime" => String(demand_regime), + "regime" => Dict{String,Any}(String(k) => v for (k, v) in pairs(regime)), + "sampler" => sampler_to_dict(sampler), + "report_stages" => Int(report_stages), + "lookahead_stages" => Int(horizon) - Int(report_stages)), + protocol_stages = Int(horizon), + protocol_scenarios = Int(protocol_scenarios), + screening_seed = Int(screening_seed), + screening_scenarios = Int(screening_scenarios), + mirror = mirror, + quiet = quiet) +end + +"Explicit battery buses from `DR_BAT_BUSES`, or `nothing` for the seeded draw." +function _env_buses() + raw = get(ENV, "DR_BAT_BUSES", "") + isempty(strip(raw)) && return nothing + return [parse(Int, strip(x)) for x in split(raw, ",") if !isempty(strip(x))] +end + +# ───────────────────────────────────────────────────────────────────────────── +# Engineered relaxation-gap modules (Phase 3A) +# ───────────────────────────────────────────────────────────────────────────── + +""" + CycleModuleSpec(; kwargs...) + +The parameters of one meshed cycle module. Every field is recorded in the +manifest and enters the case digest, so a module is reproducible from numbers +alone. + +# Fields +- `load_p`, `load_q`: the remote bus's demand, pu. +- `local_pmax`, `local_cost`: the remote generator's capacity and LINEAR cost. + `pmin` is zero, so it is a convex, always-feasible fallback and never a + commitment decision. +- `local_qmin`, `local_qmax`: its reactive band, sized so reactive feasibility + at the remote bus is never the binding physics. +- `r_hm, x_hm, r_mr, x_mr, r_hr, x_hr`: the three cycle branches' impedances. +- `rate_hm, rate_mr, rate_hr`: their apparent-power ratings. +- `vmin`, `vmax`: voltage band of the two new buses. +- `base_kv`: carried from the host bus when zero. + +# Notes +The module exists to make ONE omitted condition matter. `SOCWRConicPowerModel` +constrains each branch's voltage-product block by +`wr^2 + wi^2 <= w_fr * w_to` but never requires the recovered angle differences +to sum to zero around a cycle, so on a loop it admits `W` matrices that no real +voltage profile realises. On a radial network the condition is vacuous, which is +why the relaxation is exact there. + +The amplifier is loop RESISTANCE. Around a cycle the relaxation can settle on a +`W` whose implied flows do not pay the full `I^2 R` a consistent voltage profile +would, so SOC systematically believes the remote bus can be served more cheaply +through the loop than AC can actually serve it. The expensive local generator +turns that belief into money: where SOC imports cheap power around the cycle, +AC must dispatch the local unit instead. + +This is a hypothesis about mechanism, and the point of the Phase 3A gate is to +measure whether it produces a different STORAGE ACTION rather than merely a +different objective level. A relaxation gap in cost alone changes nothing about +what a policy should do. +""" +Base.@kwdef struct CycleModuleSpec + load_p::Float64 = 0.60 + load_q::Float64 = 0.20 + local_pmax::Float64 = 1.20 + local_cost::Float64 = 260.0 + local_qmin::Float64 = -0.80 + local_qmax::Float64 = 0.80 + r_hm::Float64 = 0.030 + x_hm::Float64 = 0.030 + r_mr::Float64 = 0.030 + x_mr::Float64 = 0.030 + r_hr::Float64 = 0.010 + x_hr::Float64 = 0.060 + rate_hm::Float64 = 0.50 + rate_mr::Float64 = 0.50 + rate_hr::Float64 = 0.50 + vmin::Float64 = 0.94 + vmax::Float64 = 1.06 + base_kv::Float64 = 0.0 +end + +""" + attach_cycle_module!(network, host_bus, spec; tag) -> NamedTuple + +Attach one triangular meshed module to `host_bus` IN PLACE, and report every +identifier and parameter it added. + +# Arguments +- `network`: a parsed PowerModels network, mutated in place. +- `host_bus`: any existing in-service bus. The module is generic; nothing here + is specific to one PGLib case. +- `spec::CycleModuleSpec`: the parameters. + +# Returns +A `NamedTuple` with the new `mid_bus`, `remote_bus`, the three `branches`, the +`gen` and `load` identifiers, and `record`, a plain dictionary of everything +added — which is what travels into the manifest and the digest. + +# Notes +Identifiers are allocated above the current maximum of EVERY table, so they +cannot collide with the backbone's own nonconsecutive numbering. Units are +untouched: the network stays in whatever per-unit convention it arrived in, and +`base_kv` is inherited from the host bus unless overridden. + +The topology is deliberately the smallest object that carries a cycle: host, +one intermediate bus, one remote bus, three branches. The battery and the +uncertainty attach at the remote bus, so the storage state sits BEHIND the loop +and its value depends on how the loop is modelled — which is the whole point. +""" +function attach_cycle_module!(network::AbstractDict, host_bus::Integer, + spec::CycleModuleSpec = CycleModuleSpec(); + tag::AbstractString = "cyc") + haskey(network["bus"], string(host_bus)) || + throw(ArgumentError("host bus $host_bus is not in the network")) + host = network["bus"][string(host_bus)] + Int(get(host, "bus_type", 1)) == 4 && + throw(ArgumentError("host bus $host_bus is out of service")) + + nextid(tbl) = isempty(get(network, tbl, Dict())) ? 1 : + maximum(parse(Int, k) for k in keys(network[tbl])) + 1 + bkv = spec.base_kv > 0 ? spec.base_kv : Float64(get(host, "base_kv", 1.0)) + + mid = nextid("bus") + rem = mid + 1 + for (id, nm) in ((mid, "$(tag)_mid"), (rem, "$(tag)_remote")) + network["bus"][string(id)] = Dict{String,Any}( + "index" => id, "bus_i" => id, "bus_type" => 1, + "vmin" => spec.vmin, "vmax" => spec.vmax, + "vm" => 1.0, "va" => 0.0, "base_kv" => bkv, + "zone" => Int(get(host, "zone", 1)), "area" => Int(get(host, "area", 1)), + "name" => nm, "source_id" => Any["bus", id]) + end + + b1, b2, b3 = nextid("branch"), nextid("branch") + 1, nextid("branch") + 2 + function addbranch!(id, f, t, r, x, rate) + network["branch"][string(id)] = Dict{String,Any}( + "index" => id, "f_bus" => f, "t_bus" => t, + "br_r" => r, "br_x" => x, + "g_fr" => 0.0, "b_fr" => 0.0, "g_to" => 0.0, "b_to" => 0.0, + "tap" => 1.0, "shift" => 0.0, "br_status" => 1, + "angmin" => -pi / 3, "angmax" => pi / 3, + "rate_a" => rate, "rate_b" => rate, "rate_c" => rate, + "transformer" => false, "source_id" => Any["branch", id]) + end + addbranch!(b1, Int(host_bus), mid, spec.r_hm, spec.x_hm, spec.rate_hm) + addbranch!(b2, mid, rem, spec.r_mr, spec.x_mr, spec.rate_mr) + addbranch!(b3, Int(host_bus), rem, spec.r_hr, spec.x_hr, spec.rate_hr) + + gid = nextid("gen") + network["gen"][string(gid)] = Dict{String,Any}( + "index" => gid, "gen_bus" => rem, + "pg" => 0.0, "qg" => 0.0, + "pmin" => 0.0, "pmax" => spec.local_pmax, + "qmin" => spec.local_qmin, "qmax" => spec.local_qmax, + "vg" => 1.0, "mbase" => Float64(network["baseMVA"]), "gen_status" => 1, + # Linear and convex on purpose: an expensive fallback, never a + # commitment decision, so nothing here makes the problem nonconvex. + "model" => 2, "ncost" => 2, "cost" => [spec.local_cost, 0.0], + "startup" => 0.0, "shutdown" => 0.0, + "source_id" => Any["gen", gid]) + + lid = nextid("load") + network["load"][string(lid)] = Dict{String,Any}( + "index" => lid, "load_bus" => rem, + "pd" => spec.load_p, "qd" => spec.load_q, "status" => 1, + "source_id" => Any["load", lid]) + + record = Dict{String,Any}( + "tag" => String(tag), "host_bus" => Int(host_bus), + "mid_bus" => mid, "remote_bus" => rem, + "branch_hm" => b1, "branch_mr" => b2, "branch_hr" => b3, + "gen" => gid, "load" => lid, "base_kv" => bkv, + "spec" => Dict{String,Any}(string(f) => getfield(spec, f) + for f in fieldnames(CycleModuleSpec))) + return (mid_bus = mid, remote_bus = rem, branches = (b1, b2, b3), + gen = gid, load = lid, record = record) +end + +""" + attach_scheduled_generator!(network, bus; pmax, cost, pmin=0.0, qmin=0.0, + qmax=0.0, availability=Float64[], tag="sched") + -> NamedTuple + +Attach one dispatchable generator to an existing bus IN PLACE, optionally +available in only some stages, and report everything it added. + +# Arguments +- `network`: a parsed PowerModels network, mutated in place. +- `bus`: any existing in-service bus. + +# Keywords +- `pmax::Real`, `pmin::Real`: active limits (pu). +- `cost`: polynomial coefficients HIGHEST ORDER FIRST, PowerModels' own + convention — `[c2, c1, c0]` for `c2 pg^2 + c1 pg + c0`. +- `qmin::Real`, `qmax::Real`: reactive limits (pu). Both default to zero, which + is a unit that supplies active power only and leaves the reactive picture of + the case exactly as it was. +- `availability`: per-stage multiplier vector written to + [`STAGE_AVAILABILITY_KEY`](@ref); empty means available in every stage. +- `tag::AbstractString`: recorded, and used in the generator's name. + +# Returns +A `NamedTuple` with the new `gen` identifier and `record`, a plain dictionary of +every parameter — which is what travels into the placement artifact and is +therefore hashed with the case. + +# Notes +Generic: any parsed PGLib case, any bus, and every value is an argument, so a +specific unit can be constructed by hand exactly as an automatic placement rule +would construct it. The identifier is allocated above the current maximum of the +generator table, so it cannot collide with a backbone's nonconsecutive numbering. + +The cost is required to be CONVEX and nondecreasing over the unit's own operating +interval — a negative quadratic coefficient, or a marginal cost that goes +negative inside `[pmin, pmax]`, would make a stage problem either nonconvex in +its relaxed form or paid to generate, and both would be found much later as a +strange dispatch rather than here as a rejected case. +""" +function attach_scheduled_generator!(network::AbstractDict, bus::Integer; + pmax::Real, + cost::AbstractVector, + pmin::Real = 0.0, + qmin::Real = 0.0, + qmax::Real = 0.0, + availability::AbstractVector = Float64[], + tag::AbstractString = "sched") + haskey(network["bus"], string(bus)) || + throw(ArgumentError("bus $bus is not in the network")) + Int(get(network["bus"][string(bus)], "bus_type", 1)) == 4 && + throw(ArgumentError("bus $bus is out of service")) + 0 <= pmin <= pmax || throw(ArgumentError("need 0 <= pmin <= pmax, got [$pmin, $pmax]")) + pmax > 0 || throw(ArgumentError("pmax must be strictly positive, got $pmax")) + qmin <= qmax || throw(ArgumentError("need qmin <= qmax, got [$qmin, $qmax]")) + c = Float64.(collect(cost)) + (!isempty(c) && all(isfinite, c)) || + throw(ArgumentError("cost must be a non-empty vector of finite coefficients")) + length(c) <= 3 || + throw(ArgumentError("only constant, linear and quadratic costs are supported, got $(length(c)) coefficients")) + # Highest order first: the quadratic coefficient is c[end-2] when present. + c2 = length(c) >= 3 ? c[end - 2] : 0.0 + c1 = length(c) >= 2 ? c[end - 1] : 0.0 + c2 >= 0 || throw(ArgumentError("quadratic cost coefficient $c2 is negative, so the cost is not convex")) + for p in (Float64(pmin), Float64(pmax)) + 2 * c2 * p + c1 >= 0 || + throw(ArgumentError("marginal cost $(2 * c2 * p + c1) at pg = $p is negative")) + end + av = Float64.(collect(availability)) + all(x -> isfinite(x) && x >= 0, av) || + throw(ArgumentError("availability multipliers must be finite and nonnegative, got $av")) + + gid = isempty(get(network, "gen", Dict())) ? 1 : + maximum(parse(Int, k) for k in keys(network["gen"])) + 1 + gen = Dict{String,Any}( + "index" => gid, "gen_bus" => Int(bus), + "pg" => 0.0, "qg" => 0.0, + "pmin" => Float64(pmin), "pmax" => Float64(pmax), + "qmin" => Float64(qmin), "qmax" => Float64(qmax), + "vg" => 1.0, "mbase" => Float64(network["baseMVA"]), "gen_status" => 1, + "model" => 2, "ncost" => length(c), "cost" => c, + "startup" => 0.0, "shutdown" => 0.0, + "name" => "$(tag)_gen", "source_id" => Any["gen", gid]) + isempty(av) || (gen[STAGE_AVAILABILITY_KEY] = av) + network["gen"][string(gid)] = gen + + record = Dict{String,Any}( + "tag" => String(tag), "gen" => gid, "bus" => Int(bus), + "pmin" => Float64(pmin), "pmax" => Float64(pmax), + "qmin" => Float64(qmin), "qmax" => Float64(qmax), + "cost" => c, "availability" => av) + return (gen = gid, record = record) +end + +""" + module_digest(backbone_hash, seed, records) -> String + +A deterministic digest over the backbone, the seed and every module parameter. + +# Notes +The generated case is never committed, so the digest is what makes a run +identifiable: two runs agree if and only if they built the same network from the +same backbone bytes. +""" +function module_digest(backbone_hash::AbstractString, seed::Integer, records) + io = IOBuffer() + print(io, backbone_hash, "|", seed) + for r in records + print(io, "|", r["tag"], ":", r["host_bus"], ":", r["mid_bus"], ":", r["remote_bus"]) + for k in sort!(collect(keys(r["spec"]))) + print(io, ",", k, "=", r["spec"][k]) + end + end + return bytes2hex(SHA.sha256(take!(io)))[1:16] +end + +if abspath(PROGRAM_FILE) == @__FILE__ + if "--verify" in ARGS + verify() + else + build() + end +end diff --git a/examples/BatteryStorageOPF/demand_variants.toml b/examples/BatteryStorageOPF/demand_variants.toml new file mode 100644 index 0000000..8265868 --- /dev/null +++ b/examples/BatteryStorageOPF/demand_variants.toml @@ -0,0 +1,85 @@ +# Regional demand variants of the frozen battery portfolio. +# +# A declarative recipe, read by `demand_variant_recipe` in battery_portfolio.jl. +# Every choice that determines a variant's finite demand support is written here +# or named here by rule; the code only evaluates the rules. The reference support +# (`portfolio_support`, the one recorded in battery_portfolio.json) is NOT +# described or changed by this file. +# +# Notation (all quantities per host, from the host's own frozen portfolio case): +# R1..R6 the portfolio regions in manifest order (descending nominal demand) +# D_r nominal active demand of region r (pu), summed over its buses +# PROF_t PORTFOLIO_PROFILE, t = 1..24; kappa = the frozen kappa_case +# +# Realized demand of load j at a bus of region r, stage t, atom k: +# p_d = p0_j * kappa * PROF_t * m[r,t,k] (q_d with the same factor) +# m[r,t,k] = (1 + a[p,t]) * (1 + sigma_r * u_r * s[p,k,t]) +# where p is the group of r. Loads at buses in no region keep multiplier 1. + +schema = "battery_storage_opf/demand_variants/1" +recipe_version = 1 +# The reference manifest these variants are built on (battery_portfolio.json). +reference_manifest_digest = "11c2f58fabbac489812d86fa2aef95833c54acaac12347337e54940a29ccb1d9" + +[groups] +# Regions are paired by rank: largest with smallest, and so on. +# pairs[p] = [first, partner]; sigma = +1 for first, -1 for partner. +rule = "rank_pairs" +pairs = [[1, 6], [2, 5], [3, 4]] + +[means] +# a[p,t] = amplitude * ( cos(theta_t - phi_p) - beta_p ), +# theta_t = 2*pi*(t - anchor_stage)/period. +form = "centered_cosine" +period = 24 +anchor_stage = 12 +# phi_1 = 0; phi_2 = pi - C with cos C = (G1^2 + G2^2 - G3^2)/(2 G1 G2); +# phi_3 = arg(-G1 - G2*exp(i*phi_2)); G_p = D_first + D_partner. +# Makes sum_p G_p exp(i phi_p) = 0 (stage totals preserved). Requires +# max_p G_p < sum_p G_p / 2; otherwise the recipe fails for that host. +phase_rule = "weighted_phasor_triangle" +# beta_p = sum_t PROF_t cos(theta_t - phi_p) / sum_t PROF_t +# (each region's daily expected energy preserved). +centering = "profile_weighted_daily" +# Tried in this order; the first admissible one is used (see [hosts]). +amplitude_candidates = [0.15, 0.10, 0.05] + +[shocks] +# delta^2 = the reference per-region variance, computed from the reference +# constants: (1/n)(HIGH-1)^2 + ((n-1)/n)(LOW-1)^2 with n = PORTFOLIO_REGIONS. +scale = "reference_region_variance" +# u_r = delta * sqrt(D_partner / D_r): partners move equal and opposite MW, +# and sum_r (D_r / sum D) u_r^2 = delta^2. +allocation = "partner_equal_mw" +probability = "uniform" + +[variants.A] +# Group sign patterns s[p,k] for atoms k = 1..4 (rows), groups p = 1..3 (columns). +# The same at every stage. +signs = [[1, 1, 1], [1, -1, -1], [-1, 1, -1], [-1, -1, 1]] + +[variants.B] +# Atom k of B equals atom k of A with group `overwrite`'s sign replaced by group +# `keep`'s sign, inside each stage block. Means and shock scales are A's. +base = "A" + +[[variants.B.blocks]] +first = 1 +last = 8 +keep = 3 +overwrite = 1 + +[[variants.B.blocks]] +first = 9 +last = 16 +keep = 1 +overwrite = 2 + +[[variants.B.blocks]] +first = 17 +last = 24 +keep = 2 +overwrite = 3 + +[hosts] +# Calibrated amplitude per host, added only after a recorded calibration. diff --git a/examples/BatteryStorageOPF/portfolio_runner.jl b/examples/BatteryStorageOPF/portfolio_runner.jl new file mode 100644 index 0000000..6cfc22b --- /dev/null +++ b/examples/BatteryStorageOPF/portfolio_runner.jl @@ -0,0 +1,1078 @@ +#!/usr/bin/env julia +# portfolio_runner.jl +# +# The production runner for the two methods THIS engine owns: `:sddp_soc` and +# `:sddp_dc`. The other two study methods, `:tsddr_nonlinear` and +# `:tsldr_recurrent_linear`, belong to the Exa engine +# (`DecisionRulesExa.jl/examples/BatteryStorageOPF/portfolio_runner.jl`) and are +# refused here BY NAME, exactly as `run_battery_method` refuses them. +# +# WHAT THIS FILE IS, AND WHAT IT IS NOT. +# It is a SEGMENT DRIVER: it turns "train this frozen case, with this frozen +# configuration, from global iteration `a` to global iteration `b`, and survive +# being killed at any moment" into files on disk. It builds no policy graph +# equation, writes no cut, chooses no duality handler and defines no cost — every +# one of those comes from `battery_sddp.jl`, `battery_powermodels.jl` and the two +# shared contract files, unchanged, and through SDDP.jl's own stock mechanisms. +# What it adds is identity binding, verified checkpoints, resumption, a stop +# protocol and an honest result record. +# +# ───────────────────────────────────────────────────────────────────────────── +# COMMAND CONTRACT +# +# julia --project= portfolio_runner.jl \ +# --case-manifest \ +# --method \ +# --config \ +# --protocol \ +# --output \ +# --resume-from +# +# Those six flags are sufficient on their own; the command above runs with no +# scheduler and no campaign controller of any kind. Five further flags exist +# purely as conveniences for an automated caller and ALL of them default: +# +# --run-id (default "standalone") +# --segment (default 1) +# --attempt (default 1) +# --stop-file (default /STOP) +# --max-seconds (default 1e9) +# --final-evaluation (default false) measure the panel once at the last +# completed index, for a run being retired on budget +# +# ───────────────────────────────────────────────────────────────────────────── +# THE FROZEN CONFIGURATION (`--config`, TOML) +# +# target_index total SDDP iterations for the WHOLE run +# segment_updates iterations this invocation may add at most +# checkpoint_every iterations between checkpoints — also the CHUNK size, see +# "continuation is the only path" below +# eval_every iterations between true-ACP evaluations on the fixed panel +# ma_window window of the reported moving average of the forward cost +# num_stages horizon T +# eval_columns screening-protocol columns forming the fixed panel +# seed the run's single seed +# max_recourse physical admissibility tolerance, pu +# method OPTIONAL; when present it must equal --method +# +# ───────────────────────────────────────────────────────────────────────────── +# THE PROTOCOL DESCRIPTOR (`--protocol`, TOML) — see `resolve_protocol` +# +# kind "screening" (or "sole" on a fixture case that declares only +# one protocol). "final" is REFUSED. +# num_stages, num_scenarios, seed, sha256 +# +# Write one for a case with: +# +# julia --project=. -e 'include("portfolio_runner.jl"); +# write_protocol_descriptor("case/pglib_opf_case118_ieee", "screening.toml")' +# +# ───────────────────────────────────────────────────────────────────────────── +# CONTINUATION IS THE ONLY PATH +# +# This runner trains in CHUNKS of `checkpoint_every` iterations. Every chunk +# rebuilds both policy graphs from the frozen case and restores the previous +# chunk's cuts through `SDDP.read_cuts_from_file`, whether or not the process was +# ever interrupted. So the code path a resumed run takes is not "the same as" +# the uninterrupted one — it IS the uninterrupted one, and a run cut at any +# checkpoint boundary reproduces the uncut run structurally rather than by +# careful arrangement. The price is a graph rebuild per checkpoint, which is +# small beside `checkpoint_every` ACP forward passes and buys the property this +# campaign's whole recovery design rests on. +# +# Sampling is made a function of the GLOBAL iteration index the same way: each +# chunk's seed is derived from `(seed, iterations already completed)`, so the +# scenarios of iterations 4–6 are the same whether they are the tail of one run +# of six or the whole of a resumed run of three. +# +# No solver object is serialized anywhere, and no SDDP internal is parsed by +# hand: cuts leave and re-enter through SDDP.jl's own reader and writer. +# +# ───────────────────────────────────────────────────────────────────────────── +# OUTPUTS, all inside `--output`, none of them ever committed +# +# checkpoints/ck_XXXXXXXX..json the checkpoint payload +# checkpoints/ck_XXXXXXXX..json.meta.toml digest, indices, lineage +# history.csv index, train_loss, panel_value, bound, solve_ok, +# solve_fail, deficit (7 columns, fixed) +# trajectory.csv index, forward_cost, forward_cost_ma, bound, wall_seconds +# evaluation.csv index, protocol, columns, acp_mean, acp_raw_mean, +# worst_recourse, complete, selected, best_cost +# result.toml the segment record +# identity.toml every coordinate this segment was bound to +# +# `acp_mean` is the CORRECTED physical cost under the shared cost contract, +# validated and projected PER BUS before anything is summed — the same estimand +# the Exa engine selects on. `acp_raw_mean` is the solver's raw objective sum, +# a diagnostic only. +# +# `train_loss` is SDDP's own per-iteration forward-pass cost — the ACP cost of +# ONE sampled scenario, a per-iteration sample of a random objective. It is not +# comparable with the fixed-panel evaluation, which is a mean over the same +# columns every time, and the two live in different columns for that reason. +# +# THE `bound` COLUMN IS ARM-DEPENDENT AND IS NOT ALWAYS A BOUND. For `:sddp_soc` +# it is the SOC-WR relaxation bound and it does bound the true ACP problem; for +# `:sddp_dc` it is the DC-approximation training bound and it bounds NOTHING +# about ACP in either direction. `bound_name` and `bound_bounds_acp` travel in +# the result and in every checkpoint so no consumer has to infer which one it is +# holding. + +using TOML +using SHA +using Dates +using Printf +using Random +using Statistics +using JSON +using SDDP + +# The certified implementation. Everything scientific comes from here; this file +# adds no second copy of any of it. Its `PROGRAM_FILE` guard keeps the include +# from launching the construction smoke. +include(joinpath(@__DIR__, "battery_sddp.jl")) + +# ───────────────────────────────────────────────────────────────────────────── +# Schemas +# ───────────────────────────────────────────────────────────────────────────── + +""" +Result-record schema. Must match the number the campaign controller verifies; +a runner speaking a different one is rejected rather than half-read. +""" +const RUNNER_RESULT_SCHEMA = 1 + +""" +Segment-checkpoint schema. + +The same tag the Exa runner writes, because the two produce the same KIND of +object — a self-contained continuation point for one segmented run — even though +their payloads are a policy and a cut set. +""" +const RUNNER_CHECKPOINT_SCHEMA = "battery_storage_opf/segment/1" + +"The two methods this engine owns, and the engine that owns the other two." +const RUNNER_METHODS = (:sddp_soc, :sddp_dc) + +# ───────────────────────────────────────────────────────────────────────────── +# Small self-contained primitives +# +# Deliberately reimplemented here rather than shared with any caller: this file +# must run standalone, from a public checkout, with no orchestration package on +# the load path. `sha256_file` is the one exception — it already exists in +# `battery_case.jl`, which this file includes, and defining a second one would +# leave two digest functions that could drift apart. +# ───────────────────────────────────────────────────────────────────────────── + +"UTC timestamp in the one format every record in this campaign uses." +utcnow() = Dates.format(now(UTC), dateformat"yyyy-mm-dd\THH:MM:SS\Z") + +"SHA-256 of a byte buffer or a string, as lowercase hex." +sha256_hex(data::Vector{UInt8}) = bytes2hex(sha256(data)) +sha256_hex(s::AbstractString) = bytes2hex(sha256(codeunits(String(s)))) + +""" + atomic_write(path, data) -> String + +Write `data` to a sibling temporary file, flush it, `fsync` it, `rename(2)` it +onto `path`, then read it back and re-hash it. Returns the digest. + +# Notes +A file half-written when the node dies is never visible under its final name, +which is the whole reason a controller may trust any file it finds. The read-back +is not paranoia about `rename`: it catches a full filesystem and a silently +truncated write, both of which this project has seen. +""" +function atomic_write(path::AbstractString, data::Vector{UInt8}) + mkpath(dirname(abspath(path))) + tmp = string(path, ".tmp.", getpid(), ".", time_ns()) + open(tmp, "w") do io + write(io, data) + flush(io) + try + ccall(:fsync, Cint, (Cint,), fd(io)) + catch + # fsync is a durability optimisation here, not a correctness one: + # the rename is what makes the file atomic. A filesystem that + # refuses it must not take the run down. + end + end + mv(tmp, path; force = true) + got = sha256_file(path) + want = sha256_hex(data) + got == want || error("atomic_write verification failed for $path") + return got +end +atomic_write(p::AbstractString, s::AbstractString) = + atomic_write(p, Vector{UInt8}(codeunits(String(s)))) + +"Serialize a dictionary to TOML and write it atomically. Returns the digest." +function write_toml_atomic(path::AbstractString, d::AbstractDict) + buf = IOBuffer() + TOML.print(buf, d; sorted = true) + return atomic_write(path, take!(buf)) +end + +""" + runner_code_digest(dir) -> String + +SHA-256 over the sorted `(relative path, file digest)` list of every `.jl` file +beside this runner. + +# Notes +Recorded in the result so a number can be tied to the exact code that produced +it even when the checkout was dirty — which, during a study, it usually is. The +git commit is recorded too, and the two answer different questions: the commit +says which revision was checked out, the digest says what was actually run. +""" +function runner_code_digest(dir::AbstractString) + rows = String[] + for (root, _, files) in walkdir(dir) + occursin("/.git", root) && continue + for f in files + endswith(f, ".jl") || continue + full = joinpath(root, f) + push!(rows, string(relpath(full, dir), " ", sha256_file(full))) + end + end + sort!(rows) + return sha256_hex(join(rows, "\n")) +end + +"The git commit of `dir`, or `\"none\"` outside a repository." +function git_commit(dir::AbstractString) + try + return strip(read(`git -C $dir rev-parse HEAD`, String)) + catch + return "none" + end +end + +""" + parse_args(args) -> Dict{String,String} + +`--key value` / `--flag` parser. + +# Notes +Unknown flags are KEPT rather than rejected, so an automated caller may pass +extras; but no flag outside the documented six is ever REQUIRED, which is what +keeps the six-flag command a complete command. +""" +function parse_args(args) + d = Dict{String,String}() + i = 1 + while i <= length(args) + if startswith(args[i], "--") + k = args[i][3:end] + if i < length(args) && !startswith(args[i+1], "--") + d[k] = args[i+1] + i += 2 + else + d[k] = "true" + i += 1 + end + else + i += 1 + end + end + return d +end + +"Read an integer vector from a TOML value that may be a list or a single number." +_int_list(v) = v isa AbstractVector ? [Int(x) for x in v] : [Int(v)] + +""" + chunk_seed(seed, completed) -> Int + +The seed of the chunk that starts at global iteration `completed`. + +``s_k = \\mathrm{hash}(seed, k) \\bmod 2^{31}`` + +# Notes +Deriving the seed from the GLOBAL index is what makes the sampling of an +iteration independent of the segmentation: iterations 4–6 face the same +scenarios whether they are the tail of one call or the whole of a resumed one. +Seeding once per run and letting the stream carry across chunks would not — a +resumed process starts with a fresh global RNG no matter what. +""" +chunk_seed(seed::Integer, completed::Integer) = + Int(hash((Int(seed), Int(completed))) % 0x7fffffff) + +# ───────────────────────────────────────────────────────────────────────────── +# The protocol descriptor +# ───────────────────────────────────────────────────────────────────────────── + +""" + write_protocol_descriptor(case_dir, out_path; kind=:screening) -> String + +Write the protocol descriptor a run is launched against, and return its path. + +# Notes +Generated from the frozen case itself: the `kind`, the shape and the digest are +copied out of the case manifest, so a descriptor cannot describe a protocol the +case does not declare. `:final` is refused here as well as at load time — a +descriptor naming the fresh panel should not exist in the first place. +""" +function write_protocol_descriptor(case_dir::AbstractString, out_path::AbstractString; + kind::Symbol = :screening) + kind === :final && error("refusing to write a descriptor for the FINAL protocol") + case = read_battery_case(case_dir) + scr = get(case.manifest, "screening", nothing) + block, resolved = scr === nothing ? (case.manifest["protocol"], "sole") : (scr, "screening") + kind === :screening || String(kind) == resolved || + error("case $(case.name) declares a $resolved protocol, not a $kind one") + write_toml_atomic(out_path, Dict{String,Any}( + "kind" => resolved, + "case" => case.name, + "num_stages" => Int(block["num_stages"]), + "num_scenarios" => Int(block["num_scenarios"]), + "seed" => Int(block["seed"]), + "sha256" => String(block["sha256"]), + "written_utc" => utcnow(), + )) + return out_path +end + +""" + resolve_protocol(case, descriptor_path) -> (matrix, kind, declared) + +Regenerate the evaluation protocol and bind it to the descriptor, fail-closed. + +# Returns +`(matrix, kind, declared)` — the `(stages × scenarios)` atom-index matrix, the +kind [`evaluation_protocol`](@ref) actually produced, and the descriptor as read. + +# Notes +FOUR refusals, in this order, and every one of them happens before a single +scenario outcome is computed: + + 1. a descriptor whose `kind` is `"final"` — training may never be selected on + the fresh panel, and the refusal must not depend on noticing it later; + 2. a descriptor whose kind disagrees with what the case declares; + 3. a shape that disagrees with the regenerated matrix; + 4. a digest that disagrees with the regenerated protocol's. + +The last one is the load-bearing check. `evaluation_protocol` already re-derives +the screening protocol from the frozen support and re-verifies it against the +manifest; the descriptor adds the statement that THIS RUN was launched against +that protocol and not another, which is the part a result file can be audited on +afterwards. +""" +function resolve_protocol(case::BatteryCase, descriptor_path::AbstractString) + isfile(descriptor_path) || error("no protocol descriptor at $descriptor_path") + d = TOML.parsefile(descriptor_path) + declared = String(get(d, "kind", "")) + declared == "final" && error( + "protocol descriptor $descriptor_path declares the FINAL protocol; " * + "training may only be selected on the screening protocol") + declared in ("screening", "sole") || error( + "protocol descriptor $descriptor_path declares kind $(repr(declared)); " * + "expected \"screening\" (or \"sole\" on a fixture case)") + + matrix, kind = evaluation_protocol(case) + String(kind) == declared || error( + "protocol descriptor declares $declared but the case regenerates a $kind protocol") + + block = kind === :sole ? case.manifest["protocol"] : case.manifest["screening"] + Int(get(d, "num_stages", -1)) == Int(block["num_stages"]) || + error("protocol descriptor stage count does not match the case") + Int(get(d, "num_scenarios", -1)) == Int(block["num_scenarios"]) || + error("protocol descriptor scenario count does not match the case") + String(get(d, "sha256", "")) == String(block["sha256"]) || + error("protocol descriptor digest does not match the case's $kind protocol") + size(matrix, 2) == Int(block["num_scenarios"]) || + error("regenerated protocol has $(size(matrix, 2)) columns, not $(block["num_scenarios"])") + return matrix, kind, d +end + +# ───────────────────────────────────────────────────────────────────────────── +# Identity +# ───────────────────────────────────────────────────────────────────────────── + +""" + run_identity(; manifest_path, case, method, config_path, conf, + protocol_path, protocol_kind, num_stages, backward) -> Dict + +Every coordinate that makes two runs scientifically different, plus one digest +over all of them. + +# The coordinates +| field | why it is here | +|---|---| +| `case_manifest_sha256` | the manifest FILE, byte for byte | +| `case_content_sha256` | the case CONTENT: the manifest's own artifact digests, so an edited manifest pointing at the same artifacts is still caught, and so is the reverse | +| `method` | which of the study's four this is | +| `config_sha256` | the frozen configuration file | +| `protocol_sha256`, `protocol_kind` | which panel selection may look at | +| `horizon` | the stage count actually trained | +| `seed` | the run's single seed | +| `formulation` | the backward PowerModels type, spelled out | +| `acp_bound_relax_factor` | the common true-ACP setting both engines state explicitly | + +# Notes +`identity_sha256` is a digest of the canonical `key=value` rendering of the +others. It is written into every checkpoint and re-derived on resume; a +mismatch on ANY coordinate refuses the resume rather than continuing a run whose +meaning changed underneath it. +""" +function run_identity(; manifest_path, case, method, config_path, conf, + protocol_path, protocol_kind, num_stages, backward) + spec = backward_spec(backward) + arts = case.manifest["artifacts"] + content = join([string(k, "=", arts[k]) for k in sort!(collect(keys(arts)))], ";") + + id = Dict{String,Any}( + "case" => case.name, + "case_manifest" => abspath(manifest_path), + "case_manifest_sha256" => sha256_file(manifest_path), + "case_content_sha256" => sha256_hex(content), + "method" => String(method), + "config_sha256" => sha256_file(config_path), + "protocol_sha256" => sha256_file(protocol_path), + "protocol_kind" => String(protocol_kind), + "horizon" => Int(num_stages), + "seed" => Int(conf["seed"]), + "formulation" => string(spec.formulation), + "forward_formulation" => string(FORWARD_FORMULATION), + "engine" => "jump", + "acp_bound_relax_factor" => ACP_BOUND_RELAX_FACTOR, + "checkpoint_schema" => RUNNER_CHECKPOINT_SCHEMA, + ) + id["identity_sha256"] = sha256_hex(join( + [string(k, "=", id[k]) for k in sort!(collect(keys(id)))], "\n")) + return id +end + +""" + assert_identity(want, got, whence) + +Refuse a continuation whose identity differs from this segment's, naming the +first field that differs. + +# Notes +Reporting the FIELD matters. "identity mismatch" sends a reader to diff two +hashes; "seed 1 vs 2" ends the investigation. +""" +function assert_identity(want::AbstractDict, got::AbstractDict, whence::AbstractString) + for k in sort!(collect(keys(want))) + k == "identity_sha256" && continue + haskey(got, k) || error("$whence is missing the identity field `$k`") + got[k] == want[k] || error( + "$whence identity mismatch on `$k`: checkpoint has $(repr(got[k])), " * + "this segment has $(repr(want[k]))") + end + String(get(got, "identity_sha256", "")) == String(want["identity_sha256"]) || + error("$whence identity digest mismatch") + return nothing +end + +# ───────────────────────────────────────────────────────────────────────────── +# Checkpoints +# ───────────────────────────────────────────────────────────────────────────── + +""" + checkpoint_paths(output, index, tag) -> (payload, sidecar) + +`checkpoints/ck_XXXXXXXX..json` and its `.meta.toml`. + +# Notes +Zero-padded so a listing sorts in run order, and TAGGED so a SOC checkpoint and +a DC checkpoint of the same case at the same iteration cannot be confused by +anything as weak as a `readdir` — the same rule [`sddp_cut_path`](@ref) enforces +on cut files, for the same reason. +""" +function checkpoint_paths(output::AbstractString, index::Integer, tag::AbstractString) + dir = joinpath(output, "checkpoints") + name = @sprintf("ck_%08d.%s.json", index, tag) + return joinpath(dir, name), joinpath(dir, name * ".meta.toml") +end + +""" + write_segment_checkpoint(output, index, cuts, state, ident; kind) -> (path, sha) + +Write one self-contained, verified, monotonically numbered checkpoint. + +# What it preserves +All cuts, as SDDP.jl's own `write_cuts_to_file` serialized them, embedded +verbatim; the completed iteration count; the sampling position (the seed and the +count the next chunk's seed is derived from); the convergence and evaluation +histories; and the best admissible true-ACP forward result with the iteration +that produced it. + +# Notes +ORDER MATTERS, and it is the same order the controller assumes. The payload is +written atomically and hashed FIRST; only then is the sidecar written naming +that digest. A crash between the two leaves a payload with no sidecar, which is +ignored; a crash before either leaves nothing. + +The cuts are EMBEDDED rather than referenced. A checkpoint that pointed at a +sibling cut file would be two files the controller hashes as one, and the pair +could be separated by any of the ways this filesystem loses things. +""" +function write_segment_checkpoint(output::AbstractString, index::Integer, + cuts, state::NamedTuple, ident::AbstractDict; + kind::AbstractString = "periodic") + path, meta_path = checkpoint_paths(output, index, state.tag) + body = Dict{String,Any}( + "segment_schema" => RUNNER_CHECKPOINT_SCHEMA, + "method" => ident["method"], + "backward" => String(state.backward), + "tag" => state.tag, + "global_index" => Int(index), + "iterations_completed" => Int(index), + "seed" => Int(state.seed), + "next_chunk_seed" => chunk_seed(state.seed, index), + "bound" => state.bound, + "bound_name" => state.bound_name, + "bound_bounds_acp" => state.bound_bounds_acp, + # `null` when no complete evaluation has been admissible yet. JSON has no + # infinity, so the sentinel is an absent value and not a magic number. + "best_cost" => isfinite(state.best_cost) ? state.best_cost : nothing, + "best_index" => Int(state.best_index), + "trajectory_checksum" => state.checksum, + "convergence" => state.convergence, + "evaluations" => state.evaluations, + "identity" => Dict{String,Any}(ident), + "written_utc" => utcnow(), + "cuts" => cuts, + ) + sha = atomic_write(path, JSON.json(body)) + + write_toml_atomic(meta_path, Dict{String,Any}( + "file" => basename(path), + "sha256" => sha, + "parent_sha256" => state.parent_sha, + "global_index" => Int(index), + "index_from" => Int(state.index_from), + "run_id" => state.run_id, + "segment" => Int(state.segment), + "attempt" => Int(state.attempt), + "kind" => kind, + "written_utc" => utcnow(), + )) + return path, sha +end + +""" + load_segment_checkpoint(path) -> Dict + +Read a parent checkpoint, refusing it unless its sidecar digest matches the +payload on disk and it carries this file's segment schema. + +# Notes +The controller verifies checkpoints too. This check is not redundant with it: a +worker resuming from a file nobody re-hashed since the scan would be trusting a +window it cannot see into, and the two ends enforce the rule independently so +neither has to assume the other ran. +""" +function load_segment_checkpoint(path::AbstractString) + isfile(path) || error("--resume-from does not exist: $path") + meta_path = path * ".meta.toml" + if isfile(meta_path) + side = TOML.parsefile(meta_path) + got = sha256_file(path) + got == String(get(side, "sha256", "")) || error( + "parent checkpoint digest mismatch: $path has $got, its sidecar claims " * + "$(get(side, "sha256", "missing"))") + end + ck = JSON.parsefile(path) + String(get(ck, "segment_schema", "")) == RUNNER_CHECKPOINT_SCHEMA || error( + "parent checkpoint $path carries segment schema " * + "$(repr(get(ck, "segment_schema", missing))) but this runner writes " * + "$RUNNER_CHECKPOINT_SCHEMA") + haskey(ck, "identity") || error( + "parent checkpoint $path carries no identity record; it was not written " * + "by this runner and cannot be continued") + return ck +end + +""" + materialize_cuts(dir, cuts, tag) -> String + +Write an embedded cut set back out as the standalone JSON file +`SDDP.read_cuts_from_file` expects, and return its path. + +# Notes +The file name carries the arm's tag, so `train_battery_sddp`'s own refusal — +which rejects a cut path that does not name the formulation — applies to a +resume exactly as it applies to a write. Round-tripping through SDDP's own +format, rather than reconstructing cut objects, is what keeps this file free of +any knowledge of how a cut is represented. +""" +function materialize_cuts(dir::AbstractString, cuts, tag::AbstractString) + mkpath(dir) + path = joinpath(dir, "resume.$tag.cuts.json") + atomic_write(path, JSON.json(cuts)) + return path +end + +# ───────────────────────────────────────────────────────────────────────────── +# The segment +# ───────────────────────────────────────────────────────────────────────────── + +""" + moving_average(v, w) -> Vector{Float64} + +Trailing moving average of window `w`, defined from the first sample: + +``\\mathrm{ma}_i = \\frac{1}{\\min(i,w)} \\sum_{j=\\max(1,i-w+1)}^{i} v_j`` + +# Notes +SDDP's per-iteration forward cost is one sample of a random objective on one +sampled scenario. It is a different estimand from the fixed-panel evaluation and +is never compared with it; only its moving average is legible, so both are +written and the raw column is kept beside it. +""" +function moving_average(v::AbstractVector, w::Integer) + n = length(v) + out = zeros(Float64, n) + s = 0.0 + for i in 1:n + s += v[i] + i > w && (s -= v[i-w]) + out[i] = s / min(i, w) + end + return out +end + +""" + evaluate_acp_panel(case, trained, matrix, columns; max_recourse) -> NamedTuple + +The true-ACP forward cost of the current policy on the fixed panel, under the +shared physical cost contract. + +# Returns +`(mean_cost, costs, worst_recourse, complete, raw_mean)`. + +# Notes +SDDP's own `Historical` replay of the panel's columns through the ACP graph +([`simulate_battery_sddp_on`](@ref)) — the same simulation +`train_battery_sddp`'s periodic callback and `cost_report` use — scored by +[`sddp_physical_path_cost`](@ref), which puts every stage through +`physical_stage_cost`, the byte-identical contract the Exa engine also selects +on. + +**Element by element, and in this order: validate, project, then sum.** The +recorders store the per-bus recourse raw; the contract projects each individual +value within `PHYSICAL_RECOURSE_TOL` to exactly zero, refuses the stage if any +individual value is outside it, and only then are the charges summed. An +aggregate test is not a weaker version of this — it is a different and wrong +test, because tolerance-scale residues of opposite sign at different buses +cancel in a total. + +`raw_mean` is the sum of the solver's raw stage objectives, carried as a +diagnostic. It is never the headline and never the selection metric: the raw +objective carries the barrier residue of whatever `bound_relax_factor` the solve +ran at, and selecting on it would select on a solver setting. +""" +function evaluate_acp_panel(case::BatteryCase, trained, matrix::AbstractMatrix{<:Integer}, + columns::AbstractVector{<:Integer}; max_recourse::Real = 1e-6) + ids = [b.index for b in case.batteries] + T = trained.num_stages + sims = simulate_battery_sddp_on(trained, matrix; ids = ids, columns = columns) + phys = [sddp_physical_path_cost(case, s, T; tol = max_recourse) for s in sims] + costs = [p.cost for p in phys] + raw = [sddp_path_cost(case, s, T) for s in sims] + worst = maximum(p.worst_recourse for p in phys; init = 0.0) + complete = length(costs) == length(columns) && all(isfinite, costs) && + all(p.admissible for p in phys) + return (mean_cost = isempty(costs) ? NaN : mean(costs), costs = costs, + worst_recourse = worst, complete = complete, + raw_mean = isempty(raw) ? NaN : mean(raw)) +end + +""" + run_segment(a) -> Int + +Drive one segment: bind identity, resume or start, train to the segment's stop +index or until asked to stop, and write a verified checkpoint and an honest +result. Returns a process exit code. + +# The loop, one chunk +Rebuild both policy graphs from the frozen case, restore the previous chunk's +cuts, run `checkpoint_every` stock SDDP iterations against the seed derived from +the global index, write out the cuts, and checkpoint. The per-iteration bound +and forward cost come from SDDP's own training log, so recording the trajectory +costs no extra solve. + +# Stopping +The stop file is polled after every COMPLETE chunk — i.e. after a checkpoint +exists for the work just done. On a stop request an honest `preempted` result is +written and the process exits 0. `complete` is reported only when the configured +target index was actually reached. +""" +function run_segment(a::AbstractDict) + t_start = time() + + # ---- the six scientific flags ----------------------------------------- + manifest_path = abspath(a["case-manifest"]) + method = Symbol(a["method"]) + config_path = abspath(a["config"]) + protocol_path = abspath(a["protocol"]) + output = abspath(a["output"]) + resume = get(a, "resume-from", "none") + resume = (resume == "none" || isempty(resume)) ? "none" : abspath(resume) + + # ---- controller conveniences, every one defaulted ---------------------- + run_id = get(a, "run-id", "standalone") + segment = parse(Int, get(a, "segment", "1")) + attempt = parse(Int, get(a, "attempt", "1")) + stop_file = abspath(get(a, "stop-file", joinpath(output, "STOP"))) + max_secs = parse(Float64, get(a, "max-seconds", "1e9")) + # See the Exa runner: a terminal measurement the caller asks for once when + # retiring a run. Default off. + final_eval = get(a, "final-evaluation", "false") in ("true", "1") + + # ---- method ownership, before anything is loaded ----------------------- + if !(method in RUNNER_METHODS) + haskey(BATTERY_METHODS, method) || error( + "unknown method :$method; the study's methods are " * + "$(sort!(collect(keys(BATTERY_METHODS))))") + error("method :$method runs on the $(battery_method(method).engine) engine " * + "(DecisionRulesExa.jl/examples/BatteryStorageOPF/portfolio_runner.jl), " * + "not on this one") + end + backward = battery_method(method).backward::Symbol + spec = backward_spec(backward) + + mkpath(joinpath(output, "checkpoints")) + conf = TOML.parsefile(config_path) + haskey(conf, "method") && String(conf["method"]) != String(method) && error( + "the frozen config names method $(conf["method"]) but --method is $method") + + target_index = Int(get(conf, "target_index", 200)) + segment_updates = Int(get(conf, "segment_updates", 50)) + ckpt_every = Int(get(conf, "checkpoint_every", 10)) + eval_every = Int(get(conf, "eval_every", 20)) + ma_window = Int(get(conf, "ma_window", 25)) + num_stages = Int(get(conf, "num_stages", 24)) + eval_columns = _int_list(get(conf, "eval_columns", [1, 2, 3, 4])) + seed = Int(get(conf, "seed", 20260804)) + max_recourse = Float64(get(conf, "max_recourse", 1e-6)) + # OPT-IN, default FALSE. A lineage that does not ask for it behaves exactly + # as it always has, byte for byte; only a config that sets it true gets the + # canonicalized cut representation. With `checkpoint_every = 1` the chunk is + # one iteration, so every cut is canonicalized before the next iteration can + # read it. + canonicalize_cuts = Bool(get(conf, "canonicalize_cuts", false)) + + # ---- the frozen case, its protocol, and the identity ------------------- + case_dir = dirname(manifest_path) + basename(manifest_path) == "case_manifest.json" || error( + "--case-manifest must name a case_manifest.json, got $(basename(manifest_path))") + case = read_battery_case(case_dir) + + eval_matrix, protocol_kind, _ = resolve_protocol(case, protocol_path) + size(eval_matrix, 1) >= num_stages || error( + "the $protocol_kind protocol covers $(size(eval_matrix, 1)) stages but " * + "training asks for $num_stages") + maximum(eval_columns) <= size(eval_matrix, 2) || error( + "panel column $(maximum(eval_columns)) is outside the $protocol_kind " * + "protocol's $(size(eval_matrix, 2)) columns") + + ident = run_identity(; manifest_path = manifest_path, case = case, method = method, + config_path = config_path, conf = conf, + protocol_path = protocol_path, protocol_kind = protocol_kind, + num_stages = num_stages, backward = backward) + write_toml_atomic(joinpath(output, "identity.toml"), ident) + + @printf("segment %s seg%d att%d · method %s (%s backward) · case %s\n", + run_id, segment, attempt, method, spec.formulation, case.name) + @printf(" panel: %s protocol, %d of %d columns · identity %s\n", + protocol_kind, length(eval_columns), size(eval_matrix, 2), + first(ident["identity_sha256"], 16)) + + # SDDP.jl writes its training log into the working directory. Run from the + # attempt directory so it lands with the rest of this segment's evidence + # instead of in whatever directory the caller happened to be in; every path + # this function touches was made absolute above. + cd(output) + + # ---- resume ------------------------------------------------------------ + index_from = 0 + parent_sha = "none" + convergence = Any[] + evaluations = Any[] + best_cost = Inf + best_index = 0 + checksum = 0.0 + cur_cuts = nothing # path of the cut file the next chunk resumes from + + if resume != "none" + ck = load_segment_checkpoint(resume) + assert_identity(ident, Dict{String,Any}(ck["identity"]), "parent checkpoint") + index_from = Int(ck["global_index"]) + checksum = Float64(ck["trajectory_checksum"]) + best_cost = ck["best_cost"] === nothing ? Inf : Float64(ck["best_cost"]) + best_index = Int(ck["best_index"]) + convergence = collect(ck["convergence"]) + evaluations = collect(ck["evaluations"]) + cur_cuts = materialize_cuts(joinpath(output, "checkpoints"), ck["cuts"], spec.tag) + parent_sha = sha256_file(resume) + @printf(" resumed from iteration %d · checksum %.10e · best %.6f\n", + index_from, checksum, best_cost) + end + + stop_target = min(target_index, index_from + segment_updates) + index_from < stop_target || error( + "nothing to do: resumed at $index_from with stop target $stop_target") + + # ---- local artifacts --------------------------------------------------- + hist_io = open(joinpath(output, "history.csv"), "w") + println(hist_io, "index,train_loss,panel_value,bound,solve_ok,solve_fail,deficit") + flush(hist_io) + + traj_rows = Tuple{Int,Float64,Float64,Float64}[] + forward_costs = Float64[] + new_evals = Any[] + # SDDP counts its own subproblem solves in `total_solves`, which restarts at + # zero on every freshly built graph — so it is accumulated across chunks + # rather than read as a running total. + solve_ok = 0 + solve_fail = 0 + worst_recourse_seen = 0.0 + # TIME ACCOUNTING, matching the Exa runner: setup, training (the SDDP + # iterations) and evaluation (the true-ACP screening panel) are measured + # separately, because the campaign's budget is on ACTIVE TRAINING and queue + # time and depot construction are the controller's to report, not this + # process's. + setup_seconds = time() - t_start + training_seconds = 0.0 + evaluation_seconds = 0.0 + + idx = index_from + last_ck_path = resume == "none" ? "" : resume + last_ck_sha = parent_sha + bound = NaN + reason = "segment_updates_reached" + + while idx < stop_target + n = min(ckpt_every, stop_target - idx) + s_k = chunk_seed(seed, idx) + + _t_train = time() + trained = train_battery_sddp(case; + backward = backward, + num_stages = num_stages, + iteration_limit = n, + seed = s_k, + print_level = 0, + resume_cuts = cur_cuts, + cut_path = joinpath(output, "checkpoints", + "chunk.$(spec.tag).cuts.json"), + canonicalize_cuts = canonicalize_cuts) + training_seconds += time() - _t_train + bound = trained.bound + + # SDDP's own per-iteration log: the bound and the forward-pass cost of + # that iteration's sampled scenario, at no extra solve. `iteration` is + # local to the call, so it is offset onto the global index here. + log = trained.backward_graph.most_recent_training_results.log + chunk_solves = 0 + for entry in log + gi = idx + Int(entry.iteration) + fc = Float64(entry.simulation_value) + checksum += fc + push!(forward_costs, fc) + push!(traj_rows, (gi, fc, Float64(entry.bound), time() - t_start)) + push!(convergence, Dict{String,Any}( + "iteration" => gi, "bound" => Float64(entry.bound), + "forward_cost" => fc, "wall_seconds" => time() - t_start)) + chunk_solves = max(chunk_solves, Int(entry.total_solves)) + entry.serious_numerical_issue && (solve_fail += 1) + end + solve_ok += chunk_solves + idx += n + + # ---- fixed-panel evaluation and checkpoint selection --------------- + panel_value = NaN + selected = false + # SEGMENTATION-INVARIANT SCHEDULE — see the Exa runner: the panel is + # measured at the global indices the config names and at the target, + # never because this process reached its segment cap. + if eval_every > 0 && (idx % eval_every == 0 || idx == target_index || + (final_eval && idx == stop_target)) + _t_eval = time() + ev = evaluate_acp_panel(case, trained, eval_matrix, eval_columns; + max_recourse = max_recourse) + evaluation_seconds += time() - _t_eval + panel_value = ev.mean_cost + worst_recourse_seen = max(worst_recourse_seen, ev.worst_recourse) + selected = ev.complete && improves(ev.mean_cost, best_cost) + selected && (best_cost = ev.mean_cost; best_index = idx) + row = Dict{String,Any}( + "index" => idx, "protocol" => String(protocol_kind), + "acp_mean" => ev.mean_cost, "acp_raw_mean" => ev.raw_mean, + "worst_recourse" => ev.worst_recourse, + "complete" => ev.complete, "selected" => selected, + "best_cost" => best_cost) + push!(evaluations, row) + push!(new_evals, row) + @printf(" iteration %d · %s panel: true-ACP mean %16.6f worst recourse %.3e complete %s%s\n", + idx, protocol_kind, ev.mean_cost, ev.worst_recourse, ev.complete, + selected ? " [selected]" : "") + end + + # One history row per ITERATION, so the controller's continuous curve has + # a point per unit of global index. The panel value and the deficit are + # attached to the last iteration of the chunk they were measured after. + for (k, r) in enumerate(traj_rows[end-length(log)+1:end]) + last = k == length(log) + @printf(hist_io, "%d,%.10f,%s,%.10f,%d,%d,%.10e\n", r[1], r[2], + (last && !isnan(panel_value)) ? @sprintf("%.10f", panel_value) : "NaN", + r[3], solve_ok, solve_fail, worst_recourse_seen) + end + flush(hist_io) + + @printf("chunk done · iterations %d→%d · %s %.6f%s\n", + idx - n, idx, trained.bound_name, trained.bound, + trained.bound_bounds_acp ? "" : " (NOT a bound on the ACP problem)") + + # ---- checkpoint ---------------------------------------------------- + cuts = JSON.parsefile(joinpath(output, "checkpoints", + "chunk.$(spec.tag).cuts.json")) + state = (tag = spec.tag, backward = backward, seed = seed, bound = trained.bound, + bound_name = trained.bound_name, + bound_bounds_acp = trained.bound_bounds_acp, + best_cost = best_cost, best_index = best_index, checksum = checksum, + convergence = convergence, evaluations = evaluations, + parent_sha = parent_sha, index_from = index_from, + run_id = run_id, segment = segment, attempt = attempt) + last_ck_path, last_ck_sha = write_segment_checkpoint( + output, idx, cuts, state, ident; + kind = idx == stop_target ? "final" : "periodic") + cur_cuts = materialize_cuts(joinpath(output, "checkpoints"), cuts, spec.tag) + + # ---- graceful stop, between complete chunks ------------------------ + if isfile(stop_file) || (time() - t_start) > max_secs + reason = isfile(stop_file) ? "signal_stop" : "max_seconds" + @info "stopping early" reason index = idx + break + end + end + close(hist_io) + + # ---- the richer local artifacts ---------------------------------------- + # Cumulative global history, not this segment's rows — the window must not + # restart at a resume. + conv_costs = [Float64(c["forward_cost"]) for c in convergence] + conv_index = [Int(c["iteration"]) for c in convergence] + ma_all = moving_average(conv_costs, ma_window) + ma_by_index = Dict(conv_index[k] => ma_all[k] for k in eachindex(conv_index)) + open(joinpath(output, "trajectory.csv"), "w") do io + println(io, "index,forward_cost,forward_cost_ma$(ma_window),bound,wall_seconds") + for r in traj_rows + @printf(io, "%d,%.10f,%.10f,%.10f,%.3f\n", r[1], r[2], + get(ma_by_index, r[1], NaN), r[3], r[4]) + end + end + open(joinpath(output, "evaluation.csv"), "w") do io + println(io, "index,protocol,columns,acp_mean,acp_raw_mean,worst_recourse,complete,selected,best_cost") + for e in new_evals + @printf(io, "%d,%s,%s,%.10f,%.10f,%.6e,%s,%s,%.10f\n", e["index"], e["protocol"], + join(eval_columns, " "), e["acp_mean"], e["acp_raw_mean"], + e["worst_recourse"], e["complete"], e["selected"], e["best_cost"]) + end + end + + # ---- the segment result ------------------------------------------------- + reached = idx >= stop_target + reached && idx >= target_index && (reason = "target_reached") + isempty(last_ck_path) && error( + "the segment produced no checkpoint; refusing to write a result that " * + "claims progress it cannot evidence") + + here = @__DIR__ + projdir = dirname(something(Base.active_project(), joinpath(here, "Project.toml"))) + result = Dict{String,Any}( + "schema" => RUNNER_RESULT_SCHEMA, + "run_id" => run_id, + "segment" => segment, + "attempt" => attempt, + "method" => String(method), + "status" => reached ? "complete" : "preempted", + # `status` is the SEGMENT's verdict, and the controller depends on that: + # a segment that finished its planned updates must be acceptable, or a + # multi-segment run could never make progress. Whether the RUN is + # finished is a different question and gets its own field — a reader + # must never infer "the run reached its target" from "the segment + # completed". `termination_reason` separates them too: + # `target_reached` only when the configured target index was reached. + "run_complete" => idx >= target_index, + "termination_reason" => reason, + "command" => join(vcat(["julia", "--project=" * projdir, @__FILE__], + ARGS), " "), + "julia_version" => string(VERSION), + "project_toml_sha256" => sha256_file(joinpath(projdir, "Project.toml")), + "manifest_toml_sha256" => isfile(joinpath(projdir, "Manifest.toml")) ? + sha256_file(joinpath(projdir, "Manifest.toml")) : "none", + "code_commit" => git_commit(here), + "code_digest" => runner_code_digest(here), + "case_manifest" => manifest_path, + "case_digest" => sha256_file(manifest_path), + "config_digest" => sha256_file(config_path), + "protocol_digest" => sha256_file(protocol_path), + "protocol_kind" => String(protocol_kind), + "support_digest" => String(case.manifest["support"]["sha256"]), + "identity_digest" => ident["identity_sha256"], + "parent_checkpoint" => resume, + "parent_sha256" => parent_sha, + "child_checkpoint" => last_ck_path, + "child_sha256" => last_ck_sha, + "index_from" => index_from, + "index_to" => idx, + "updates_completed" => idx - index_from, + "target_index" => target_index, + "wall_seconds" => round(time() - t_start, digits = 3), + "setup_seconds" => round(setup_seconds, digits = 3), + "training_seconds" => round(training_seconds, digits = 3), + "evaluation_seconds" => round(evaluation_seconds, digits = 3), + "gpu_seconds" => 0.0, + "solve_total" => solve_ok + solve_fail, + "solve_optimal" => solve_ok, + "solve_failed" => solve_fail, + "physical_deficit" => worst_recourse_seen, + "physical_surplus" => 0.0, + # The scalar and, beside it, what it is allowed to be called and whether + # it bounds anything about the true ACP problem. The DC arm's does not, + # and it must never acquire the word on the strength of a column name. + "sddp_bound" => bound, + "bound_name" => spec.bound_name, + "bound_bounds_acp" => spec.bounds_acp, + "backward_formulation" => string(spec.formulation), + "best_panel_cost" => best_cost == Inf ? NaN : best_cost, + "best_panel_index" => best_index, + "trajectory_checksum" => checksum, + "history_rows" => length(traj_rows), + "history_sha256" => sha256_file(joinpath(output, "history.csv")), + "slurm_job_id" => get(ENV, "SLURM_JOB_ID", ""), + "slurm_array_id" => get(ENV, "SLURM_ARRAY_JOB_ID", ""), + "node" => gethostname(), + "started_utc" => Dates.format(unix2datetime(t_start), + dateformat"yyyy-mm-dd\THH:MM:SS\Z"), + "finished_utc" => utcnow(), + "eval_indices" => [e["index"] for e in new_evals], + "eval_values" => [e["acp_mean"] for e in new_evals], + ) + write_toml_atomic(joinpath(output, "result.toml"), result) + @printf("segment done · %s · iteration %d→%d · checksum %.10e · best %.6f\n", + result["status"], index_from, idx, checksum, best_cost) + return 0 +end + +""" + main(args=ARGS) -> Int + +Check that the six scientific flags are present, then run one segment. +""" +function main(args = ARGS) + a = parse_args(args) + for k in ("case-manifest", "method", "config", "protocol", "output") + haskey(a, k) || error("--$k is required; see the header of $(@__FILE__)") + end + return run_segment(a) +end + +if abspath(PROGRAM_FILE) == @__FILE__ + exit(main(ARGS)) +end diff --git a/examples/BatteryStorageOPF/renewable_variant.jl b/examples/BatteryStorageOPF/renewable_variant.jl new file mode 100644 index 0000000..f2373b6 --- /dev/null +++ b/examples/BatteryStorageOPF/renewable_variant.jl @@ -0,0 +1,813 @@ +# Renewable/storage variants of a frozen battery case. +# +# A variant is a TRANSFORMATION of a frozen case, not a new case family: the +# host network, its battery buses, its temporal profile and its cost data all +# come from the base case, and this file only applies the edits named in +# `renewable_variants.toml`. That is what lets the same recipe be carried to +# another host without rewriting anything, and what lets a reader reconstruct a +# variant from the base case plus one TOML file. +# +# Four edits, in this order: +# +# 1. assign every bus a REGION (a role, assigned by the recipe — not a +# property discovered in the case); +# 2. apply a GENERATION TRANSITION, deleting the units the recipe retires; +# 3. add RENEWABLE units with a per-stage availability schedule; +# 4. RESIZE the battery fleet by region, preserving the base case's total +# power and total energy exactly. +# +# and one optional convention: a matched initial/terminal inventory. +# +# Commands: +# julia --project=. renewable_variant.jl --base DIR --out DIR --template reliable + +using JSON +using Printf +using TOML + +@isdefined(read_battery_case) || include(joinpath(@__DIR__, "battery_case.jl")) + +"Schema tag of the renewable-variant recipe." +const RENEWABLE_VARIANT_SCHEMA = "battery_storage_opf/renewable_variants/1" + +"Default location of the recipe, beside this file." +renewable_variants_path(dir::AbstractString = @__DIR__) = + joinpath(dir, "renewable_variants.toml") + +# ───────────────────────────────────────────────────────────────────────────── +# A — the recipe +# ───────────────────────────────────────────────────────────────────────────── + +""" + renewable_variant_recipe(path = renewable_variants_path()) -> Dict + +Read and check the variant recipe. + +# Notes +The schema tag is checked here and nowhere else, so a recipe written for a later +version of this file fails at the first call rather than silently producing a +case built from half of its fields. +""" +function renewable_variant_recipe(path::AbstractString = renewable_variants_path()) + isfile(path) || error("no renewable-variant recipe at $path") + rec = TOML.parsefile(path) + get(rec, "schema", nothing) == RENEWABLE_VARIANT_SCHEMA || + error("recipe schema $(get(rec, "schema", nothing)); expected $RENEWABLE_VARIANT_SCHEMA") + haskey(rec, "regions") || error("recipe has no [regions]") + haskey(rec, "transition") || error("recipe has no [transition]") + haskey(rec, "renewables") || error("recipe has no [renewables]") + return rec +end + +# ───────────────────────────────────────────────────────────────────────────── +# B — regions +# ───────────────────────────────────────────────────────────────────────────── + +""" + variant_bus_region(network, recipe) -> Dict{Int,String} + +Map every bus identifier of `network` to the name of the region it belongs to. + +# Notes +A zone named by no region is an error rather than a silent assignment to the +rest-of-system region: a recipe that forgot a zone has forgotten part of the +host, and the resulting case would carry a region map nobody wrote down. +""" +function variant_bus_region(network::AbstractDict, recipe::AbstractDict) + zone_of_region = Dict{Int,String}() + for (name, spec) in recipe["regions"], z in spec["zones"] + haskey(zone_of_region, z) && + error("zone $z is claimed by both $(zone_of_region[z]) and $name") + zone_of_region[z] = name + end + out = Dict{Int,String}() + for (_, bus) in network["bus"] + i = Int(bus["bus_i"]) + z = Int(bus["zone"]) + haskey(zone_of_region, z) || error("zone $z (bus $i) is named by no region") + out[i] = zone_of_region[z] + end + return out +end + +""" + variant_region_names(recipe) -> Vector{String} + +Region names in a deterministic order: the recipe's own key order is a `Dict` +order and must never reach a result. +""" +variant_region_names(recipe::AbstractDict) = sort!(collect(keys(recipe["regions"]))) + +# ───────────────────────────────────────────────────────────────────────────── +# C — generator economics +# ───────────────────────────────────────────────────────────────────────────── + +""" + gen_cost_coefficients(gen) -> (c2, c1, c0) + +The generator's cost polynomial padded to degree two, in the per-unit basis +PowerModels stores it in: + +```math +C(p) = c_2 p^2 + c_1 p + c_0 ,\\qquad p \\text{ in per unit}. +``` + +# Notes +`ncost` is the number of coefficients actually present, highest order first, so +a linear cost is `[c1, c0]` and a constant cost is `[c0]`. Padding here means no +caller has to branch on `ncost`, which is exactly the branch that gets forgotten. +""" +function gen_cost_coefficients(gen::AbstractDict) + c = gen["cost"] + n = Int(gen["ncost"]) + n == 3 && return (Float64(c[1]), Float64(c[2]), Float64(c[3])) + n == 2 && return (0.0, Float64(c[1]), Float64(c[2])) + n == 1 && return (0.0, 0.0, Float64(c[1])) + return (0.0, 0.0, 0.0) +end + +""" + gen_marginal_cost(gen, p, baseMVA) -> Float64 + +Marginal cost ``dC/dp`` at output `p` (per unit), converted to currency per MWh: + +```math +\\frac{dC}{dp}(p) = 2 c_2 p + c_1 +\\quad\\text{[currency per pu·h]},\\qquad +\\text{divide by } \\texttt{baseMVA} \\text{ for currency per MWh.} +``` +""" +function gen_marginal_cost(gen::AbstractDict, p::Real, baseMVA::Real) + c2, c1, _ = gen_cost_coefficients(gen) + return (2 * c2 * p + c1) / baseMVA +end + +""" + gen_average_variable_cost(gen, baseMVA) -> Float64 + +Average variable cost at RATED output, in currency per MWh: + +```math +\\mathrm{avc} = \\frac{c_2 \\overline p^2 + c_1 \\overline p}{\\overline p} + \\Big/ \\texttt{baseMVA} + = \\frac{c_2 \\overline p + c_1}{\\texttt{baseMVA}} . +``` + +# Notes +This is the measure the transition rule uses, and it is not interchangeable with +the linear coefficient. On a host where many units carry a nonzero quadratic +term, a unit with `c1 = 0` can have an average variable cost of tens of currency +units per MWh; ordering by `c1` would retire it as if it were free. The constant +term is deliberately absent: it is charged whether or not the unit generates and +so says nothing about the cost of energy from it. + +Returns `0.0` for a unit with `pmax == 0`, which has no rated output to average +over and is never the marginal unit anyway. +""" +function gen_average_variable_cost(gen::AbstractDict, baseMVA::Real) + pmax = Float64(gen["pmax"]) + pmax > 0 || return 0.0 + c2, c1, _ = gen_cost_coefficients(gen) + return (c2 * pmax + c1) / baseMVA +end + +# ───────────────────────────────────────────────────────────────────────────── +# D — the generation transition +# ───────────────────────────────────────────────────────────────────────────── + +""" + variant_retirements(network, recipe) -> Vector{Int} + +Generator indices the transition retires, sorted ascending. + +# Notes +Only IN-SERVICE units are candidates: PGLib files carry out-of-service +generators that PowerModels never puts in `ref`, and retiring one of those would +be a retirement that changes nothing while appearing in the record as if it had. + +The two exception rules are applied AFTER the threshold, so the record shows a +one-line rule plus a named list of units kept in spite of it, rather than a +threshold tuned until the right units fell out. +""" +function variant_retirements(network::AbstractDict, recipe::AbstractDict) + tr = recipe["transition"] + tr["measure"] == "average_variable_cost_at_pmax" || + error("unknown transition measure $(tr["measure"])") + baseMVA = Float64(network["baseMVA"]) + θ = Float64(tr["threshold_usd_per_mwh"]) + + live = [g for (_, g) in network["gen"] if Int(get(g, "gen_status", 1)) == 1] + sort!(live; by = g -> Int(g["index"])) + + # Geographic exemption, applied first and by PLACE: a generator standing in + # an exempt zone is not a candidate at all, whatever its cost. Resolving it + # through the bus's own `zone` field rather than through a list of indices is + # what makes it survive a change of host or of generator numbering. + exempt_zones = Set{Int}(Int.(get(tr, "exempt_zones", Int[]))) + zone_of_bus = Dict{Int,Int}(Int(b["bus_i"]) => Int(b["zone"]) for (_, b) in network["bus"]) + in_exempt_zone(g) = get(zone_of_bus, Int(g["gen_bus"]), -1) in exempt_zones + + keep = Set{Int}(Int.(get(tr, "keep_extra_gen_indices", Int[]))) + # Restored-for-service units are excluded from the retirement exactly as the + # adequacy exceptions are, so they keep their own bounds, cost curve and + # reactive limits. They are listed separately in the recipe and recorded + # separately in the manifest so the two reasons never blur together. + union!(keep, Set{Int}(Int.(get(tr, "restore_gen_indices", Int[])))) + if get(tr, "keep_reference_bus_unit", false) + refs = sort!([Int(b["bus_i"]) for (_, b) in network["bus"] if Int(b["bus_type"]) == 3]) + length(refs) == 1 || error("expected exactly one reference bus, found $(length(refs))") + at_ref = [g for g in live if Int(g["gen_bus"]) == refs[1] && + gen_average_variable_cost(g, baseMVA) <= θ && + !in_exempt_zone(g)] + if !isempty(at_ref) + # Lowest average variable cost, ties broken by index: two identical + # units at the reference bus must not make the case depend on which + # one a JSON reader happened to hand back first. + best = first(sort(at_ref; by = g -> (gen_average_variable_cost(g, baseMVA), + Int(g["index"])))) + push!(keep, Int(best["index"])) + end + end + return sort!([Int(g["index"]) for g in live + if gen_average_variable_cost(g, baseMVA) <= θ && + !(Int(g["index"]) in keep) && !in_exempt_zone(g)]) +end + +# ───────────────────────────────────────────────────────────────────────────── +# E — the renewable build +# ───────────────────────────────────────────────────────────────────────────── + +""" + variant_availability(recipe, region, template, horizon) -> Vector{Float64} + +The per-stage availability multipliers of `region` under `template`. + +# Notes +A key `"_