From 6332d1dccbd6054afe629f14a0e396594441efc5 Mon Sep 17 00:00:00 2001
From: Fredrik Ahlgren <fredrik@sourceful-labs.com>
Date: Mon, 31 Aug 2026 19:47:41 +0200
Subject: [PATCH] fix(prices): take Nord Pool when the Sourceful harvest is
 stale

Sourceful's ENTSO-E cache for SE3 can sit unhealthy for a day after
Nord Pool has published tomorrow. Predicted from midnight was that
gap, not a chart bug. Fall back to Nord Pool's day-ahead dataportal
and fetch again at 13:05 Europe/Stockholm.
---
 .changeset/day-ahead-nordpool.md    |   5 +
 go/internal/prices/fallback.go      |  49 ++++++++++
 go/internal/prices/nordpool.go      | 147 ++++++++++++++++++++++++++++
 go/internal/prices/nordpool_test.go | 138 ++++++++++++++++++++++++++
 go/internal/prices/prices.go        |  38 +++++--
 5 files changed, 371 insertions(+), 6 deletions(-)
 create mode 100644 .changeset/day-ahead-nordpool.md
 create mode 100644 go/internal/prices/fallback.go
 create mode 100644 go/internal/prices/nordpool.go
 create mode 100644 go/internal/prices/nordpool_test.go

diff --git a/.changeset/day-ahead-nordpool.md b/.changeset/day-ahead-nordpool.md
new file mode 100644
index 000000000..efca07157
--- /dev/null
+++ b/.changeset/day-ahead-nordpool.md
@@ -0,0 +1,5 @@
+---
+"ftw": patch
+---
+
+When the Sourceful price harvest is stale, FTW takes tomorrow's day-ahead from Nord Pool instead of filling the night with the ML twin.
diff --git a/go/internal/prices/fallback.go b/go/internal/prices/fallback.go
new file mode 100644
index 000000000..413806342
--- /dev/null
+++ b/go/internal/prices/fallback.go
@@ -0,0 +1,49 @@
+package prices
+
+import (
+	"context"
+	"log/slog"
+	"time"
+)
+
+// fallbackProvider tries primary first. If that day is empty (not yet in
+// the harvest cache), it asks secondary. Name stays the primary so a
+// configured sourceful box still reports sourceful unless the fallback
+// actually supplied the rows — Fetch logs that case.
+type fallbackProvider struct {
+	primary, secondary Provider
+}
+
+func withFallback(primary, secondary Provider) Provider {
+	if primary == nil {
+		return secondary
+	}
+	if secondary == nil {
+		return primary
+	}
+	return fallbackProvider{primary: primary, secondary: secondary}
+}
+
+func (f fallbackProvider) Name() string { return f.primary.Name() }
+
+func (f fallbackProvider) Fetch(ctx context.Context, zone string, day time.Time) ([]RawPrice, error) {
+	rows, err := f.primary.Fetch(ctx, zone, day)
+	if err == nil && len(rows) > 0 {
+		return rows, nil
+	}
+	rows2, err2 := f.secondary.Fetch(ctx, zone, day)
+	if err2 != nil {
+		if err != nil {
+			return nil, err
+		}
+		return nil, err2
+	}
+	if len(rows2) > 0 {
+		slog.Info("price: primary empty, using Nord Pool day-ahead",
+			"primary", f.primary.Name(),
+			"zone", zone,
+			"day", day.Format("2006-01-02"),
+			"slots", len(rows2))
+	}
+	return rows2, nil
+}
diff --git a/go/internal/prices/nordpool.go b/go/internal/prices/nordpool.go
new file mode 100644
index 000000000..343cd1b09
--- /dev/null
+++ b/go/internal/prices/nordpool.go
@@ -0,0 +1,147 @@
+package prices
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+// NordPoolProvider reads the public Nord Pool day-ahead dataportal.
+// No API key. Used when the Sourceful harvest has not yet stored a day
+// that Nord Pool has already published.
+type NordPoolProvider struct {
+	Client   *http.Client
+	BaseURL  string
+	Currency string
+	FX       FXConverter
+}
+
+const nordPoolDefaultURL = "https://dataportal-api.nordpoolgroup.com/api/DayAheadPrices"
+
+func NewNordPool() *NordPoolProvider {
+	return &NordPoolProvider{
+		Client:  &http.Client{Timeout: 15 * time.Second},
+		BaseURL: nordPoolDefaultURL,
+	}
+}
+
+func (n *NordPoolProvider) Name() string { return "nordpool" }
+
+func (n *NordPoolProvider) apiCurrency() string {
+	want := strings.ToUpper(strings.TrimSpace(n.Currency))
+	switch want {
+	case "SEK", "EUR", "NOK", "DKK", "GBP":
+		return want
+	default:
+		if want == "" {
+			return "SEK"
+		}
+		return "EUR"
+	}
+}
+
+func (n *NordPoolProvider) Fetch(ctx context.Context, zone string, day time.Time) ([]RawPrice, error) {
+	zone = strings.ToUpper(strings.TrimSpace(zone))
+	if zone == "" {
+		zone = "SE3"
+	}
+	loc, err := time.LoadLocation("Europe/Stockholm")
+	if err != nil {
+		loc = time.UTC
+	}
+	date := day.In(loc).Format("2006-01-02")
+	apiCur := n.apiCurrency()
+	base := n.BaseURL
+	if base == "" {
+		base = nordPoolDefaultURL
+	}
+	endpoint := fmt.Sprintf("%s?date=%s&market=DayAhead&deliveryArea=%s&currency=%s",
+		base, url.QueryEscape(date), url.QueryEscape(zone), url.QueryEscape(apiCur))
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "ftw (https://github.com/srcfl/ftw)")
+	client := n.Client
+	if client == nil {
+		client = http.DefaultClient
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusNotFound {
+		return nil, nil
+	}
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
+		return nil, fmt.Errorf("nordpool: status %d: %s", resp.StatusCode, string(body))
+	}
+	var payload struct {
+		Currency         string `json:"currency"`
+		MultiAreaEntries []struct {
+			DeliveryStart string             `json:"deliveryStart"`
+			DeliveryEnd   string             `json:"deliveryEnd"`
+			EntryPerArea  map[string]float64 `json:"entryPerArea"`
+		} `json:"multiAreaEntries"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
+		return nil, fmt.Errorf("nordpool: decode: %w", err)
+	}
+	if payload.Currency != "" && !strings.EqualFold(payload.Currency, apiCur) {
+		return nil, fmt.Errorf("nordpool: asked for %s, got %q", apiCur, payload.Currency)
+	}
+	want := strings.ToUpper(strings.TrimSpace(n.Currency))
+	if want == "" {
+		want = "SEK"
+	}
+	convert := want != apiCur
+	if convert && n.FX == nil {
+		return nil, fmt.Errorf("nordpool: no exchange rate source for %s→%s", apiCur, want)
+	}
+	out := make([]RawPrice, 0, len(payload.MultiAreaEntries))
+	for _, e := range payload.MultiAreaEntries {
+		price, ok := e.EntryPerArea[zone]
+		if !ok {
+			for k, v := range e.EntryPerArea {
+				if strings.EqualFold(k, zone) {
+					price, ok = v, true
+					break
+				}
+			}
+		}
+		if !ok {
+			continue
+		}
+		start, err := time.Parse(time.RFC3339, e.DeliveryStart)
+		if err != nil {
+			return nil, fmt.Errorf("nordpool: deliveryStart %q: %w", e.DeliveryStart, err)
+		}
+		slotMin := 15
+		if e.DeliveryEnd != "" {
+			if end, err := time.Parse(time.RFC3339, e.DeliveryEnd); err == nil {
+				d := int(end.Sub(start).Minutes())
+				if d >= 5 && d <= 120 {
+					slotMin = d
+				}
+			}
+		}
+		perKWh := price / 1000.0
+		if convert {
+			native, ok := n.FX.Convert(perKWh, apiCur, want)
+			if !ok {
+				return nil, fmt.Errorf("nordpool: no %s→%s rate yet", apiCur, want)
+			}
+			perKWh = native
+		}
+		out = append(out, RawPrice{SlotStart: start, SlotLenMin: slotMin, SEKPerKWh: perKWh})
+	}
+	return out, nil
+}
diff --git a/go/internal/prices/nordpool_test.go b/go/internal/prices/nordpool_test.go
new file mode 100644
index 000000000..ef7ca3062
--- /dev/null
+++ b/go/internal/prices/nordpool_test.go
@@ -0,0 +1,138 @@
+package prices
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestNordPoolParsesDayAhead(t *testing.T) {
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("date") != "2026-09-01" {
+			t.Errorf("date = %s", r.URL.Query().Get("date"))
+		}
+		if r.URL.Query().Get("deliveryArea") != "SE3" {
+			t.Errorf("area = %s", r.URL.Query().Get("deliveryArea"))
+		}
+		if r.Header.Get("User-Agent") == "" {
+			t.Error("User-Agent required")
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"currency": "SEK",
+			"multiAreaEntries": []map[string]any{
+				{
+					"deliveryStart": "2026-08-31T22:00:00Z",
+					"deliveryEnd":   "2026-08-31T22:15:00Z",
+					"entryPerArea":  map[string]float64{"SE3": 1243.02},
+				},
+				{
+					"deliveryStart": "2026-08-31T22:15:00Z",
+					"deliveryEnd":   "2026-08-31T22:30:00Z",
+					"entryPerArea":  map[string]float64{"SE3": 1083.3},
+				},
+			},
+		})
+	})
+	srv := httptest.NewServer(handler)
+	defer srv.Close()
+	p := &NordPoolProvider{Client: srv.Client(), BaseURL: srv.URL, Currency: "SEK"}
+	day := time.Date(2026, 9, 1, 8, 0, 0, 0, time.FixedZone("CEST", 2*3600))
+	rows, err := p.Fetch(context.Background(), "se3", day)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rows) != 2 {
+		t.Fatalf("got %d rows", len(rows))
+	}
+	if rows[0].SlotLenMin != 15 {
+		t.Errorf("slot = %d", rows[0].SlotLenMin)
+	}
+	if rows[0].SEKPerKWh < 1.24 || rows[0].SEKPerKWh > 1.25 {
+		t.Errorf("SEK/kWh = %g", rows[0].SEKPerKWh)
+	}
+}
+
+func TestNordPoolUnpublishedDay(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer srv.Close()
+	p := &NordPoolProvider{Client: srv.Client(), BaseURL: srv.URL, Currency: "SEK"}
+	rows, err := p.Fetch(context.Background(), "SE3", time.Now())
+	if err != nil {
+		t.Fatalf("404 should not error: %v", err)
+	}
+	if len(rows) != 0 {
+		t.Errorf("got %d", len(rows))
+	}
+}
+
+func TestSourcefulFallsBackToNordPool(t *testing.T) {
+	primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer primary.Close()
+	secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"currency": "SEK",
+			"multiAreaEntries": []map[string]any{{
+				"deliveryStart": "2026-08-31T22:00:00Z",
+				"deliveryEnd":   "2026-08-31T22:15:00Z",
+				"entryPerArea":  map[string]float64{"SE3": 1000},
+			}},
+		})
+	}))
+	defer secondary.Close()
+	p := withFallback(
+		&SourcefulProvider{Client: primary.Client(), BaseURL: primary.URL},
+		&NordPoolProvider{Client: secondary.Client(), BaseURL: secondary.URL, Currency: "SEK"},
+	)
+	if p.Name() != "sourceful" {
+		t.Errorf("name = %s", p.Name())
+	}
+	rows, err := p.Fetch(context.Background(), "SE3", time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rows) != 1 {
+		t.Fatalf("got %d, want Nord Pool row", len(rows))
+	}
+}
+
+func TestNextDayAheadCatch(t *testing.T) {
+	loc, err := time.LoadLocation("Europe/Stockholm")
+	if err != nil {
+		t.Fatal(err)
+	}
+	before := time.Date(2026, 8, 31, 12, 0, 0, 0, loc)
+	got := nextDayAheadCatch(before)
+	want := time.Date(2026, 8, 31, 13, 5, 0, 0, loc)
+	if !got.Equal(want) {
+		t.Errorf("before publication: got %s want %s", got, want)
+	}
+	after := time.Date(2026, 8, 31, 13, 6, 0, 0, loc)
+	got = nextDayAheadCatch(after)
+	want = time.Date(2026, 9, 1, 13, 5, 0, 0, loc)
+	if !got.Equal(want) {
+		t.Errorf("after publication: got %s want %s", got, want)
+	}
+}
+
+func TestNordPoolRejectsCurrencyMismatch(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"currency":         "EUR",
+			"multiAreaEntries": []any{},
+		})
+	}))
+	defer srv.Close()
+	p := &NordPoolProvider{Client: srv.Client(), BaseURL: srv.URL, Currency: "SEK"}
+	_, err := p.Fetch(context.Background(), "SE3", time.Now())
+	if err == nil || !strings.Contains(err.Error(), "asked for SEK") {
+		t.Fatalf("err = %v", err)
+	}
+}
diff --git a/go/internal/prices/prices.go b/go/internal/prices/prices.go
index 3defa4963..e47bbb63f 100644
--- a/go/internal/prices/prices.go
+++ b/go/internal/prices/prices.go
@@ -5,10 +5,12 @@
 //   - sourceful — Default. Keyless European day-ahead prices through
 //     Sourceful's cached ENTSO-E API, for every zone in zones.go.
 //     Resolution varies per bidding zone (currently 15m in most of Europe).
+//     When that harvest has not stored a day Nord Pool has already
+//     published, Fetch falls back to Nord Pool's public dataportal.
 //   - elprisetjustnu — Sweden, zones SE1-SE4, no API key. Since late 2025
 //     NordPool publishes in 15-minute PTU (quarterly) resolution; this
 //     package defaults to the quarterly endpoint and can fall back to
-//     hourly if the provider returns that.
+//     hourly if the provider returns that. Same Nord Pool fallback.
 //   - entsoe — All EU, needs ENTSO-E transparency platform API key.
 //     Resolution varies per bidding zone (15m or 60m).
 //
@@ -574,15 +576,18 @@ func FromConfig(cfg *config.Price, st *state.Store, fx FXConverter) *Service {
 	if currency == "" {
 		currency = "SEK"
 	}
+	np := NewNordPool()
+	np.Currency = currency
+	np.FX = fx
 	var p Provider
 	switch cfg.Provider {
 	case "sourceful":
 		sp := NewSourceful()
 		sp.Currency = currency
 		sp.FX = fx
-		p = sp
+		p = withFallback(sp, np)
 	case "elprisetjustnu":
-		p = NewElpriser()
+		p = withFallback(NewElpriser(), np)
 	case "entsoe":
 		ep := NewENTSOE(cfg.APIKey)
 		ep.Currency = currency
@@ -652,20 +657,41 @@ func (s *Service) loop(ctx context.Context) {
 	defer close(s.done)
 	// Initial fetch (today + tomorrow in case day-ahead is already published)
 	s.fetchAndStore(ctx)
-	t := time.NewTicker(time.Hour)
-	defer t.Stop()
+	hourly := time.NewTicker(time.Hour)
+	defer hourly.Stop()
+	catch := time.NewTimer(time.Until(nextDayAheadCatch(time.Now())))
+	defer catch.Stop()
 	for {
 		select {
 		case <-s.stop:
 			return
 		case <-ctx.Done():
 			return
-		case <-t.C:
+		case <-hourly.C:
 			s.fetchAndStore(ctx)
+		case <-catch.C:
+			s.fetchAndStore(ctx)
+			catch.Reset(time.Until(nextDayAheadCatch(time.Now().Add(time.Minute))))
 		}
 	}
 }
 
+// nextDayAheadCatch is 13:05 Europe/Stockholm, when tomorrow's Nord Pool
+// day-ahead is normally on the dataportal. Hourly ticks alone can miss
+// that window for up to an hour.
+func nextDayAheadCatch(now time.Time) time.Time {
+	loc, err := time.LoadLocation("Europe/Stockholm")
+	if err != nil {
+		loc = time.FixedZone("CET", 3600)
+	}
+	now = now.In(loc)
+	target := time.Date(now.Year(), now.Month(), now.Day(), 13, 5, 0, 0, loc)
+	if !now.Before(target) {
+		target = target.Add(24 * time.Hour)
+	}
+	return target
+}
+
 func (s *Service) fetchAndStore(ctx context.Context) {
 	now := time.Now()
 	for _, offset := range []int{0, 1} { // today + tomorrow