From 451bc69ee20dbafda319e6377f1754cc322fa03c Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Fri, 16 Oct 2026 17:11:58 +0200 Subject: [PATCH 1/2] fix(Raw): Remove open interest from raw data channels --- README.md | 3 +-- cli_raw.go | 11 +++++------ raw.go | 4 ++-- raw_test.go | 17 ++++++++++++++++- types.go | 11 +++++------ 5 files changed, 29 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 6d099e2..f804f88 100644 --- a/README.md +++ b/README.md @@ -207,9 +207,8 @@ Raw trades, top-of-book quotes and derivative ticks for Binance, OKX and Hyperli | `mark_price` | Mark price, one row per change | yes | yes | yes | | `index_price` | Index price, one row per change | yes | yes | yes | | `funding_rate` | Funding rate and next funding time | yes | yes | yes | -| `open_interest` | Open interest, one row per change | yes | yes | yes | -Hyperliquid's derivative feed carries no exchange time, so in its `mark_price`, `index_price`, `funding_rate` and `open_interest` files `exchange_timestamp` is modelled, and an `exchange_timestamp_kind` column (`"modelled"`) follows it. Raw L2 order books are not offered. +Hyperliquid's derivative feed carries no exchange time, so in its `mark_price`, `index_price` and `funding_rate` files `exchange_timestamp` is modelled, and an `exchange_timestamp_kind` column (`"modelled"`) follows it. Raw L2 order books are not offered. Every file starts with `exchange_timestamp` (the venue's time) and `local_timestamp` (when the event reached our capture machine). Timestamps are UTC. diff --git a/cli_raw.go b/cli_raw.go index d53bdd7..8f0c1aa 100644 --- a/cli_raw.go +++ b/cli_raw.go @@ -14,12 +14,11 @@ import ( ) var rawDatasetDescriptions = map[RawDataset]string{ - RawTrades: "Every trade: id, taker side, price, amount", - RawQuotes: "Top of book: best bid/ask price and amount", - RawMarkPrice: "Mark price, one row per change", - RawIndexPrice: "Index price, one row per change", - RawFundingRate: "Funding rate and next funding time", - RawOpenInterest: "Open interest, one row per change", + RawTrades: "Every trade: id, taker side, price, amount", + RawQuotes: "Top of book: best bid/ask price and amount", + RawMarkPrice: "Mark price, one row per change", + RawIndexPrice: "Index price, one row per change", + RawFundingRate: "Funding rate and next funding time", } type rawDownloadFlags struct { diff --git a/raw.go b/raw.go index 4a0c66f..e5ff5fc 100644 --- a/raw.go +++ b/raw.go @@ -43,7 +43,7 @@ const ( var ( // RawDatasets lists every raw dataset in catalog order. - RawDatasets = []RawDataset{RawTrades, RawQuotes, RawMarkPrice, RawIndexPrice, RawFundingRate, RawOpenInterest} + RawDatasets = []RawDataset{RawTrades, RawQuotes, RawMarkPrice, RawIndexPrice, RawFundingRate} // Exchanges lists every supported exchange. Exchanges = []Exchange{ExchangeBinanceFutures, ExchangeOkxPerps, ExchangeHyperliquidPerps} @@ -69,7 +69,7 @@ var ( ) // RawExchanges returns the exchanges a raw dataset is served for: every one. -// On Hyperliquid, whose feed carries no exchange time, the four derivative +// On Hyperliquid, whose feed carries no exchange time, the three derivative // datasets have a modelled exchange_timestamp and an exchange_timestamp_kind // column. func RawExchanges(RawDataset) []Exchange { diff --git a/raw_test.go b/raw_test.go index 464832d..bf74642 100644 --- a/raw_test.go +++ b/raw_test.go @@ -313,6 +313,14 @@ func TestRawFilePath(t *testing.T) { } func TestValidateRawQuery(t *testing.T) { + want := []RawDataset{RawTrades, RawQuotes, RawMarkPrice, RawIndexPrice, RawFundingRate} + if !slices.Equal(RawDatasets, want) { + t.Fatalf("expected five raw datasets, got %v", RawDatasets) + } + if err := ValidateRawQuery("open_interest", ExchangeBinanceFutures); err == nil { + t.Error("expected raw open interest to be rejected") + } + // The API serves every raw dataset on every exchange, Hyperliquid's // derivative datasets included (with a modelled exchange_timestamp). for _, dataset := range RawDatasets { @@ -323,7 +331,7 @@ func TestValidateRawQuery(t *testing.T) { } } - if err := ValidateRawQuery("l2_book", ExchangeBinanceFutures); err == nil || !strings.Contains(err.Error(), "open_interest") { + if err := ValidateRawQuery("l2_book", ExchangeBinanceFutures); err == nil || !strings.Contains(err.Error(), "funding_rate") { t.Errorf("expected an unknown dataset to list the valid ones, got %v", err) } if err := ValidateRawQuery(RawTrades, "bybit"); err == nil || !strings.Contains(err.Error(), "okx-perps") { @@ -706,6 +714,7 @@ func TestCLI_Raw_ValidatesBeforeCallingTheAPI(t *testing.T) { want string }{ {"no dataset", []string{"raw", "--symbol", rawTestSymbol, "--output-dir", outputDir}, "raw dataset is required"}, + {"raw open interest", append([]string{"raw", "open_interest", "--symbol", rawTestSymbol, "--output-dir", outputDir}, dates...), `unknown raw dataset "open_interest"`}, {"unknown dataset", append([]string{"raw", "book", "--symbol", rawTestSymbol, "--output-dir", outputDir}, dates...), `unknown raw dataset "book"`}, {"unknown exchange", append([]string{"raw", "trades", "--exchange", "bybit", "--symbol", rawTestSymbol, "--output-dir", outputDir}, dates...), `unknown exchange "bybit"`}, {"missing symbol", append([]string{"raw", "trades", "--output-dir", outputDir}, dates...), "--symbol is required"}, @@ -750,6 +759,9 @@ func TestCLI_Raw_RequiresAKeyWithoutPreview(t *testing.T) { func TestCLI_Raw_Help(t *testing.T) { for _, args := range [][]string{{"raw"}, {"raw", "help"}, {"raw", "--help"}, {"raw", "trades", "-h"}} { stdout, _, code := runCLI(args...) + if strings.Contains(stdout, "open_interest") { + t.Errorf("raw help must not list open interest: %s", stdout) + } if code != 0 { t.Errorf("%v: expected exit code 0, got %d", args, code) } @@ -761,6 +773,9 @@ func TestCLI_Raw_Help(t *testing.T) { } stdout, _, _ := runCLI("help") + if !strings.Contains(stdout, string(MetricOpenInterest)) { + t.Error("expected open-interest metrics to remain in top-level help") + } if !strings.Contains(stdout, "aperiodic raw ") { t.Error("expected the top-level help to list the raw command") } diff --git a/types.go b/types.go index 94b8f4e..7897e1a 100644 --- a/types.go +++ b/types.go @@ -100,12 +100,11 @@ type SymbolsResponse struct { type RawDataset string const ( - RawTrades RawDataset = "trades" - RawQuotes RawDataset = "quotes" - RawMarkPrice RawDataset = "mark_price" - RawIndexPrice RawDataset = "index_price" - RawFundingRate RawDataset = "funding_rate" - RawOpenInterest RawDataset = "open_interest" + RawTrades RawDataset = "trades" + RawQuotes RawDataset = "quotes" + RawMarkPrice RawDataset = "mark_price" + RawIndexPrice RawDataset = "index_price" + RawFundingRate RawDataset = "funding_rate" ) type RawFileInfo struct { From 33b602aee8eb54399d3bbe29f3c212e66025f153 Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Thu, 15 Oct 2026 21:15:32 +0200 Subject: [PATCH 2/2] test: accept empty raw preview responses --- raw_live_test.go | 11 +++++++++++ raw_test.go | 31 ++++++++++++++++++++++++------- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/raw_live_test.go b/raw_live_test.go index b8eb75f..9aef6c3 100644 --- a/raw_live_test.go +++ b/raw_live_test.go @@ -91,6 +91,17 @@ func TestCLI_Raw_Live_PreviewNeedsNoKey(t *testing.T) { if code != 0 { t.Fatalf("expected exit code 0, got %d; stderr: %s", code, stderr) } + // Raw previews may currently have no files, which is a successful response. + if strings.Contains(stdout, "No data found for the given criteria") { + entries, err := os.ReadDir(outputDir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Errorf("expected no files for an empty preview, got %v", entries) + } + return + } if !strings.Contains(stdout, "Successfully downloaded 1 files") { t.Errorf("expected one file downloaded, got: %s", stdout) } diff --git a/raw_test.go b/raw_test.go index bf74642..ad2f198 100644 --- a/raw_test.go +++ b/raw_test.go @@ -692,14 +692,31 @@ func TestCLI_Raw_ReportsMissingPeriods(t *testing.T) { } func TestCLI_Raw_NoFiles(t *testing.T) { - newRawStub(t, staticListing()) + for _, preview := range []bool{false, true} { + t.Run(fmt.Sprintf("preview=%t", preview), func(t *testing.T) { + stub := newRawStub(t, staticListing()) + args := rawArgs(t.TempDir()) + if preview { + t.Setenv("APERIODIC_API_KEY", "") + args = append(args, "--preview") + } else { + args = append(args, "--start-date", "2025-06-01", "--end-date", "2025-06-30") + } - stdout, stderr, code := runCLI(rawArgs(t.TempDir(), "--start-date", "2025-06-01", "--end-date", "2025-06-30")...) - if code != 0 { - t.Fatalf("expected exit code 0, got %d; stderr: %s", code, stderr) - } - if !strings.Contains(stdout, "No data found") { - t.Errorf("expected a no-data message, got: %s", stdout) + stdout, stderr, code := runCLI(args...) + if code != 0 { + t.Fatalf("expected exit code 0, got %d; stderr: %s", code, stderr) + } + if !strings.Contains(stdout, "No data found") { + t.Errorf("expected a no-data message, got: %s", stdout) + } + if preview { + calls := stub.calls() + if len(calls) != 1 || calls[0].path != "/data/raw/preview/trades" || calls[0].key != DemoAPIKey { + t.Errorf("expected one preview request with the demo key, got %v", calls) + } + } + }) } }