fix(storage): prefer compact data sets over legacy ones

packages/synapse-core/src/chains.ts

@@ -21,6 +21,10 @@ export interface FilecoinChain extends ViemChain {
    * The genesis timestamp of the chain in seconds (Unix timestamp)
    */
   genesisTimestamp: number
+  /**
+   * First data set ID using compact piece storage. Lower IDs use legacy storage.
+   */
+  legacyPieceStorageIdLimit: bigint
   /**
    * The contracts of the chain
    */
@@ -154,6 +158,7 @@ export const mainnet: FilecoinChain = {
    * Filecoin Mainnet genesis: August 24, 2020 22:00:00 UTC
    */
   genesisTimestamp: 1598306400,
+  legacyPieceStorageIdLimit: 1559n,
 }
 
 /**
@@ -247,6 +252,7 @@ export const calibration: FilecoinChain = {
    * Filecoin Calibration testnet genesis: November 1, 2022 18:13:00 UTC
    */
   genesisTimestamp: 1667326380,
+  legacyPieceStorageIdLimit: 32331n,
 }
 
 /**
@@ -320,6 +326,8 @@ export const devnet: FilecoinChain = {
    * are unaffected as they use epochs directly.
    */
   genesisTimestamp: 0,
+  // A fresh devnet has no legacy history, so every data set is compact.
+  legacyPieceStorageIdLimit: 1n,
 }
 
 export namespace getChain {
@@ -366,6 +374,8 @@ export function asChain(chain: ViemChain): FilecoinChain {
     'filecoinPay' in chain.contracts &&
     'fwss' in chain.contracts &&
     'genesisTimestamp' in chain &&
+    'legacyPieceStorageIdLimit' in chain &&
+    typeof (chain as Partial<FilecoinChain>).legacyPieceStorageIdLimit === 'bigint' &&
     [mainnet.id, calibration.id, devnet.id].includes(chain.id)
   ) {
     return chain as FilecoinChain

packages/synapse-core/src/devnet/index.ts

@@ -104,6 +104,8 @@ export function toChain(devnetInfo: VersionedDevnetInfo): FilecoinChain {
      * are unaffected as they use epochs directly.
      */
     genesisTimestamp: 0,
+    // A fresh devnet has no legacy history, so every data set is compact.
+    legacyPieceStorageIdLimit: 1n,
   }
 }
 

packages/synapse-core/src/warm-storage/fetch-provider-selection-input.ts

@@ -1,4 +1,5 @@
 import type { Address, Chain, Client, Transport } from 'viem'
+import { asChain } from '../chains.ts'
 import { getEndorsedProviderIds } from '../endorsements/get-endorsed-provider-ids.ts'
 import { paginate } from '../pagination.ts'
 import { getApprovedPDPProviders } from '../sp-registry/get-pdp-providers.ts'
@@ -43,5 +44,6 @@ export async function fetchProviderSelectionInput(
     providers,
     endorsedIds,
     clientDataSets: pdpDataSets,
+    legacyPieceStorageIdLimit: asChain(client.chain).legacyPieceStorageIdLimit,
   }
 }

packages/synapse-core/src/warm-storage/find-matching-data-sets.ts

@@ -41,21 +41,34 @@ export function metadataMatches(dataSetMetadata: MetadataObject, requestedMetada
  * Only active datasets are considered (live, managed, pdpEndEpoch === 0n).
  *
  * Sort order:
- *   1. Datasets with pieces before empty datasets
- *   2. Within each group, older datasets (lower ID) first
+ *   1. Compact datasets (dataSetId >= legacyPieceStorageIdLimit) before legacy ones
+ *   2. Within each group, datasets with pieces before empty datasets
+ *   3. Within each group, older datasets (lower ID) first
  *
  * @param dataSets - Datasets to search (typically filtered to a single provider)
  * @param metadata - Desired metadata keys and values
+ * @param legacyPieceStorageIdLimit - Data set ID boundary between legacy and
+ *   compact piece storage (see `FilecoinChain.legacyPieceStorageIdLimit`).
+ *   Defaults to 0n, which treats every dataset as compact.
  * @returns Matching datasets in preference order
  */
-export function findMatchingDataSets(dataSets: SelectionDataSet[], metadata: MetadataObject): SelectionDataSet[] {
+export function findMatchingDataSets(
+  dataSets: SelectionDataSet[],
+  metadata: MetadataObject,
+  legacyPieceStorageIdLimit: bigint = 0n
+): SelectionDataSet[] {
   const matching = dataSets.filter(
     (ds) => ds.live && ds.managed && ds.pdpEndEpoch === 0n && metadataMatches(ds.metadata, metadata)
   )
 
   return matching.sort((a, b) => {
+    const aCompact = a.dataSetId >= legacyPieceStorageIdLimit
+    const bCompact = b.dataSetId >= legacyPieceStorageIdLimit
+    if (aCompact !== bCompact) return aCompact ? -1 : 1
     if (a.hasActivePieces && !b.hasActivePieces) return -1
     if (b.hasActivePieces && !a.hasActivePieces) return 1
-    return Number(a.dataSetId - b.dataSetId)
+    if (a.dataSetId < b.dataSetId) return -1
+    if (a.dataSetId > b.dataSetId) return 1
+    return 0
   })
 }

packages/synapse-core/src/warm-storage/location-types.ts

@@ -46,6 +46,13 @@ export interface ProviderSelectionInput {
   endorsedIds: bigint[]
   /** Client's existing datasets with metadata and piece-presence information */
   clientDataSets: SelectionDataSet[]
+  /**
+   * Data set ID boundary between legacy and compact piece storage (see
+   * `FilecoinChain.legacyPieceStorageIdLimit`). Data sets at or above this ID
+   * are compact and are preferred over legacy ones. Defaults to 0n (every
+   * data set treated as compact) when omitted.
+   */
+  legacyPieceStorageIdLimit?: bigint
 }
 
 /**

packages/synapse-core/src/warm-storage/select-providers.ts

@@ -12,8 +12,9 @@ import type { ProviderSelectionOptions, ResolvedLocation } from './location-type
  * dataset are preferred (reuses payment rail). Otherwise a provider
  * without a matching dataset is selected (new dataset created on commit).
  *
- * Within matching datasets, those with existing pieces sort before empty
- * ones, and older datasets (lower ID) sort before newer ones.
+ * Within matching datasets, compact datasets sort before legacy ones, then
+ * those with existing pieces sort before empty ones, and older datasets
+ * (lower ID) sort before newer ones.
  *
  * This function does NOT perform health checks — the caller should
  * validate reachability via SP.ping() and call again with
@@ -43,7 +44,7 @@ export function selectProviders(options: ProviderSelectionOptions): ResolvedLoca
 
   // Find metadata-matching datasets from eligible providers
   const eligibleDataSets = options.clientDataSets.filter((ds) => providerMap.has(ds.providerId))
-  const matchingDataSets = findMatchingDataSets(eligibleDataSets, metadata)
+  const matchingDataSets = findMatchingDataSets(eligibleDataSets, metadata, options.legacyPieceStorageIdLimit)
 
   const results: ResolvedLocation[] = []
   const selectedProviderIds: bigint[] = []

packages/synapse-core/test/chains.test.ts

@@ -86,5 +86,23 @@ describe('chains', () => {
         (err: unknown) => UnsupportedChainError.is(err)
       )
     })
+
+    it('should throw for a chain missing legacyPieceStorageIdLimit', () => {
+      const { legacyPieceStorageIdLimit, ...chainWithoutLimit } = mainnet
+
+      assert.throws(
+        () => asChain(chainWithoutLimit as ViemChain),
+        (err: unknown) => UnsupportedChainError.is(err)
+      )
+    })
+
+    it('should throw for a chain with a non-bigint legacyPieceStorageIdLimit', () => {
+      const badChain = { ...mainnet, legacyPieceStorageIdLimit: 1559 }
+
+      assert.throws(
+        () => asChain(badChain as unknown as ViemChain),
+        (err: unknown) => UnsupportedChainError.is(err)
+      )
+    })
   })
 })

packages/synapse-core/test/find-matching-data-sets.test.ts

@@ -147,4 +147,42 @@ describe('findMatchingDataSets', () => {
       [5n, 8n, 3n, 10n]
     )
   })
+
+  it('prefers compact datasets over legacy ones, even when the legacy one has pieces', () => {
+    const dataSets = [
+      makeDataSet({ dataSetId: 1n, providerId: 1n, metadata: { source: 'app' }, hasActivePieces: true }),
+      makeDataSet({ dataSetId: 5n, providerId: 2n, metadata: { source: 'app' }, hasActivePieces: false }),
+    ]
+    const result = findMatchingDataSets(dataSets, { source: 'app' }, 5n)
+    assert.deepEqual(
+      result.map((ds) => ds.dataSetId),
+      [5n, 1n]
+    )
+  })
+
+  it('applies piece presence and ID ordering within the compact and legacy groups', () => {
+    const dataSets = [
+      makeDataSet({ dataSetId: 20n, providerId: 1n, metadata: { source: 'app' }, hasActivePieces: false }),
+      makeDataSet({ dataSetId: 10n, providerId: 2n, metadata: { source: 'app' }, hasActivePieces: true }),
+      makeDataSet({ dataSetId: 2n, providerId: 3n, metadata: { source: 'app' }, hasActivePieces: false }),
+      makeDataSet({ dataSetId: 1n, providerId: 4n, metadata: { source: 'app' }, hasActivePieces: true }),
+    ]
+    const result = findMatchingDataSets(dataSets, { source: 'app' }, 10n)
+    assert.deepEqual(
+      result.map((ds) => ds.dataSetId),
+      [10n, 20n, 1n, 2n]
+    )
+  })
+
+  it('treats every dataset as compact when legacyPieceStorageIdLimit is omitted', () => {
+    const dataSets = [
+      makeDataSet({ dataSetId: 10n, providerId: 1n, metadata: { source: 'app' }, hasActivePieces: true }),
+      makeDataSet({ dataSetId: 5n, providerId: 2n, metadata: { source: 'app' }, hasActivePieces: true }),
+    ]
+    const result = findMatchingDataSets(dataSets, { source: 'app' })
+    assert.deepEqual(
+      result.map((ds) => ds.dataSetId),
+      [5n, 10n]
+    )
+  })
 })

packages/synapse-core/test/select-providers.test.ts

@@ -336,6 +336,30 @@ describe('selectProviders', () => {
       })
       assert.equal(result[0].dataSetId, 5n)
     })
+
+    it('prefers a compact dataset over a legacy one with pieces when legacyPieceStorageIdLimit is set', () => {
+      const result = selectProviders({
+        providers: [provider1],
+        endorsedIds: [],
+        clientDataSets: [
+          makeDataSet({
+            dataSetId: 5n,
+            providerId: 1n,
+            metadata: { source: 'app' },
+            hasActivePieces: true,
+          }),
+          makeDataSet({
+            dataSetId: 10n,
+            providerId: 1n,
+            metadata: { source: 'app' },
+            hasActivePieces: false,
+          }),
+        ],
+        metadata: { source: 'app' },
+        legacyPieceStorageIdLimit: 10n,
+      })
+      assert.equal(result[0].dataSetId, 10n)
+    })
   })
 
   describe('metadata filtering', () => {