diff --git a/packages/synapse-core/src/chains.ts b/packages/synapse-core/src/chains.ts index 54bb649fa..5d0741590 100644 --- a/packages/synapse-core/src/chains.ts +++ b/packages/synapse-core/src/chains.ts @@ -21,6 +21,10 @@ export interface FilecoinChain extends ViemChain { * The genesis timestamp of the chain in seconds (Unix timestamp) */ genesisTimestamp: number + /** + * First data set ID using compact piece storage. Lower IDs use legacy storage. + */ + legacyPieceStorageIdLimit: bigint /** * The contracts of the chain */ @@ -154,6 +158,7 @@ export const mainnet: FilecoinChain = { * Filecoin Mainnet genesis: August 24, 2020 22:00:00 UTC */ genesisTimestamp: 1598306400, + legacyPieceStorageIdLimit: 1559n, } /** @@ -247,6 +252,7 @@ export const calibration: FilecoinChain = { * Filecoin Calibration testnet genesis: November 1, 2022 18:13:00 UTC */ genesisTimestamp: 1667326380, + legacyPieceStorageIdLimit: 32331n, } /** @@ -320,6 +326,8 @@ export const devnet: FilecoinChain = { * are unaffected as they use epochs directly. */ genesisTimestamp: 0, + // A fresh devnet has no legacy history, so every data set is compact. + legacyPieceStorageIdLimit: 1n, } export namespace getChain { @@ -366,6 +374,8 @@ export function asChain(chain: ViemChain): FilecoinChain { 'filecoinPay' in chain.contracts && 'fwss' in chain.contracts && 'genesisTimestamp' in chain && + 'legacyPieceStorageIdLimit' in chain && + typeof (chain as Partial).legacyPieceStorageIdLimit === 'bigint' && [mainnet.id, calibration.id, devnet.id].includes(chain.id) ) { return chain as FilecoinChain diff --git a/packages/synapse-core/src/devnet/index.ts b/packages/synapse-core/src/devnet/index.ts index d343b6386..bddca2eff 100644 --- a/packages/synapse-core/src/devnet/index.ts +++ b/packages/synapse-core/src/devnet/index.ts @@ -104,6 +104,8 @@ export function toChain(devnetInfo: VersionedDevnetInfo): FilecoinChain { * are unaffected as they use epochs directly. */ genesisTimestamp: 0, + // A fresh devnet has no legacy history, so every data set is compact. + legacyPieceStorageIdLimit: 1n, } } diff --git a/packages/synapse-core/src/warm-storage/fetch-provider-selection-input.ts b/packages/synapse-core/src/warm-storage/fetch-provider-selection-input.ts index 15adf48e6..5a66aa120 100644 --- a/packages/synapse-core/src/warm-storage/fetch-provider-selection-input.ts +++ b/packages/synapse-core/src/warm-storage/fetch-provider-selection-input.ts @@ -1,4 +1,5 @@ import type { Address, Chain, Client, Transport } from 'viem' +import { asChain } from '../chains.ts' import { getEndorsedProviderIds } from '../endorsements/get-endorsed-provider-ids.ts' import { paginate } from '../pagination.ts' import { getApprovedPDPProviders } from '../sp-registry/get-pdp-providers.ts' @@ -43,5 +44,6 @@ export async function fetchProviderSelectionInput( providers, endorsedIds, clientDataSets: pdpDataSets, + legacyPieceStorageIdLimit: asChain(client.chain).legacyPieceStorageIdLimit, } } diff --git a/packages/synapse-core/src/warm-storage/find-matching-data-sets.ts b/packages/synapse-core/src/warm-storage/find-matching-data-sets.ts index 373479e94..6e4c4ec75 100644 --- a/packages/synapse-core/src/warm-storage/find-matching-data-sets.ts +++ b/packages/synapse-core/src/warm-storage/find-matching-data-sets.ts @@ -41,21 +41,34 @@ export function metadataMatches(dataSetMetadata: MetadataObject, requestedMetada * Only active datasets are considered (live, managed, pdpEndEpoch === 0n). * * Sort order: - * 1. Datasets with pieces before empty datasets - * 2. Within each group, older datasets (lower ID) first + * 1. Compact datasets (dataSetId >= legacyPieceStorageIdLimit) before legacy ones + * 2. Within each group, datasets with pieces before empty datasets + * 3. Within each group, older datasets (lower ID) first * * @param dataSets - Datasets to search (typically filtered to a single provider) * @param metadata - Desired metadata keys and values + * @param legacyPieceStorageIdLimit - Data set ID boundary between legacy and + * compact piece storage (see `FilecoinChain.legacyPieceStorageIdLimit`). + * Defaults to 0n, which treats every dataset as compact. * @returns Matching datasets in preference order */ -export function findMatchingDataSets(dataSets: SelectionDataSet[], metadata: MetadataObject): SelectionDataSet[] { +export function findMatchingDataSets( + dataSets: SelectionDataSet[], + metadata: MetadataObject, + legacyPieceStorageIdLimit: bigint = 0n +): SelectionDataSet[] { const matching = dataSets.filter( (ds) => ds.live && ds.managed && ds.pdpEndEpoch === 0n && metadataMatches(ds.metadata, metadata) ) return matching.sort((a, b) => { + const aCompact = a.dataSetId >= legacyPieceStorageIdLimit + const bCompact = b.dataSetId >= legacyPieceStorageIdLimit + if (aCompact !== bCompact) return aCompact ? -1 : 1 if (a.hasActivePieces && !b.hasActivePieces) return -1 if (b.hasActivePieces && !a.hasActivePieces) return 1 - return Number(a.dataSetId - b.dataSetId) + if (a.dataSetId < b.dataSetId) return -1 + if (a.dataSetId > b.dataSetId) return 1 + return 0 }) } diff --git a/packages/synapse-core/src/warm-storage/location-types.ts b/packages/synapse-core/src/warm-storage/location-types.ts index 315fc2abb..90a343ebc 100644 --- a/packages/synapse-core/src/warm-storage/location-types.ts +++ b/packages/synapse-core/src/warm-storage/location-types.ts @@ -46,6 +46,13 @@ export interface ProviderSelectionInput { endorsedIds: bigint[] /** Client's existing datasets with metadata and piece-presence information */ clientDataSets: SelectionDataSet[] + /** + * Data set ID boundary between legacy and compact piece storage (see + * `FilecoinChain.legacyPieceStorageIdLimit`). Data sets at or above this ID + * are compact and are preferred over legacy ones. Defaults to 0n (every + * data set treated as compact) when omitted. + */ + legacyPieceStorageIdLimit?: bigint } /** diff --git a/packages/synapse-core/src/warm-storage/select-providers.ts b/packages/synapse-core/src/warm-storage/select-providers.ts index 890d4876e..42328fe54 100644 --- a/packages/synapse-core/src/warm-storage/select-providers.ts +++ b/packages/synapse-core/src/warm-storage/select-providers.ts @@ -12,8 +12,9 @@ import type { ProviderSelectionOptions, ResolvedLocation } from './location-type * dataset are preferred (reuses payment rail). Otherwise a provider * without a matching dataset is selected (new dataset created on commit). * - * Within matching datasets, those with existing pieces sort before empty - * ones, and older datasets (lower ID) sort before newer ones. + * Within matching datasets, compact datasets sort before legacy ones, then + * those with existing pieces sort before empty ones, and older datasets + * (lower ID) sort before newer ones. * * This function does NOT perform health checks — the caller should * validate reachability via SP.ping() and call again with @@ -43,7 +44,7 @@ export function selectProviders(options: ProviderSelectionOptions): ResolvedLoca // Find metadata-matching datasets from eligible providers const eligibleDataSets = options.clientDataSets.filter((ds) => providerMap.has(ds.providerId)) - const matchingDataSets = findMatchingDataSets(eligibleDataSets, metadata) + const matchingDataSets = findMatchingDataSets(eligibleDataSets, metadata, options.legacyPieceStorageIdLimit) const results: ResolvedLocation[] = [] const selectedProviderIds: bigint[] = [] diff --git a/packages/synapse-core/test/chains.test.ts b/packages/synapse-core/test/chains.test.ts index 7564085c0..70789ea9d 100644 --- a/packages/synapse-core/test/chains.test.ts +++ b/packages/synapse-core/test/chains.test.ts @@ -86,5 +86,23 @@ describe('chains', () => { (err: unknown) => UnsupportedChainError.is(err) ) }) + + it('should throw for a chain missing legacyPieceStorageIdLimit', () => { + const { legacyPieceStorageIdLimit, ...chainWithoutLimit } = mainnet + + assert.throws( + () => asChain(chainWithoutLimit as ViemChain), + (err: unknown) => UnsupportedChainError.is(err) + ) + }) + + it('should throw for a chain with a non-bigint legacyPieceStorageIdLimit', () => { + const badChain = { ...mainnet, legacyPieceStorageIdLimit: 1559 } + + assert.throws( + () => asChain(badChain as unknown as ViemChain), + (err: unknown) => UnsupportedChainError.is(err) + ) + }) }) }) diff --git a/packages/synapse-core/test/find-matching-data-sets.test.ts b/packages/synapse-core/test/find-matching-data-sets.test.ts index e1da3d7d5..fa42f9688 100644 --- a/packages/synapse-core/test/find-matching-data-sets.test.ts +++ b/packages/synapse-core/test/find-matching-data-sets.test.ts @@ -147,4 +147,42 @@ describe('findMatchingDataSets', () => { [5n, 8n, 3n, 10n] ) }) + + it('prefers compact datasets over legacy ones, even when the legacy one has pieces', () => { + const dataSets = [ + makeDataSet({ dataSetId: 1n, providerId: 1n, metadata: { source: 'app' }, hasActivePieces: true }), + makeDataSet({ dataSetId: 5n, providerId: 2n, metadata: { source: 'app' }, hasActivePieces: false }), + ] + const result = findMatchingDataSets(dataSets, { source: 'app' }, 5n) + assert.deepEqual( + result.map((ds) => ds.dataSetId), + [5n, 1n] + ) + }) + + it('applies piece presence and ID ordering within the compact and legacy groups', () => { + const dataSets = [ + makeDataSet({ dataSetId: 20n, providerId: 1n, metadata: { source: 'app' }, hasActivePieces: false }), + makeDataSet({ dataSetId: 10n, providerId: 2n, metadata: { source: 'app' }, hasActivePieces: true }), + makeDataSet({ dataSetId: 2n, providerId: 3n, metadata: { source: 'app' }, hasActivePieces: false }), + makeDataSet({ dataSetId: 1n, providerId: 4n, metadata: { source: 'app' }, hasActivePieces: true }), + ] + const result = findMatchingDataSets(dataSets, { source: 'app' }, 10n) + assert.deepEqual( + result.map((ds) => ds.dataSetId), + [10n, 20n, 1n, 2n] + ) + }) + + it('treats every dataset as compact when legacyPieceStorageIdLimit is omitted', () => { + const dataSets = [ + makeDataSet({ dataSetId: 10n, providerId: 1n, metadata: { source: 'app' }, hasActivePieces: true }), + makeDataSet({ dataSetId: 5n, providerId: 2n, metadata: { source: 'app' }, hasActivePieces: true }), + ] + const result = findMatchingDataSets(dataSets, { source: 'app' }) + assert.deepEqual( + result.map((ds) => ds.dataSetId), + [5n, 10n] + ) + }) }) diff --git a/packages/synapse-core/test/select-providers.test.ts b/packages/synapse-core/test/select-providers.test.ts index 11d142d76..9019b8e9f 100644 --- a/packages/synapse-core/test/select-providers.test.ts +++ b/packages/synapse-core/test/select-providers.test.ts @@ -336,6 +336,30 @@ describe('selectProviders', () => { }) assert.equal(result[0].dataSetId, 5n) }) + + it('prefers a compact dataset over a legacy one with pieces when legacyPieceStorageIdLimit is set', () => { + const result = selectProviders({ + providers: [provider1], + endorsedIds: [], + clientDataSets: [ + makeDataSet({ + dataSetId: 5n, + providerId: 1n, + metadata: { source: 'app' }, + hasActivePieces: true, + }), + makeDataSet({ + dataSetId: 10n, + providerId: 1n, + metadata: { source: 'app' }, + hasActivePieces: false, + }), + ], + metadata: { source: 'app' }, + legacyPieceStorageIdLimit: 10n, + }) + assert.equal(result[0].dataSetId, 10n) + }) }) describe('metadata filtering', () => { diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 8016a3f0d..39ab889b0 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -289,7 +289,8 @@ export class StorageContext { providerId, options.metadata ?? {}, options.warmStorageService, - spRegistry + spRegistry, + options.synapse.readClient.chain.legacyPieceStorageIdLimit ) ) ) @@ -432,7 +433,8 @@ export class StorageContext { options.providerId, requestedMetadata, warmStorageService, - spRegistry + spRegistry, + synapse.readClient.chain.legacyPieceStorageIdLimit ) } @@ -498,24 +500,20 @@ export class StorageContext { * Resolve the best matching DataSet for a Provider using a specific provider ID. * * Selection logic: - * 1. Filters for the provider's active datasets owned by the client - * 2. Sorts by dataSetId ascending (oldest first) - * 3. Evaluates datasets oldest-first through a sliding pool of at most - * RESOLVE_CONCURRENCY reads, reading metadata before checking for pieces - * 4. Prefers the oldest metadata match that has active pieces, otherwise the - * oldest metadata match; returns null when nothing matches - * 5. Stops starting datasets newer than the oldest non-empty match once it is - * known, so the read fan-out shrinks to roughly the match position - * - * The pool caps RPC fan-out for clients with many datasets per provider - * (FilOzone/synapse-sdk#631). + * 1. Filters for the provider's active datasets owned by the client, sorted + * ascending by dataSetId (oldest first) + * 2. Checks compact datasets first; falls back to legacy datasets only + * when none match the metadata + * 3. Within a tier, {@link findBestDataSetMatch} prefers the oldest metadata + * match that has active pieces, otherwise the oldest metadata match */ private static async resolveByProviderId( clientAddress: Address, providerId: bigint, requestedMetadata: Record, warmStorageService: WarmStorageService, - spRegistry: SPRegistryService + spRegistry: SPRegistryService, + legacyPieceStorageIdLimit: bigint ): Promise { const [provider, dataSets] = await Promise.all([ spRegistry.getProvider({ providerId }), @@ -528,31 +526,73 @@ export class StorageContext { throw createError('StorageContext', 'resolveByProviderId', `Provider ID ${providerId} not found in registry`) } - // Filter for this provider's active datasets - const providerDataSets = dataSets.filter( - (dataSet) => dataSet.dataSetId && dataSet.providerId === provider.id && dataSet.pdpEndEpoch === 0n - ) + // Filter for this provider's active datasets, sorted ascending by ID + // (oldest first). Compare as bigint so ordering stays correct for IDs + // beyond Number.MAX_SAFE_INTEGER. + const providerDataSets = dataSets + .filter((dataSet) => dataSet.dataSetId && dataSet.providerId === provider.id && dataSet.pdpEndEpoch === 0n) + .sort((a, b) => { + if (a.dataSetId < b.dataSetId) return -1 + if (a.dataSetId > b.dataSetId) return 1 + return 0 + }) + + // Legacy IDs are always lower than compact IDs, so the ascending sort + // above already groups them into a legacy prefix and a compact suffix. + const splitIndex = providerDataSets.findIndex((dataSet) => dataSet.dataSetId >= legacyPieceStorageIdLimit) + const legacyDataSets = splitIndex === -1 ? providerDataSets : providerDataSets.slice(0, splitIndex) + const compactDataSets = splitIndex === -1 ? [] : providerDataSets.slice(splitIndex) + + const selectedDataSet = + (await StorageContext.findBestDataSetMatch(compactDataSets, requestedMetadata, warmStorageService)) ?? + (await StorageContext.findBestDataSetMatch(legacyDataSets, requestedMetadata, warmStorageService)) + + if (selectedDataSet != null) { + return { + provider, + dataSetId: selectedDataSet.dataSetId, + dataSetMetadata: selectedDataSet.dataSetMetadata, + } + } + + return { + provider, + dataSetId: null, + dataSetMetadata: requestedMetadata, + } + } + /** + * Find the best metadata-matching dataset within a single legacy/compact + * tier, given datasets already sorted ascending by ID (oldest first). + * + * Evaluates datasets oldest-first through a sliding pool of at most + * RESOLVE_CONCURRENCY reads, reading metadata before checking for pieces. + * Prefers the oldest metadata match that has active pieces, otherwise the + * oldest metadata match; returns null when nothing matches. Stops starting + * datasets newer than the oldest non-empty match once it is known, so the + * read fan-out shrinks to roughly the match position. + * + * The pool caps RPC fan-out for clients with many datasets per provider + * (FilOzone/synapse-sdk#631). + */ + private static async findBestDataSetMatch( + dataSets: { dataSetId: bigint }[], + requestedMetadata: Record, + warmStorageService: WarmStorageService + ): Promise<{ dataSetId: bigint; dataSetMetadata: Record } | null> { type EvaluatedDataSet = { dataSetId: bigint dataSetMetadata: Record hasPieces: boolean } - // Sort ascending by ID (oldest first) for deterministic selection. Compare - // as bigint so ordering stays correct for IDs beyond Number.MAX_SAFE_INTEGER. - const sortedDataSets = providerDataSets.sort((a, b) => { - if (a.dataSetId < b.dataSetId) return -1 - if (a.dataSetId > b.dataSetId) return 1 - return 0 - }) - // Result is selected by index, not completion order, because reads finish out // of order: `bestNonEmptyIndex` is the oldest non-empty metadata match and // `firstMatchIndex` the oldest metadata match (the fallback). Metadata is read // first and hasActivePieces only on a metadata match, so non-matching // datasets skip the leaf-count read. - const evaluated: (EvaluatedDataSet | null)[] = new Array(sortedDataSets.length).fill(null) + const evaluated: (EvaluatedDataSet | null)[] = new Array(dataSets.length).fill(null) let firstMatchIndex = Number.POSITIVE_INFINITY let bestNonEmptyIndex = Number.POSITIVE_INFINITY @@ -591,15 +631,15 @@ export class StorageContext { const inFlight = new Set>() let nextIndex = 0 let failure: unknown - while (nextIndex < sortedDataSets.length || inFlight.size > 0) { + while (nextIndex < dataSets.length || inFlight.size > 0) { while ( failure == null && inFlight.size < RESOLVE_CONCURRENCY && - nextIndex < sortedDataSets.length && + nextIndex < dataSets.length && nextIndex <= bestNonEmptyIndex ) { const index = nextIndex++ - const task = evaluate(index, sortedDataSets[index].dataSetId) + const task = evaluate(index, dataSets[index].dataSetId) .catch((error) => { failure ??= error }) @@ -625,21 +665,7 @@ export class StorageContext { } const selectedIndex = bestNonEmptyIndex === Number.POSITIVE_INFINITY ? firstMatchIndex : bestNonEmptyIndex - const selectedDataSet = selectedIndex === Number.POSITIVE_INFINITY ? null : evaluated[selectedIndex] - - if (selectedDataSet != null) { - return { - provider, - dataSetId: selectedDataSet.dataSetId, - dataSetMetadata: selectedDataSet.dataSetMetadata, - } - } - - return { - provider, - dataSetId: null, - dataSetMetadata: requestedMetadata, - } + return selectedIndex === Number.POSITIVE_INFINITY ? null : evaluated[selectedIndex] } /** diff --git a/packages/synapse-sdk/src/test/storage.test.ts b/packages/synapse-sdk/src/test/storage.test.ts index e11390029..2e4781b01 100644 --- a/packages/synapse-sdk/src/test/storage.test.ts +++ b/packages/synapse-sdk/src/test/storage.test.ts @@ -365,6 +365,62 @@ describe('StorageService', () => { assert.equal(service.dataSetId, 2n) }) + it('should prefer a compact data set over a legacy one with existing pieces', async () => { + const expectedDataSetBase = { + cacheMissRailId: 0n, + cdnRailId: 0n, + clientDataSetId: 0n, + commissionBps: 100n, + payee: Mocks.ADDRESSES.serviceProvider1, + payer: Mocks.ADDRESSES.client1, + pdpEndEpoch: 0n, + providerId: 1n, + pendingOneTimePayments: 0n, + lifecycleReserveBalance: 0n, + serviceProvider: Mocks.ADDRESSES.serviceProvider1, + } + // Legacy data set (last ID below calibration's compact cutover) has + // pieces; compact data set (first ID at the cutover) is empty. Compact + // must still win. + const legacyId = calibration.legacyPieceStorageIdLimit - 1n + const compactId = calibration.legacyPieceStorageIdLimit + const expectedDataSets = [ + { ...expectedDataSetBase, dataSetId: legacyId, pdpRailId: 1n }, + { ...expectedDataSetBase, dataSetId: compactId, pdpRailId: 2n }, + ] + server.use( + Mocks.JSONRPC({ + ...Mocks.presets.basic, + pdpVerifier: { + ...Mocks.presets.basic.pdpVerifier, + getDataSetLeafCount: (args) => { + const [dataSetId] = args + return [dataSetId === legacyId ? 1n : 0n] + }, + }, + warmStorageView: { + ...Mocks.presets.basic.warmStorageView, + getClientDataSets: () => [expectedDataSets], + getAllDataSetMetadata: () => [[], []], + getDataSet: (args) => { + const [dataSetId] = args + return [expectedDataSets.find((ds) => ds.dataSetId === dataSetId) ?? ({} as (typeof expectedDataSets)[0])] + }, + }, + }), + Mocks.PING({ + baseUrl: Mocks.PROVIDERS.provider1.products[0].offering.serviceURL, + }) + ) + const synapse = new Synapse({ client, source: null }) + const warmStorageService = new WarmStorageService({ client }) + + const service = await StorageContext.create({ synapse, warmStorageService, providerId: 1n }) + + // Should select the compact data set despite the legacy one having pieces + assert.equal(service.dataSetId, compactId) + }) + it('should bound RPC fan-out when a provider has many data sets (#631)', async () => { // One provider with many active, metadata-matching data sets owned by the // client, the oldest of which already has pieces. The fan-out must stay