diff --git a/docs/swagger.md b/docs/swagger.md index ebcb5f869..7215fe64c 100644 --- a/docs/swagger.md +++ b/docs/swagger.md @@ -52,4 +52,5 @@ This API is organized around REST principles, so if you've interacted with RESTf * To fetch the next set of items strictly after a specific entry, copy its `searchAfter` cursor value and pass it as a query parameter in your subsequent request: * Example: `?searchAfter=` * **Important:** For the cursor-based pagination to work correctly, the `from` parameter must be set to `0` or omitted entirely from the request. + * **Disclaimer:** On some endpoints (such as `/transfers` or `/accounts/{address}/transactions`), the items can be reordered after they are retrieved. In that case, consistency is only guaranteed for the `searchAfter` of the last item in the response. Continuing from an item in the middle of the list may return items that were already received or skip some of them. * Using `searchAfter` is recommended for deep pagination as it avoids performance issues associated with large offset values. diff --git a/src/endpoints/transactions/transaction.service.ts b/src/endpoints/transactions/transaction.service.ts index 4dc86e898..ab8577fc0 100644 --- a/src/endpoints/transactions/transaction.service.ts +++ b/src/endpoints/transactions/transaction.service.ts @@ -44,6 +44,7 @@ import { GasBucket } from './entities/gas.bucket'; import { GasBucketConstants } from './constants/gas.bucket.constants'; import { TransactionAction } from "./transaction-action/entities/transaction.action"; import { TransactionActionCategory } from "./transaction-action/entities/transaction.action.category"; +import { SearchAfterUtils } from 'src/utils/search.after.utils'; @Injectable() export class TransactionService { @@ -110,7 +111,11 @@ export class TransactionService { return await this.indexerService.getTransactionCount(filter, address); } - public reorderAccountSentTransactionsByNonce(transactions: TransactionDetailed[], accountAddress: string): TransactionDetailed[] { + public reorderAccountSentTransactionsByNonce(transactions: TransactionDetailed[], accountAddress: string, order?: SortOrder): TransactionDetailed[] { + return SearchAfterUtils.sortKeepingSearchAfterPositions(transactions, items => this.reorderSentTransactionsByNonce(items, accountAddress, order)); + } + + private reorderSentTransactionsByNonce(transactions: TransactionDetailed[], accountAddress: string, order?: SortOrder): TransactionDetailed[] { const sentPositions: number[] = []; const sentTransactions: TransactionDetailed[] = []; @@ -124,7 +129,7 @@ export class TransactionService { sentTransactions.sort((a, b) => { const nonceA = a.nonce ?? 0; const nonceB = b.nonce ?? 0; - return nonceB - nonceA; + return order === SortOrder.asc ? nonceA - nonceB : nonceB - nonceA; }); const result = [...transactions]; @@ -226,8 +231,8 @@ export class TransactionService { const hasSenderFilter = filter.sender || (filter.senders && filter.senders.length > 0); const hasReceiverFilter = filter.receivers && filter.receivers.length > 0; - if (address && !hasSenderFilter && !hasReceiverFilter && pagination.searchAfter === undefined) { - transactions = this.reorderAccountSentTransactionsByNonce(transactions, address); + if (address && !hasSenderFilter && !hasReceiverFilter) { + transactions = this.reorderAccountSentTransactionsByNonce(transactions, address, filter.order); } if (filter.hashes) { diff --git a/src/endpoints/transfers/transfer.service.ts b/src/endpoints/transfers/transfer.service.ts index 41c5b8c81..0a51dfb9b 100644 --- a/src/endpoints/transfers/transfer.service.ts +++ b/src/endpoints/transfers/transfer.service.ts @@ -1,4 +1,4 @@ -import { BadRequestException, forwardRef, Inject, Injectable } from "@nestjs/common"; +import { forwardRef, Inject, Injectable } from "@nestjs/common"; import { QueryPagination } from "src/common/entities/query.pagination"; import { TransactionFilter } from "../transactions/entities/transaction.filter"; import { TransactionType } from "../transactions/entities/transaction.type"; @@ -9,6 +9,8 @@ import { IndexerService } from "src/common/indexer/indexer.service"; import { TransactionQueryOptions } from "../transactions/entities/transactions.query.options"; import { TransactionDetailed } from "../transactions/entities/transaction.detailed"; import { OriginLogger } from "@multiversx/sdk-nestjs-common"; +import { SearchAfterUtils } from "src/utils/search.after.utils"; +import { SortOrder } from "src/common/entities/sort.order"; @Injectable() export class TransferService { @@ -20,7 +22,7 @@ export class TransferService { private readonly transactionService: TransactionService, ) { } - private sortElasticTransfers(elasticTransfers: any[]): any[] { + private sortElasticTransfers(elasticTransfers: any[], order?: SortOrder): any[] { const transactionMap = new Map(); for (const transfer of elasticTransfers) { if (transfer.txHash) { @@ -41,22 +43,23 @@ export class TransferService { } } - return elasticTransfers.sortedDescending( - (item) => item.timestamp, - (item) => item.order - ); + return SearchAfterUtils.sortKeepingSearchAfterPositions(elasticTransfers, items => { + const criteria = [(item: any) => item.timestamp, (item: any) => item.order]; + + return order === SortOrder.asc ? items.sorted(...criteria) : items.sortedDescending(...criteria); + }); } - private async sortElasticTransfersByTxsOrder(elasticTransfers: any[], miniBlockHash: string): Promise { + private async sortElasticTransfersByTxsOrder(elasticTransfers: any[], miniBlockHash: string, order?: SortOrder): Promise { if (!miniBlockHash) { - return this.sortElasticTransfers(elasticTransfers); + return this.sortElasticTransfers(elasticTransfers, order); } try { const block = await this.indexerService.getBlockByMiniBlockHash(miniBlockHash); if (!block || !block.miniBlocksDetails) { - return this.sortElasticTransfers(elasticTransfers); + return this.sortElasticTransfers(elasticTransfers, order); } const miniBlockDetails = block.miniBlocksDetails.find((mb: any) => { @@ -65,7 +68,7 @@ export class TransferService { }); if (!miniBlockDetails || !miniBlockDetails.executionOrderTxsIndices || !miniBlockDetails.txsHashes) { - return this.sortElasticTransfers(elasticTransfers); + return this.sortElasticTransfers(elasticTransfers, order); } const txHashToOrder: Record = {}; @@ -100,14 +103,14 @@ export class TransferService { } } - return elasticTransfers.sortedDescending( + return SearchAfterUtils.sortKeepingSearchAfterPositions(elasticTransfers, items => items.sortedDescending( (item) => -item.order, (item) => item.timestamp - ); + )); } catch (error) { this.logger.error(`Error getting block execution order: ${error}`); - return this.sortElasticTransfers(elasticTransfers); + return this.sortElasticTransfers(elasticTransfers, order); } } @@ -115,15 +118,9 @@ export class TransferService { let elasticOperations = await this.indexerService.getTransfers(filter, pagination); if (queryOptions.withTxsOrder && filter.miniBlockHash) { - if (pagination.searchAfter) { - throw new BadRequestException('searchAfter pagination is not supported when withTxsOrder and miniBlockHash filters are used'); - } - - elasticOperations = await this.sortElasticTransfersByTxsOrder(elasticOperations, filter.miniBlockHash); + elasticOperations = await this.sortElasticTransfersByTxsOrder(elasticOperations, filter.miniBlockHash, filter.order); } else { - if (!pagination.searchAfter) { - elasticOperations = this.sortElasticTransfers(elasticOperations); - } + elasticOperations = this.sortElasticTransfers(elasticOperations, filter.order); } let transactions: TransactionDetailed[] = []; @@ -151,8 +148,8 @@ export class TransferService { const hasSenderFilter = filter.sender || (filter.senders && filter.senders.length > 0); const hasReceiverFilter = filter.receivers && filter.receivers.length > 0; - if (filter.address && !hasSenderFilter && !hasReceiverFilter && pagination.searchAfter === undefined) { - transactions = this.transactionService.reorderAccountSentTransactionsByNonce(transactions, filter.address); + if (filter.address && !hasSenderFilter && !hasReceiverFilter) { + transactions = this.transactionService.reorderAccountSentTransactionsByNonce(transactions, filter.address, filter.order); } if (queryOptions.withBlockInfo || (fields && fields.includesSome(['senderBlockHash', 'receiverBlockHash', 'senderBlockNonce', 'receiverBlockNonce']))) { diff --git a/src/test/unit/services/transactions.spec.ts b/src/test/unit/services/transactions.spec.ts index a52be339d..94272d544 100644 --- a/src/test/unit/services/transactions.spec.ts +++ b/src/test/unit/services/transactions.spec.ts @@ -19,6 +19,7 @@ import { TransactionGetService } from "src/endpoints/transactions/transaction.ge import { TransactionPriceService } from "src/endpoints/transactions/transaction.price.service"; import { TransactionService } from "src/endpoints/transactions/transaction.service"; import { TransactionDetailed } from "src/endpoints/transactions/entities/transaction.detailed"; +import { SortOrder } from "src/common/entities/sort.order"; import { TransactionStatus } from "src/endpoints/transactions/entities/transaction.status"; import { UsernameService } from "src/endpoints/usernames/username.service"; @@ -688,4 +689,33 @@ describe('TransactionService', () => { expect(results[1].nonce).toBe(12); }); }); + describe('reorderAccountSentTransactionsByNonce', () => { + it('keeps the elastic cursors on their positions', () => { + const address = 'erd1sender'; + const transactions = [ + new TransactionDetailed({ txHash: 'a', sender: address, nonce: 1, searchAfter: 'cursor-1' }), + new TransactionDetailed({ txHash: 'b', sender: 'erd1other', nonce: 9, searchAfter: 'cursor-2' }), + new TransactionDetailed({ txHash: 'c', sender: address, nonce: 2, searchAfter: 'cursor-3' }), + ]; + + const result = service.reorderAccountSentTransactionsByNonce(transactions, address); + + expect(result.map(transaction => transaction.txHash)).toEqual(['c', 'b', 'a']); + expect(result.map(transaction => transaction.searchAfter)).toEqual(['cursor-1', 'cursor-2', 'cursor-3']); + }); + + it('orders the sent transactions by ascending nonce when the order is ascending', () => { + const address = 'erd1sender'; + const transactions = [ + new TransactionDetailed({ txHash: 'a', sender: address, nonce: 2, searchAfter: 'cursor-1' }), + new TransactionDetailed({ txHash: 'b', sender: 'erd1other', nonce: 9, searchAfter: 'cursor-2' }), + new TransactionDetailed({ txHash: 'c', sender: address, nonce: 1, searchAfter: 'cursor-3' }), + ]; + + const result = service.reorderAccountSentTransactionsByNonce(transactions, address, SortOrder.asc); + + expect(result.map(transaction => transaction.txHash)).toEqual(['c', 'b', 'a']); + expect(result.map(transaction => transaction.searchAfter)).toEqual(['cursor-1', 'cursor-2', 'cursor-3']); + }); + }); }); diff --git a/src/test/unit/services/transfers.spec.ts b/src/test/unit/services/transfers.spec.ts index e8dda59db..35cea40c7 100644 --- a/src/test/unit/services/transfers.spec.ts +++ b/src/test/unit/services/transfers.spec.ts @@ -7,6 +7,9 @@ import { TransactionType } from "src/endpoints/transactions/entities/transaction import { TransactionGetService } from "src/endpoints/transactions/transaction.get.service"; import { TransactionService } from "src/endpoints/transactions/transaction.service"; import { TransferService } from "src/endpoints/transfers/transfer.service"; +import { QueryPagination } from "src/common/entities/query.pagination"; +import { SortOrder } from "src/common/entities/sort.order"; +import { TransactionQueryOptions } from "src/endpoints/transactions/entities/transactions.query.options"; describe('Transfers Service', () => { let service: TransferService; @@ -20,6 +23,7 @@ describe('Transfers Service', () => { useValue: { getTransfers: jest.fn(), getTransfersCount: jest.fn(), + getBlockByMiniBlockHash: jest.fn(), }, }, { @@ -27,6 +31,7 @@ describe('Transfers Service', () => { useValue: { applyBlockInfo: jest.fn(), processTransactions: jest.fn(), + processRelayedInfo: jest.fn(), }, }, { @@ -270,4 +275,66 @@ describe('Transfers Service', () => { expect(result).toStrictEqual(2); }); }); + describe('getTransfers - local sorting', () => { + // elastic order, the smart contract result's parent is not in this page so it is moved last + const options = (init?: Partial) => new TransactionQueryOptions({ withOperations: false, withLogs: false, ...init }); + + const elasticOperations = () => [ + { txHash: 'a', type: 'normal', nonce: 5, timestamp: 100, searchAfter: 'cursor-1' }, + { txHash: 'x', type: 'unsigned', originalTxHash: 'missing', timestamp: 100, searchAfter: 'cursor-2' }, + { txHash: 'c', type: 'normal', nonce: 4, timestamp: 100, searchAfter: 'cursor-3' }, + { txHash: 'b', type: 'normal', nonce: 3, timestamp: 100, searchAfter: 'cursor-4' }, + ]; + + it('keeps the elastic cursors on their positions after sorting', async () => { + jest.spyOn(service['indexerService'], 'getTransfers').mockResolvedValue(elasticOperations() as any); + + const result = await service.getTransfers(new TransactionFilter(), new QueryPagination({ size: 4 }), options()); + + expect(result.map(transfer => transfer.txHash)).toEqual(['a', 'c', 'b', 'x']); + expect(result.map(transfer => transfer.searchAfter)).toEqual(['cursor-1', 'cursor-2', 'cursor-3', 'cursor-4']); + }); + + it('sorts the pages requested with searchAfter as well', async () => { + jest.spyOn(service['indexerService'], 'getTransfers').mockResolvedValue(elasticOperations() as any); + + const result = await service.getTransfers(new TransactionFilter(), new QueryPagination({ size: 4, searchAfter: 'cursor-0' }), options()); + + expect(result.map(transfer => transfer.txHash)).toEqual(['a', 'c', 'b', 'x']); + expect(result[result.length - 1].searchAfter).toBe('cursor-4'); + }); + + it('follows the requested order when it is ascending', async () => { + const ascendingOperations = [ + { txHash: 'b', type: 'normal', nonce: 3, timestamp: 100, searchAfter: 'cursor-1' }, + { txHash: 'c', type: 'normal', nonce: 4, timestamp: 100, searchAfter: 'cursor-2' }, + { txHash: 'x', type: 'unsigned', originalTxHash: 'missing', timestamp: 100, searchAfter: 'cursor-3' }, + { txHash: 'a', type: 'normal', nonce: 5, timestamp: 100, searchAfter: 'cursor-4' }, + ]; + jest.spyOn(service['indexerService'], 'getTransfers').mockResolvedValue(ascendingOperations as any); + + const result = await service.getTransfers( + new TransactionFilter({ order: SortOrder.asc }), + new QueryPagination({ size: 4, searchAfter: 'cursor-0' }), + options(), + ); + + expect(result.map(transfer => transfer.txHash)).toEqual(['x', 'b', 'c', 'a']); + expect(result.map(transfer => transfer.searchAfter)).toEqual(['cursor-1', 'cursor-2', 'cursor-3', 'cursor-4']); + }); + + it('accepts searchAfter together with withTxsOrder and miniBlockHash', async () => { + jest.spyOn(service['indexerService'], 'getTransfers').mockResolvedValue(elasticOperations() as any); + jest.spyOn(service['indexerService'], 'getBlockByMiniBlockHash').mockResolvedValue(undefined); + + const result = await service.getTransfers( + new TransactionFilter({ miniBlockHash: 'miniblock' }), + new QueryPagination({ size: 4, searchAfter: 'cursor-0' }), + options({ withTxsOrder: true }), + ); + + expect(result).toHaveLength(4); + expect(result[result.length - 1].searchAfter).toBe('cursor-4'); + }); + }); }); diff --git a/src/test/unit/utils/search.after.utils.spec.ts b/src/test/unit/utils/search.after.utils.spec.ts new file mode 100644 index 000000000..7a16e7054 --- /dev/null +++ b/src/test/unit/utils/search.after.utils.spec.ts @@ -0,0 +1,27 @@ +import { SearchAfterUtils } from 'src/utils/search.after.utils'; + +describe('SearchAfterUtils', () => { + describe('sortKeepingSearchAfterPositions', () => { + it('keeps every searchAfter on its original position', () => { + const items = [ + { id: 'a', rank: 1, searchAfter: 'cursor-1' }, + { id: 'b', rank: 3, searchAfter: 'cursor-2' }, + { id: 'c', rank: 2, searchAfter: 'cursor-3' }, + ]; + + const sorted = SearchAfterUtils.sortKeepingSearchAfterPositions(items, list => [...list].sort((a, b) => b.rank - a.rank)); + + expect(sorted.map(item => item.id)).toEqual(['b', 'c', 'a']); + expect(sorted.map(item => item.searchAfter)).toEqual(['cursor-1', 'cursor-2', 'cursor-3']); + }); + + it('leaves items without searchAfter untouched', () => { + const items: { id: string, rank: number, searchAfter?: string }[] = [{ id: 'b', rank: 2 }, { id: 'a', rank: 1 }]; + + const sorted = SearchAfterUtils.sortKeepingSearchAfterPositions(items, list => [...list].sort((a, b) => a.rank - b.rank)); + + expect(sorted.map(item => item.id)).toEqual(['a', 'b']); + expect(sorted.every(item => item.searchAfter === undefined)).toBe(true); + }); + }); +}); diff --git a/src/utils/search.after.utils.ts b/src/utils/search.after.utils.ts new file mode 100644 index 000000000..6096079b3 --- /dev/null +++ b/src/utils/search.after.utils.ts @@ -0,0 +1,15 @@ +export class SearchAfterUtils { + // a searchAfter cursor marks a position in the elastic results, not the item itself, so when items are + // reordered locally every position keeps its cursor and the last item still continues from the last elastic hit + static sortKeepingSearchAfterPositions(items: T[], sort: (items: T[]) => T[]): T[] { + const searchAfters = items.map(item => item.searchAfter); + + const sortedItems = sort(items); + + for (const [index, item] of sortedItems.entries()) { + item.searchAfter = searchAfters[index]; + } + + return sortedItems; + } +}