Keep searchAfter cursors on their positions when sorting locally

docs/swagger.md

@@ -52,4 +52,5 @@ This API is organized around REST principles, so if you've interacted with RESTf
     * To fetch the next set of items strictly after a specific entry, copy its `searchAfter` cursor value and pass it as a query parameter in your subsequent request:
         * Example: `?searchAfter=<base64_cursor_string>`
     * **Important:** For the cursor-based pagination to work correctly, the `from` parameter must be set to `0` or omitted entirely from the request.
+    * **Disclaimer:** On some endpoints (such as `/transfers` or `/accounts/{address}/transactions`), the items can be reordered after they are retrieved. In that case, consistency is only guaranteed for the `searchAfter` of the last item in the response. Continuing from an item in the middle of the list may return items that were already received or skip some of them.
     * Using `searchAfter` is recommended for deep pagination as it avoids performance issues associated with large offset values.

src/endpoints/transactions/transaction.service.ts

@@ -44,6 +44,7 @@ import { GasBucket } from './entities/gas.bucket';
 import { GasBucketConstants } from './constants/gas.bucket.constants';
 import { TransactionAction } from "./transaction-action/entities/transaction.action";
 import { TransactionActionCategory } from "./transaction-action/entities/transaction.action.category";
+import { SearchAfterUtils } from 'src/utils/search.after.utils';
 
 @Injectable()
 export class TransactionService {
@@ -110,7 +111,11 @@ export class TransactionService {
     return await this.indexerService.getTransactionCount(filter, address);
   }
 
-  public reorderAccountSentTransactionsByNonce(transactions: TransactionDetailed[], accountAddress: string): TransactionDetailed[] {
+  public reorderAccountSentTransactionsByNonce(transactions: TransactionDetailed[], accountAddress: string, order?: SortOrder): TransactionDetailed[] {
+    return SearchAfterUtils.sortKeepingSearchAfterPositions(transactions, items => this.reorderSentTransactionsByNonce(items, accountAddress, order));
+  }
+
+  private reorderSentTransactionsByNonce(transactions: TransactionDetailed[], accountAddress: string, order?: SortOrder): TransactionDetailed[] {
     const sentPositions: number[] = [];
     const sentTransactions: TransactionDetailed[] = [];
 
@@ -124,7 +129,7 @@ export class TransactionService {
     sentTransactions.sort((a, b) => {
       const nonceA = a.nonce ?? 0;
       const nonceB = b.nonce ?? 0;
-      return nonceB - nonceA;
+      return order === SortOrder.asc ? nonceA - nonceB : nonceB - nonceA;
     });
 
     const result = [...transactions];
@@ -226,8 +231,8 @@ export class TransactionService {
     const hasSenderFilter = filter.sender || (filter.senders && filter.senders.length > 0);
     const hasReceiverFilter = filter.receivers && filter.receivers.length > 0;
 
-    if (address && !hasSenderFilter && !hasReceiverFilter && pagination.searchAfter === undefined) {
-      transactions = this.reorderAccountSentTransactionsByNonce(transactions, address);
+    if (address && !hasSenderFilter && !hasReceiverFilter) {
+      transactions = this.reorderAccountSentTransactionsByNonce(transactions, address, filter.order);
     }
 
     if (filter.hashes) {

src/endpoints/transfers/transfer.service.ts

@@ -1,4 +1,4 @@
-import { BadRequestException, forwardRef, Inject, Injectable } from "@nestjs/common";
+import { forwardRef, Inject, Injectable } from "@nestjs/common";
 import { QueryPagination } from "src/common/entities/query.pagination";
 import { TransactionFilter } from "../transactions/entities/transaction.filter";
 import { TransactionType } from "../transactions/entities/transaction.type";
@@ -9,6 +9,8 @@ import { IndexerService } from "src/common/indexer/indexer.service";
 import { TransactionQueryOptions } from "../transactions/entities/transactions.query.options";
 import { TransactionDetailed } from "../transactions/entities/transaction.detailed";
 import { OriginLogger } from "@multiversx/sdk-nestjs-common";
+import { SearchAfterUtils } from "src/utils/search.after.utils";
+import { SortOrder } from "src/common/entities/sort.order";
 
 @Injectable()
 export class TransferService {
@@ -20,7 +22,7 @@ export class TransferService {
     private readonly transactionService: TransactionService,
   ) { }
 
-  private sortElasticTransfers(elasticTransfers: any[]): any[] {
+  private sortElasticTransfers(elasticTransfers: any[], order?: SortOrder): any[] {
     const transactionMap = new Map<string, any>();
     for (const transfer of elasticTransfers) {
       if (transfer.txHash) {
@@ -41,22 +43,23 @@ export class TransferService {
       }
     }
 
-    return elasticTransfers.sortedDescending(
-      (item) => item.timestamp,
-      (item) => item.order
-    );
+    return SearchAfterUtils.sortKeepingSearchAfterPositions(elasticTransfers, items => {
+      const criteria = [(item: any) => item.timestamp, (item: any) => item.order];
+
+      return order === SortOrder.asc ? items.sorted(...criteria) : items.sortedDescending(...criteria);
+    });
   }
 
-  private async sortElasticTransfersByTxsOrder(elasticTransfers: any[], miniBlockHash: string): Promise<any[]> {
+  private async sortElasticTransfersByTxsOrder(elasticTransfers: any[], miniBlockHash: string, order?: SortOrder): Promise<any[]> {
     if (!miniBlockHash) {
-      return this.sortElasticTransfers(elasticTransfers);
+      return this.sortElasticTransfers(elasticTransfers, order);
     }
 
     try {
       const block = await this.indexerService.getBlockByMiniBlockHash(miniBlockHash);
 
       if (!block || !block.miniBlocksDetails) {
-        return this.sortElasticTransfers(elasticTransfers);
+        return this.sortElasticTransfers(elasticTransfers, order);
       }
 
       const miniBlockDetails = block.miniBlocksDetails.find((mb: any) => {
@@ -65,7 +68,7 @@ export class TransferService {
       });
 
       if (!miniBlockDetails || !miniBlockDetails.executionOrderTxsIndices || !miniBlockDetails.txsHashes) {
-        return this.sortElasticTransfers(elasticTransfers);
+        return this.sortElasticTransfers(elasticTransfers, order);
       }
 
       const txHashToOrder: Record<string, number> = {};
@@ -100,30 +103,24 @@ export class TransferService {
         }
       }
 
-      return elasticTransfers.sortedDescending(
+      return SearchAfterUtils.sortKeepingSearchAfterPositions(elasticTransfers, items => items.sortedDescending(
         (item) => -item.order,
         (item) => item.timestamp
-      );
+      ));
 
     } catch (error) {
       this.logger.error(`Error getting block execution order: ${error}`);
-      return this.sortElasticTransfers(elasticTransfers);
+      return this.sortElasticTransfers(elasticTransfers, order);
     }
   }
 
   async getTransfers(filter: TransactionFilter, pagination: QueryPagination, queryOptions: TransactionQueryOptions, fields?: string[]): Promise<Transaction[]> {
     let elasticOperations = await this.indexerService.getTransfers(filter, pagination);
 
     if (queryOptions.withTxsOrder && filter.miniBlockHash) {
-      if (pagination.searchAfter) {
-        throw new BadRequestException('searchAfter pagination is not supported when withTxsOrder and miniBlockHash filters are used');
-      }
-
-      elasticOperations = await this.sortElasticTransfersByTxsOrder(elasticOperations, filter.miniBlockHash);
+      elasticOperations = await this.sortElasticTransfersByTxsOrder(elasticOperations, filter.miniBlockHash, filter.order);
     } else {
-      if (!pagination.searchAfter) {
-        elasticOperations = this.sortElasticTransfers(elasticOperations);
-      }
+      elasticOperations = this.sortElasticTransfers(elasticOperations, filter.order);
     }
 
     let transactions: TransactionDetailed[] = [];
@@ -151,8 +148,8 @@ export class TransferService {
     const hasSenderFilter = filter.sender || (filter.senders && filter.senders.length > 0);
     const hasReceiverFilter = filter.receivers && filter.receivers.length > 0;
 
-    if (filter.address && !hasSenderFilter && !hasReceiverFilter && pagination.searchAfter === undefined) {
-      transactions = this.transactionService.reorderAccountSentTransactionsByNonce(transactions, filter.address);
+    if (filter.address && !hasSenderFilter && !hasReceiverFilter) {
+      transactions = this.transactionService.reorderAccountSentTransactionsByNonce(transactions, filter.address, filter.order);
     }
 
     if (queryOptions.withBlockInfo || (fields && fields.includesSome(['senderBlockHash', 'receiverBlockHash', 'senderBlockNonce', 'receiverBlockNonce']))) {

src/test/unit/services/transactions.spec.ts

@@ -19,6 +19,7 @@ import { TransactionGetService } from "src/endpoints/transactions/transaction.ge
 import { TransactionPriceService } from "src/endpoints/transactions/transaction.price.service";
 import { TransactionService } from "src/endpoints/transactions/transaction.service";
 import { TransactionDetailed } from "src/endpoints/transactions/entities/transaction.detailed";
+import { SortOrder } from "src/common/entities/sort.order";
 import { TransactionStatus } from "src/endpoints/transactions/entities/transaction.status";
 import { UsernameService } from "src/endpoints/usernames/username.service";
 
@@ -688,4 +689,33 @@ describe('TransactionService', () => {
       expect(results[1].nonce).toBe(12);
     });
   });
+  describe('reorderAccountSentTransactionsByNonce', () => {
+    it('keeps the elastic cursors on their positions', () => {
+      const address = 'erd1sender';
+      const transactions = [
+        new TransactionDetailed({ txHash: 'a', sender: address, nonce: 1, searchAfter: 'cursor-1' }),
+        new TransactionDetailed({ txHash: 'b', sender: 'erd1other', nonce: 9, searchAfter: 'cursor-2' }),
+        new TransactionDetailed({ txHash: 'c', sender: address, nonce: 2, searchAfter: 'cursor-3' }),
+      ];
+
+      const result = service.reorderAccountSentTransactionsByNonce(transactions, address);
+
+      expect(result.map(transaction => transaction.txHash)).toEqual(['c', 'b', 'a']);
+      expect(result.map(transaction => transaction.searchAfter)).toEqual(['cursor-1', 'cursor-2', 'cursor-3']);
+    });
+
+    it('orders the sent transactions by ascending nonce when the order is ascending', () => {
+      const address = 'erd1sender';
+      const transactions = [
+        new TransactionDetailed({ txHash: 'a', sender: address, nonce: 2, searchAfter: 'cursor-1' }),
+        new TransactionDetailed({ txHash: 'b', sender: 'erd1other', nonce: 9, searchAfter: 'cursor-2' }),
+        new TransactionDetailed({ txHash: 'c', sender: address, nonce: 1, searchAfter: 'cursor-3' }),
+      ];
+
+      const result = service.reorderAccountSentTransactionsByNonce(transactions, address, SortOrder.asc);
+
+      expect(result.map(transaction => transaction.txHash)).toEqual(['c', 'b', 'a']);
+      expect(result.map(transaction => transaction.searchAfter)).toEqual(['cursor-1', 'cursor-2', 'cursor-3']);
+    });
+  });
 });

src/test/unit/services/transfers.spec.ts

@@ -7,6 +7,9 @@ import { TransactionType } from "src/endpoints/transactions/entities/transaction
 import { TransactionGetService } from "src/endpoints/transactions/transaction.get.service";
 import { TransactionService } from "src/endpoints/transactions/transaction.service";
 import { TransferService } from "src/endpoints/transfers/transfer.service";
+import { QueryPagination } from "src/common/entities/query.pagination";
+import { SortOrder } from "src/common/entities/sort.order";
+import { TransactionQueryOptions } from "src/endpoints/transactions/entities/transactions.query.options";
 
 describe('Transfers Service', () => {
   let service: TransferService;
@@ -20,13 +23,15 @@ describe('Transfers Service', () => {
           useValue: {
             getTransfers: jest.fn(),
             getTransfersCount: jest.fn(),
+            getBlockByMiniBlockHash: jest.fn(),
           },
         },
         {
           provide: TransactionService,
           useValue: {
             applyBlockInfo: jest.fn(),
             processTransactions: jest.fn(),
+            processRelayedInfo: jest.fn(),
           },
         },
         {
@@ -270,4 +275,66 @@ describe('Transfers Service', () => {
       expect(result).toStrictEqual(2);
     });
   });
+  describe('getTransfers - local sorting', () => {
+    // elastic order, the smart contract result's parent is not in this page so it is moved last
+    const options = (init?: Partial<TransactionQueryOptions>) => new TransactionQueryOptions({ withOperations: false, withLogs: false, ...init });
+
+    const elasticOperations = () => [
+      { txHash: 'a', type: 'normal', nonce: 5, timestamp: 100, searchAfter: 'cursor-1' },
+      { txHash: 'x', type: 'unsigned', originalTxHash: 'missing', timestamp: 100, searchAfter: 'cursor-2' },
+      { txHash: 'c', type: 'normal', nonce: 4, timestamp: 100, searchAfter: 'cursor-3' },
+      { txHash: 'b', type: 'normal', nonce: 3, timestamp: 100, searchAfter: 'cursor-4' },
+    ];
+
+    it('keeps the elastic cursors on their positions after sorting', async () => {
+      jest.spyOn(service['indexerService'], 'getTransfers').mockResolvedValue(elasticOperations() as any);
+
+      const result = await service.getTransfers(new TransactionFilter(), new QueryPagination({ size: 4 }), options());
+
+      expect(result.map(transfer => transfer.txHash)).toEqual(['a', 'c', 'b', 'x']);
+      expect(result.map(transfer => transfer.searchAfter)).toEqual(['cursor-1', 'cursor-2', 'cursor-3', 'cursor-4']);
+    });
+
+    it('sorts the pages requested with searchAfter as well', async () => {
+      jest.spyOn(service['indexerService'], 'getTransfers').mockResolvedValue(elasticOperations() as any);
+
+      const result = await service.getTransfers(new TransactionFilter(), new QueryPagination({ size: 4, searchAfter: 'cursor-0' }), options());
+
+      expect(result.map(transfer => transfer.txHash)).toEqual(['a', 'c', 'b', 'x']);
+      expect(result[result.length - 1].searchAfter).toBe('cursor-4');
+    });
+
+    it('follows the requested order when it is ascending', async () => {
+      const ascendingOperations = [
+        { txHash: 'b', type: 'normal', nonce: 3, timestamp: 100, searchAfter: 'cursor-1' },
+        { txHash: 'c', type: 'normal', nonce: 4, timestamp: 100, searchAfter: 'cursor-2' },
+        { txHash: 'x', type: 'unsigned', originalTxHash: 'missing', timestamp: 100, searchAfter: 'cursor-3' },
+        { txHash: 'a', type: 'normal', nonce: 5, timestamp: 100, searchAfter: 'cursor-4' },
+      ];
+      jest.spyOn(service['indexerService'], 'getTransfers').mockResolvedValue(ascendingOperations as any);
+
+      const result = await service.getTransfers(
+        new TransactionFilter({ order: SortOrder.asc }),
+        new QueryPagination({ size: 4, searchAfter: 'cursor-0' }),
+        options(),
+      );
+
+      expect(result.map(transfer => transfer.txHash)).toEqual(['x', 'b', 'c', 'a']);
+      expect(result.map(transfer => transfer.searchAfter)).toEqual(['cursor-1', 'cursor-2', 'cursor-3', 'cursor-4']);
+    });
+
+    it('accepts searchAfter together with withTxsOrder and miniBlockHash', async () => {
+      jest.spyOn(service['indexerService'], 'getTransfers').mockResolvedValue(elasticOperations() as any);
+      jest.spyOn(service['indexerService'], 'getBlockByMiniBlockHash').mockResolvedValue(undefined);
+
+      const result = await service.getTransfers(
+        new TransactionFilter({ miniBlockHash: 'miniblock' }),
+        new QueryPagination({ size: 4, searchAfter: 'cursor-0' }),
+        options({ withTxsOrder: true }),
+      );
+
+      expect(result).toHaveLength(4);
+      expect(result[result.length - 1].searchAfter).toBe('cursor-4');
+    });
+  });
 });

src/test/unit/utils/search.after.utils.spec.ts

@@ -0,0 +1,27 @@
+import { SearchAfterUtils } from 'src/utils/search.after.utils';
+
+describe('SearchAfterUtils', () => {
+  describe('sortKeepingSearchAfterPositions', () => {
+    it('keeps every searchAfter on its original position', () => {
+      const items = [
+        { id: 'a', rank: 1, searchAfter: 'cursor-1' },
+        { id: 'b', rank: 3, searchAfter: 'cursor-2' },
+        { id: 'c', rank: 2, searchAfter: 'cursor-3' },
+      ];
+
+      const sorted = SearchAfterUtils.sortKeepingSearchAfterPositions(items, list => [...list].sort((a, b) => b.rank - a.rank));
+
+      expect(sorted.map(item => item.id)).toEqual(['b', 'c', 'a']);
+      expect(sorted.map(item => item.searchAfter)).toEqual(['cursor-1', 'cursor-2', 'cursor-3']);
+    });
+
+    it('leaves items without searchAfter untouched', () => {
+      const items: { id: string, rank: number, searchAfter?: string }[] = [{ id: 'b', rank: 2 }, { id: 'a', rank: 1 }];
+
+      const sorted = SearchAfterUtils.sortKeepingSearchAfterPositions(items, list => [...list].sort((a, b) => a.rank - b.rank));
+
+      expect(sorted.map(item => item.id)).toEqual(['a', 'b']);
+      expect(sorted.every(item => item.searchAfter === undefined)).toBe(true);
+    });
+  });
+});

src/utils/search.after.utils.ts

@@ -0,0 +1,15 @@
+export class SearchAfterUtils {
+  // a searchAfter cursor marks a position in the elastic results, not the item itself, so when items are
+  // reordered locally every position keeps its cursor and the last item still continues from the last elastic hit
+  static sortKeepingSearchAfterPositions<T extends { searchAfter?: string }>(items: T[], sort: (items: T[]) => T[]): T[] {
+    const searchAfters = items.map(item => item.searchAfter);
+
+    const sortedItems = sort(items);
+
+    for (const [index, item] of sortedItems.entries()) {
+      item.searchAfter = searchAfters[index];
+    }
+
+    return sortedItems;
+  }
+}