From 152027e2bd97d50ccba6b1838fd5ab797154c9ea Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Tue, 29 Sep 2026 11:38:43 +0300 Subject: [PATCH 01/20] take pool count from gateway and handle pool too large --- src/common/gateway/gateway.service.ts | 22 ++++++- .../cache.warmer/cache.warmer.service.ts | 13 +++- .../transaction.pool.too.large.exception.ts | 15 +++++ src/endpoints/pool/pool.controller.ts | 5 +- src/endpoints/pool/pool.service.ts | 39 ++++++++++- src/test/unit/services/pool.spec.ts | 66 ++++++++++++++++++- src/utils/cache.info.ts | 13 ++++ 7 files changed, 166 insertions(+), 7 deletions(-) create mode 100644 src/endpoints/pool/entities/transaction.pool.too.large.exception.ts diff --git a/src/common/gateway/gateway.service.ts b/src/common/gateway/gateway.service.ts index 735def27e5..76895665b0 100644 --- a/src/common/gateway/gateway.service.ts +++ b/src/common/gateway/gateway.service.ts @@ -182,8 +182,26 @@ export class GatewayService { return new NftData(result.tokenData); } - async getTransactionPool(): Promise { - return await this.get(`transaction/pool?fields=*`, GatewayComponentRequest.transactionPool); + // undefined when the pool is larger than the response size allowed for it in getRaw + async getTransactionPool(): Promise { + // eslint-disable-next-line require-await + return await this.get(`transaction/pool?fields=*`, GatewayComponentRequest.transactionPool, async (error) => { + return error?.message?.startsWith('maxContentLength size of') === true; + }); + } + + // null on gateways that do not have this endpoint yet + async getTransactionPoolCount(): Promise { + // eslint-disable-next-line require-await + const result = await this.get('transaction/pool/count', GatewayComponentRequest.transactionPool, async (error) => { + return error?.response?.status === 404; + }); + + if (!result?.txPoolCounts) { + return null; + } + + return Object.values(result.txPoolCounts).reduce((total, count) => total + count, 0); } async getTransaction(txHash: string): Promise { diff --git a/src/crons/cache.warmer/cache.warmer.service.ts b/src/crons/cache.warmer/cache.warmer.service.ts index e50c8b3914..51e696342b 100644 --- a/src/crons/cache.warmer/cache.warmer.service.ts +++ b/src/crons/cache.warmer/cache.warmer.service.ts @@ -30,6 +30,7 @@ import { TokenDetailed } from "src/endpoints/tokens/entities/token.detailed"; import { DataApiService } from "src/common/data-api/data-api.service"; import { BlockService } from "src/endpoints/blocks/block.service"; import { PoolService } from "src/endpoints/pool/pool.service"; +import { TransactionPoolTooLargeException } from "src/endpoints/pool/entities/transaction.pool.too.large.exception"; import * as JsonDiff from "json-diff"; import { QueryPagination } from "src/common/entities/query.pagination"; import { StakeService } from "src/endpoints/stake/stake.service"; @@ -151,7 +152,17 @@ export class CacheWarmerService { @Lock({ name: 'Transaction pool invalidation', verbose: true }) async handleTxPoolInvalidations() { - const pool = await this.poolService.getTxPoolRaw(); + let pool; + try { + pool = await this.poolService.getTxPoolRaw(); + } catch (error) { + // nothing to warm while the pool is too large, which getTxPoolRaw already remembers + if (error instanceof TransactionPoolTooLargeException) { + return; + } + + throw error; + } await this.invalidateKey(CacheInfo.TransactionPool.key, pool, this.apiConfigService.getTransactionPoolCacheWarmerTtlInSeconds()); } diff --git a/src/endpoints/pool/entities/transaction.pool.too.large.exception.ts b/src/endpoints/pool/entities/transaction.pool.too.large.exception.ts new file mode 100644 index 0000000000..b679891c40 --- /dev/null +++ b/src/endpoints/pool/entities/transaction.pool.too.large.exception.ts @@ -0,0 +1,15 @@ +import { HttpException, HttpStatus } from "@nestjs/common"; + +// the pool is read from the gateway in a single response, which is limited in size. above that limit there +// is no pool to show, and clients can tell this case apart from other failures by the code in the body +export class TransactionPoolTooLargeException extends HttpException { + static readonly code = 'transaction_pool_too_large'; + + constructor() { + super({ + statusCode: HttpStatus.SERVICE_UNAVAILABLE, + code: TransactionPoolTooLargeException.code, + message: 'The transaction pool is too large to be displayed', + }, HttpStatus.SERVICE_UNAVAILABLE); + } +} diff --git a/src/endpoints/pool/pool.controller.ts b/src/endpoints/pool/pool.controller.ts index f6948e4f99..81c6b1d926 100644 --- a/src/endpoints/pool/pool.controller.ts +++ b/src/endpoints/pool/pool.controller.ts @@ -1,6 +1,6 @@ import { ParseAddressAndMetachainPipe, ParseAddressPipe, ParseEnumPipe, ParseIntPipe, ParseTransactionHashPipe, ParseArrayPipe } from "@multiversx/sdk-nestjs-common"; import { Controller, DefaultValuePipe, Get, NotFoundException, Param, Query } from "@nestjs/common"; -import { ApiExcludeEndpoint, ApiNotFoundResponse, ApiOkResponse, ApiOperation, ApiQuery, ApiTags } from "@nestjs/swagger"; +import { ApiExcludeEndpoint, ApiNotFoundResponse, ApiOkResponse, ApiOperation, ApiQuery, ApiServiceUnavailableResponse, ApiTags } from "@nestjs/swagger"; import { PoolService } from "./pool.service"; import { QueryPagination } from "src/common/entities/query.pagination"; import { TransactionInPool } from "./entities/transaction.in.pool.dto"; @@ -18,6 +18,7 @@ export class PoolController { @Get("/pool") @ApiOperation({ summary: 'Transactions pool', description: 'Returns the transactions that are currently in the memory pool.' }) @ApiOkResponse({ type: TransactionInPool, isArray: true }) + @ApiServiceUnavailableResponse({ description: 'The transaction pool is too large to be displayed' }) @ApiQuery({ name: 'from', description: 'Number of items to skip for the result set', required: false }) @ApiQuery({ name: 'size', description: 'Number of items to retrieve', required: false }) @ApiQuery({ name: 'sender', description: 'Search in transaction pool by a specific sender', required: false }) @@ -50,6 +51,7 @@ export class PoolController { @Get("/pool/count") @ApiOperation({ summary: 'Transactions pool count', description: 'Returns the number of transactions that are currently in the memory pool.' }) @ApiOkResponse({ type: Number }) + @ApiServiceUnavailableResponse({ description: 'The transaction pool is too large to be displayed' }) @ApiQuery({ name: 'sender', description: 'Returns the number of transactions with a specific sender', required: false }) @ApiQuery({ name: 'receiver', description: 'Search in transaction pool by a specific receiver', required: false }) @ApiQuery({ name: 'senderShard', description: 'The shard of the sender', required: false }) @@ -85,6 +87,7 @@ export class PoolController { @ApiOperation({ summary: 'Transaction from pool', description: 'Returns a transaction from the memory pool.' }) @ApiOkResponse({ type: TransactionInPool }) @ApiNotFoundResponse({ description: 'Transaction not found' }) + @ApiServiceUnavailableResponse({ description: 'The transaction pool is too large to be displayed' }) async getTransactionFromPool( @Param('txhash', ParseTransactionHashPipe) txHash: string, ): Promise { diff --git a/src/endpoints/pool/pool.service.ts b/src/endpoints/pool/pool.service.ts index 501261fdf9..5dd8acf177 100644 --- a/src/endpoints/pool/pool.service.ts +++ b/src/endpoints/pool/pool.service.ts @@ -12,6 +12,7 @@ import { TxInPoolFields } from "src/common/gateway/entities/tx.in.pool.fields"; import { TransactionActionService } from "../transactions/transaction-action/transaction.action.service"; import { Transaction } from "../transactions/entities/transaction"; import { ApiUtils } from "@multiversx/sdk-nestjs-http"; +import { TransactionPoolTooLargeException } from "./entities/transaction.pool.too.large.exception"; @Injectable() export class PoolService { @@ -28,7 +29,31 @@ export class PoolService { } async getPoolCount(filter: PoolFilter): Promise { - const pool = await this.getPoolWithFilters(filter); + const { type, ...otherFilters } = filter; + if (Object.values(otherFilters).some(value => value !== undefined)) { + const pool = await this.getPoolWithFilters(filter); + return pool.length; + } + + // the total and the count for each type are cached on their own + return await this.cacheService.getOrSet( + CacheInfo.TransactionPoolCount(type).key, + async () => await this.getPoolCountRaw(type), + CacheInfo.TransactionPoolCount(type).ttl, + ); + } + + // the gateway counts the pool itself, so the total needs neither the pool nor its size limit. counts by + // type, and the total on gateways that do not have that endpoint yet, are taken from the pool + private async getPoolCountRaw(type?: TransactionType): Promise { + if (type === undefined) { + const count = await this.gatewayService.getTransactionPoolCount(); + if (count !== null) { + return count; + } + } + + const pool = await this.getPoolWithFilters(new PoolFilter({ type })); return pool.length; } @@ -57,8 +82,20 @@ export class PoolService { return this.applyFilters(pool, filter); } + // a pool found too large is remembered for a while, so that it is not downloaded again, up to the size + // limit, on every request until then async getTxPoolRaw(): Promise { + const isTooLarge = await this.cacheService.get(CacheInfo.TransactionPoolTooLarge.key); + if (isTooLarge) { + throw new TransactionPoolTooLargeException(); + } + const pool = await this.gatewayService.getTransactionPool(); + if (!pool) { + await this.cacheService.set(CacheInfo.TransactionPoolTooLarge.key, true, CacheInfo.TransactionPoolTooLarge.ttl); + throw new TransactionPoolTooLargeException(); + } + return this.parseTransactions(pool); } diff --git a/src/test/unit/services/pool.spec.ts b/src/test/unit/services/pool.spec.ts index a276ca2e3b..d3741e53e2 100644 --- a/src/test/unit/services/pool.spec.ts +++ b/src/test/unit/services/pool.spec.ts @@ -8,6 +8,8 @@ import { PoolService } from "src/endpoints/pool/pool.service"; import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; import { ProtocolService } from "../../../common/protocol/protocol.service"; import { TransactionActionService } from "../../../endpoints/transactions/transaction-action/transaction.action.service"; +import { TransactionPoolTooLargeException } from "src/endpoints/pool/entities/transaction.pool.too.large.exception"; +import { CacheInfo } from "src/utils/cache.info"; describe('PoolService', () => { let service: PoolService; @@ -22,12 +24,15 @@ describe('PoolService', () => { provide: GatewayService, useValue: { getTransactionPool: jest.fn(), + getTransactionPoolCount: jest.fn(), }, }, { provide: CacheService, useValue: { getOrSet: jest.fn(), + get: jest.fn(), + set: jest.fn(), }, }, { @@ -60,7 +65,10 @@ describe('PoolService', () => { gatewayService.getTransactionPool = jest.fn().mockResolvedValue(data); const txPoolRaw = await service.getTxPoolRaw(); - cacheService.getOrSet = jest.fn().mockResolvedValue(txPoolRaw); + // the pool is served from the cache, every other value is computed + cacheService.getOrSet = jest.fn().mockImplementation(async (key: string, createValueFunc: () => Promise) => { + return key === CacheInfo.TransactionPool.key ? txPoolRaw : await createValueFunc(); + }); }); @@ -94,11 +102,42 @@ describe('PoolService', () => { }); describe('getPoolCount', () => { - it('should work and return the pool count', async () => { + it('should return the total counted by the gateway', async () => { + gatewayService.getTransactionPoolCount = jest.fn().mockResolvedValue(42); + + const poolCount = await service.getPoolCount(new PoolFilter()); + expect(poolCount).toStrictEqual(42); + expect(cacheService.getOrSet).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, expect.any(Function), CacheInfo.TransactionPoolCount().ttl); + expect(cacheService.getOrSet).not.toHaveBeenCalledWith(CacheInfo.TransactionPool.key, expect.anything(), expect.anything()); + }); + + it('should count the pool when the gateway cannot count it', async () => { + gatewayService.getTransactionPoolCount = jest.fn().mockResolvedValue(null); + const poolCount = await service.getPoolCount(new PoolFilter()); expect(poolCount).toStrictEqual(7); }); + it('should cache the count for a type apart from the total', async () => { + const poolCount = await service.getPoolCount(new PoolFilter({ type: TransactionType.Reward })); + expect(poolCount).toStrictEqual(5); + expect(cacheService.getOrSet).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(TransactionType.Reward).key, expect.any(Function), CacheInfo.TransactionPoolCount(TransactionType.Reward).ttl); + expect(CacheInfo.TransactionPoolCount(TransactionType.Reward).key).not.toStrictEqual(CacheInfo.TransactionPoolCount().key); + expect(gatewayService.getTransactionPoolCount).not.toHaveBeenCalled(); + }); + + it('should count the pool, without caching the count, for other filters', async () => { + const filter = new PoolFilter({ type: TransactionType.Transaction, senderShard: 0 }); + + const poolCount = await service.getPoolCount(filter); + expect(cacheService.getOrSet).toHaveBeenCalledTimes(1); + expect(cacheService.getOrSet).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, expect.any(Function), CacheInfo.TransactionPool.ttl); + + const pool = await service.getPool(new QueryPagination({ from: 0, size: 100 }), filter); + expect(pool.length).toBeGreaterThan(0); + expect(poolCount).toStrictEqual(pool.length); + }); + it('should work and return the pool count with filters', async () => { let poolCount = await service.getPoolCount(new PoolFilter({ type: TransactionType.Transaction })); expect(poolCount).toStrictEqual(1); @@ -111,6 +150,29 @@ describe('PoolService', () => { }); }); + describe('getTxPoolRaw', () => { + it('should remember a pool too large for the gateway response limit', async () => { + gatewayService.getTransactionPool = jest.fn().mockResolvedValue(undefined); + + await expect(service.getTxPoolRaw()).rejects.toBeInstanceOf(TransactionPoolTooLargeException); + expect(cacheService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolTooLarge.key, true, CacheInfo.TransactionPoolTooLarge.ttl); + }); + + it('should not download the pool again while it is remembered as too large', async () => { + cacheService.get = jest.fn().mockResolvedValue(true); + gatewayService.getTransactionPool = jest.fn(); + + await expect(service.getTxPoolRaw()).rejects.toBeInstanceOf(TransactionPoolTooLargeException); + expect(gatewayService.getTransactionPool).not.toHaveBeenCalled(); + }); + + it('should answer with a code the clients can recognize', () => { + const exception = new TransactionPoolTooLargeException(); + expect(exception.getStatus()).toStrictEqual(503); + expect(exception.getResponse()).toEqual(expect.objectContaining({ code: 'transaction_pool_too_large' })); + }); + }); + describe('getTransactionFromPool', () => { it('should work and return the transaction', async () => { const tx = await service.getTransactionFromPool("e07af9835b6da5740d0f791cfe65491a562852c57d44af63fdc14be5d73f01da"); diff --git a/src/utils/cache.info.ts b/src/utils/cache.info.ts index e88079edfd..37bd993718 100644 --- a/src/utils/cache.info.ts +++ b/src/utils/cache.info.ts @@ -2,6 +2,7 @@ import { Constants } from "@multiversx/sdk-nestjs-common"; import { QueryPagination } from "src/common/entities/query.pagination"; import { BlockFilter } from "src/endpoints/blocks/entities/block.filter"; import { TpsInterval } from "src/endpoints/tps/entities/tps.interval"; +import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; export class CacheInfo { key: string = ""; @@ -57,6 +58,18 @@ export class CacheInfo { ttl: Constants.oneSecond(), }; + static TransactionPoolCount(type?: TransactionType): CacheInfo { + return { + key: type ? `txpool:count:${type}` : 'txpool:count', + ttl: Constants.oneSecond(), + }; + } + + static TransactionPoolTooLarge: CacheInfo = { + key: 'txpool:tooLarge', + ttl: Constants.oneMinute(), + }; + static ApplicationMostUsed: CacheInfo = { key: 'applicationMostUsed', ttl: Constants.oneHour(), From 15f5290de9999c0f2f26b9b18eb84d41351b0381 Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Tue, 29 Sep 2026 11:47:23 +0300 Subject: [PATCH 02/20] warm pool counts in cache warmer and drop gateway count fallback --- src/common/gateway/gateway.service.ts | 12 +--- .../cache.warmer/cache.warmer.service.ts | 17 +++++- src/endpoints/pool/pool.service.ts | 9 +-- .../unit/services/cache.warmer.pool.spec.ts | 61 +++++++++++++++++++ src/test/unit/services/pool.spec.ts | 7 --- 5 files changed, 81 insertions(+), 25 deletions(-) create mode 100644 src/test/unit/services/cache.warmer.pool.spec.ts diff --git a/src/common/gateway/gateway.service.ts b/src/common/gateway/gateway.service.ts index 76895665b0..85de1034e1 100644 --- a/src/common/gateway/gateway.service.ts +++ b/src/common/gateway/gateway.service.ts @@ -190,16 +190,8 @@ export class GatewayService { }); } - // null on gateways that do not have this endpoint yet - async getTransactionPoolCount(): Promise { - // eslint-disable-next-line require-await - const result = await this.get('transaction/pool/count', GatewayComponentRequest.transactionPool, async (error) => { - return error?.response?.status === 404; - }); - - if (!result?.txPoolCounts) { - return null; - } + async getTransactionPoolCount(): Promise { + const result = await this.get('transaction/pool/count', GatewayComponentRequest.transactionPool); return Object.values(result.txPoolCounts).reduce((total, count) => total + count, 0); } diff --git a/src/crons/cache.warmer/cache.warmer.service.ts b/src/crons/cache.warmer/cache.warmer.service.ts index 51e696342b..4f5bba20b1 100644 --- a/src/crons/cache.warmer/cache.warmer.service.ts +++ b/src/crons/cache.warmer/cache.warmer.service.ts @@ -31,6 +31,7 @@ import { DataApiService } from "src/common/data-api/data-api.service"; import { BlockService } from "src/endpoints/blocks/block.service"; import { PoolService } from "src/endpoints/pool/pool.service"; import { TransactionPoolTooLargeException } from "src/endpoints/pool/entities/transaction.pool.too.large.exception"; +import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; import * as JsonDiff from "json-diff"; import { QueryPagination } from "src/common/entities/query.pagination"; import { StakeService } from "src/endpoints/stake/stake.service"; @@ -152,11 +153,17 @@ export class CacheWarmerService { @Lock({ name: 'Transaction pool invalidation', verbose: true }) async handleTxPoolInvalidations() { + const ttl = this.apiConfigService.getTransactionPoolCacheWarmerTtlInSeconds(); + + // the gateway counts the pool itself, so the total is warmed even while the pool is too large to be read + const count = await this.poolService.getPoolCountRaw(); + await this.invalidateKey(CacheInfo.TransactionPoolCount().key, count, ttl); + let pool; try { pool = await this.poolService.getTxPoolRaw(); } catch (error) { - // nothing to warm while the pool is too large, which getTxPoolRaw already remembers + // nothing else to warm while the pool is too large, which getTxPoolRaw already remembers if (error instanceof TransactionPoolTooLargeException) { return; } @@ -164,7 +171,13 @@ export class CacheWarmerService { throw error; } - await this.invalidateKey(CacheInfo.TransactionPool.key, pool, this.apiConfigService.getTransactionPoolCacheWarmerTtlInSeconds()); + await this.invalidateKey(CacheInfo.TransactionPool.key, pool, ttl); + + // counted from the pool warmed above + for (const type of Object.values(TransactionType)) { + const typeCount = await this.poolService.getPoolCountRaw(type); + await this.invalidateKey(CacheInfo.TransactionPoolCount(type).key, typeCount, ttl); + } } @Cron('*/2 * * * *') diff --git a/src/endpoints/pool/pool.service.ts b/src/endpoints/pool/pool.service.ts index 5dd8acf177..f478486d2b 100644 --- a/src/endpoints/pool/pool.service.ts +++ b/src/endpoints/pool/pool.service.ts @@ -44,13 +44,10 @@ export class PoolService { } // the gateway counts the pool itself, so the total needs neither the pool nor its size limit. counts by - // type, and the total on gateways that do not have that endpoint yet, are taken from the pool - private async getPoolCountRaw(type?: TransactionType): Promise { + // type are taken from the pool + async getPoolCountRaw(type?: TransactionType): Promise { if (type === undefined) { - const count = await this.gatewayService.getTransactionPoolCount(); - if (count !== null) { - return count; - } + return await this.gatewayService.getTransactionPoolCount(); } const pool = await this.getPoolWithFilters(new PoolFilter({ type })); diff --git a/src/test/unit/services/cache.warmer.pool.spec.ts b/src/test/unit/services/cache.warmer.pool.spec.ts new file mode 100644 index 0000000000..5cf541d248 --- /dev/null +++ b/src/test/unit/services/cache.warmer.pool.spec.ts @@ -0,0 +1,61 @@ +import { Locker, LockResult } from "@multiversx/sdk-nestjs-common"; +import { CacheWarmerService } from "src/crons/cache.warmer/cache.warmer.service"; +import { TransactionPoolTooLargeException } from "src/endpoints/pool/entities/transaction.pool.too.large.exception"; +import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; +import { CacheInfo } from "src/utils/cache.info"; + +describe('CacheWarmerService transaction pool', () => { + const ttl = 10; + const pool = [{ txHash: 'a', type: TransactionType.Transaction }]; + const counts: Record = { + total: 12, + [TransactionType.Transaction]: 3, + [TransactionType.SmartContractResult]: 4, + [TransactionType.Reward]: 5, + }; + + let warmer: CacheWarmerService; + let poolService: any; + let cachingService: any; + + beforeEach(() => { + // run the handler without the lock around it, which records metrics and swallows the errors + jest.spyOn(Locker, 'lock').mockImplementation(async (_key: string, func: () => Promise) => { + await func(); + return LockResult.SUCCESS; + }); + + poolService = { + getPoolCountRaw: jest.fn().mockImplementation(async (type?: TransactionType) => await Promise.resolve(counts[type ?? 'total'])), + getTxPoolRaw: jest.fn().mockResolvedValue(pool), + }; + cachingService = { set: jest.fn() }; + + // only the dependencies of the pool warming, without the constructor, which also schedules every cron + warmer = Object.assign(Object.create(CacheWarmerService.prototype), { + poolService, + cachingService, + apiConfigService: { getTransactionPoolCacheWarmerTtlInSeconds: () => ttl }, + clientProxy: { emit: jest.fn() }, + }); + }); + + it('should warm the pool, its total count and the count for every transaction type', async () => { + await warmer.handleTxPoolInvalidations(); + + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, counts.total, ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, pool, ttl); + for (const type of Object.values(TransactionType)) { + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(type).key, counts[type], ttl); + } + }); + + it('should warm only the total count while the pool is too large', async () => { + poolService.getTxPoolRaw.mockRejectedValue(new TransactionPoolTooLargeException()); + + await warmer.handleTxPoolInvalidations(); + + expect(cachingService.set).toHaveBeenCalledTimes(1); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, counts.total, ttl); + }); +}); diff --git a/src/test/unit/services/pool.spec.ts b/src/test/unit/services/pool.spec.ts index d3741e53e2..5434a37e4a 100644 --- a/src/test/unit/services/pool.spec.ts +++ b/src/test/unit/services/pool.spec.ts @@ -111,13 +111,6 @@ describe('PoolService', () => { expect(cacheService.getOrSet).not.toHaveBeenCalledWith(CacheInfo.TransactionPool.key, expect.anything(), expect.anything()); }); - it('should count the pool when the gateway cannot count it', async () => { - gatewayService.getTransactionPoolCount = jest.fn().mockResolvedValue(null); - - const poolCount = await service.getPoolCount(new PoolFilter()); - expect(poolCount).toStrictEqual(7); - }); - it('should cache the count for a type apart from the total', async () => { const poolCount = await service.getPoolCount(new PoolFilter({ type: TransactionType.Reward })); expect(poolCount).toStrictEqual(5); From 099d6903caf2218fbad12f850e9228d8a72959ab Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Tue, 29 Sep 2026 11:50:13 +0300 Subject: [PATCH 03/20] cache pool too large as null in txpool key instead of separate key --- .../cache.warmer/cache.warmer.service.ts | 18 ++++-------- src/endpoints/pool/pool.service.ts | 18 ++++++------ .../unit/services/cache.warmer.pool.spec.ts | 9 +++--- src/test/unit/services/pool.spec.ts | 28 +++++++++++++------ src/utils/cache.info.ts | 5 ---- 5 files changed, 37 insertions(+), 41 deletions(-) diff --git a/src/crons/cache.warmer/cache.warmer.service.ts b/src/crons/cache.warmer/cache.warmer.service.ts index 4f5bba20b1..bbc7a5b7d5 100644 --- a/src/crons/cache.warmer/cache.warmer.service.ts +++ b/src/crons/cache.warmer/cache.warmer.service.ts @@ -30,7 +30,6 @@ import { TokenDetailed } from "src/endpoints/tokens/entities/token.detailed"; import { DataApiService } from "src/common/data-api/data-api.service"; import { BlockService } from "src/endpoints/blocks/block.service"; import { PoolService } from "src/endpoints/pool/pool.service"; -import { TransactionPoolTooLargeException } from "src/endpoints/pool/entities/transaction.pool.too.large.exception"; import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; import * as JsonDiff from "json-diff"; import { QueryPagination } from "src/common/entities/query.pagination"; @@ -159,19 +158,12 @@ export class CacheWarmerService { const count = await this.poolService.getPoolCountRaw(); await this.invalidateKey(CacheInfo.TransactionPoolCount().key, count, ttl); - let pool; - try { - pool = await this.poolService.getTxPoolRaw(); - } catch (error) { - // nothing else to warm while the pool is too large, which getTxPoolRaw already remembers - if (error instanceof TransactionPoolTooLargeException) { - return; - } - - throw error; - } - + // null when the pool is too large, which is cached as well, so that requests answer that from the cache + const pool = await this.poolService.getTxPoolRaw(); await this.invalidateKey(CacheInfo.TransactionPool.key, pool, ttl); + if (pool === null) { + return; + } // counted from the pool warmed above for (const type of Object.values(TransactionType)) { diff --git a/src/endpoints/pool/pool.service.ts b/src/endpoints/pool/pool.service.ts index f478486d2b..1705cc201c 100644 --- a/src/endpoints/pool/pool.service.ts +++ b/src/endpoints/pool/pool.service.ts @@ -76,21 +76,19 @@ export class PoolService { CacheInfo.TransactionPool.ttl, ); - return this.applyFilters(pool, filter); - } - - // a pool found too large is remembered for a while, so that it is not downloaded again, up to the size - // limit, on every request until then - async getTxPoolRaw(): Promise { - const isTooLarge = await this.cacheService.get(CacheInfo.TransactionPoolTooLarge.key); - if (isTooLarge) { + if (pool === null) { throw new TransactionPoolTooLargeException(); } + return this.applyFilters(pool, filter); + } + + // null only when the pool is too large to be read. it is cached like the pool itself, so that until it + // expires the pool is not downloaded again, up to the size limit, on every request + async getTxPoolRaw(): Promise { const pool = await this.gatewayService.getTransactionPool(); if (!pool) { - await this.cacheService.set(CacheInfo.TransactionPoolTooLarge.key, true, CacheInfo.TransactionPoolTooLarge.ttl); - throw new TransactionPoolTooLargeException(); + return null; } return this.parseTransactions(pool); diff --git a/src/test/unit/services/cache.warmer.pool.spec.ts b/src/test/unit/services/cache.warmer.pool.spec.ts index 5cf541d248..a30bf439e0 100644 --- a/src/test/unit/services/cache.warmer.pool.spec.ts +++ b/src/test/unit/services/cache.warmer.pool.spec.ts @@ -1,6 +1,5 @@ import { Locker, LockResult } from "@multiversx/sdk-nestjs-common"; import { CacheWarmerService } from "src/crons/cache.warmer/cache.warmer.service"; -import { TransactionPoolTooLargeException } from "src/endpoints/pool/entities/transaction.pool.too.large.exception"; import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; import { CacheInfo } from "src/utils/cache.info"; @@ -50,12 +49,14 @@ describe('CacheWarmerService transaction pool', () => { } }); - it('should warm only the total count while the pool is too large', async () => { - poolService.getTxPoolRaw.mockRejectedValue(new TransactionPoolTooLargeException()); + it('should cache the pool as null, besides the total count, while it is too large', async () => { + poolService.getTxPoolRaw.mockResolvedValue(null); await warmer.handleTxPoolInvalidations(); - expect(cachingService.set).toHaveBeenCalledTimes(1); + expect(cachingService.set).toHaveBeenCalledTimes(2); expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, counts.total, ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, null, ttl); + expect(poolService.getPoolCountRaw).toHaveBeenCalledTimes(1); }); }); diff --git a/src/test/unit/services/pool.spec.ts b/src/test/unit/services/pool.spec.ts index 5434a37e4a..3372822e2f 100644 --- a/src/test/unit/services/pool.spec.ts +++ b/src/test/unit/services/pool.spec.ts @@ -31,8 +31,6 @@ describe('PoolService', () => { provide: CacheService, useValue: { getOrSet: jest.fn(), - get: jest.fn(), - set: jest.fn(), }, }, { @@ -143,22 +141,34 @@ describe('PoolService', () => { }); }); - describe('getTxPoolRaw', () => { - it('should remember a pool too large for the gateway response limit', async () => { + describe('pool too large', () => { + it('should read a pool too large for the gateway response limit as null', async () => { gatewayService.getTransactionPool = jest.fn().mockResolvedValue(undefined); - await expect(service.getTxPoolRaw()).rejects.toBeInstanceOf(TransactionPoolTooLargeException); - expect(cacheService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolTooLarge.key, true, CacheInfo.TransactionPoolTooLarge.ttl); + expect(await service.getTxPoolRaw()).toBeNull(); }); - it('should not download the pool again while it is remembered as too large', async () => { - cacheService.get = jest.fn().mockResolvedValue(true); + it('should answer from the cached null without downloading the pool again', async () => { + cacheService.getOrSet = jest.fn().mockImplementation(async (key: string, createValueFunc: () => Promise) => { + return key === CacheInfo.TransactionPool.key ? null : await createValueFunc(); + }); gatewayService.getTransactionPool = jest.fn(); - await expect(service.getTxPoolRaw()).rejects.toBeInstanceOf(TransactionPoolTooLargeException); + await expect(service.getPool(new QueryPagination(), new PoolFilter())).rejects.toBeInstanceOf(TransactionPoolTooLargeException); + await expect(service.getTransactionFromPool('e07af9835b6da5740d0f791cfe65491a562852c57d44af63fdc14be5d73f01da')).rejects.toBeInstanceOf(TransactionPoolTooLargeException); + await expect(service.getPoolCount(new PoolFilter({ type: TransactionType.Reward }))).rejects.toBeInstanceOf(TransactionPoolTooLargeException); expect(gatewayService.getTransactionPool).not.toHaveBeenCalled(); }); + it('should still count the total through the gateway', async () => { + gatewayService.getTransactionPoolCount = jest.fn().mockResolvedValue(42); + cacheService.getOrSet = jest.fn().mockImplementation(async (key: string, createValueFunc: () => Promise) => { + return key === CacheInfo.TransactionPool.key ? null : await createValueFunc(); + }); + + expect(await service.getPoolCount(new PoolFilter())).toStrictEqual(42); + }); + it('should answer with a code the clients can recognize', () => { const exception = new TransactionPoolTooLargeException(); expect(exception.getStatus()).toStrictEqual(503); diff --git a/src/utils/cache.info.ts b/src/utils/cache.info.ts index 37bd993718..af6ec15098 100644 --- a/src/utils/cache.info.ts +++ b/src/utils/cache.info.ts @@ -65,11 +65,6 @@ export class CacheInfo { }; } - static TransactionPoolTooLarge: CacheInfo = { - key: 'txpool:tooLarge', - ttl: Constants.oneMinute(), - }; - static ApplicationMostUsed: CacheInfo = { key: 'applicationMostUsed', ttl: Constants.oneHour(), From d51c3164f7131e8f975f445cb2805ebe2bd7f78d Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Tue, 29 Sep 2026 11:54:08 +0300 Subject: [PATCH 04/20] explicit cacheNullable for txpool and warm pool counts in parallel --- .../cache.warmer/cache.warmer.service.ts | 32 ++++++++++--------- src/endpoints/pool/pool.service.ts | 2 ++ .../unit/services/cache.warmer.pool.spec.ts | 15 +++++++++ src/test/unit/services/pool.spec.ts | 2 +- 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/crons/cache.warmer/cache.warmer.service.ts b/src/crons/cache.warmer/cache.warmer.service.ts index bbc7a5b7d5..c9d2c6e190 100644 --- a/src/crons/cache.warmer/cache.warmer.service.ts +++ b/src/crons/cache.warmer/cache.warmer.service.ts @@ -154,22 +154,24 @@ export class CacheWarmerService { async handleTxPoolInvalidations() { const ttl = this.apiConfigService.getTransactionPoolCacheWarmerTtlInSeconds(); - // the gateway counts the pool itself, so the total is warmed even while the pool is too large to be read - const count = await this.poolService.getPoolCountRaw(); - await this.invalidateKey(CacheInfo.TransactionPoolCount().key, count, ttl); - - // null when the pool is too large, which is cached as well, so that requests answer that from the cache - const pool = await this.poolService.getTxPoolRaw(); - await this.invalidateKey(CacheInfo.TransactionPool.key, pool, ttl); - if (pool === null) { - return; - } + const warmCount = async (type?: TransactionType) => { + const count = await this.poolService.getPoolCountRaw(type); + await this.invalidateKey(CacheInfo.TransactionPoolCount(type).key, count, ttl); + }; + + const warmPool = async () => { + // null when the pool is too large, which is cached as well, so that requests answer that from the cache + const pool = await this.poolService.getTxPoolRaw(); + await this.invalidateKey(CacheInfo.TransactionPool.key, pool, ttl); + + // counted from the pool warmed above, so only once it is there + if (pool !== null) { + await Promise.all(Object.values(TransactionType).map(type => warmCount(type))); + } + }; - // counted from the pool warmed above - for (const type of Object.values(TransactionType)) { - const typeCount = await this.poolService.getPoolCountRaw(type); - await this.invalidateKey(CacheInfo.TransactionPoolCount(type).key, typeCount, ttl); - } + // the gateway counts the pool itself, so the total is warmed even while the pool is too large to be read + await Promise.all([warmCount(), warmPool()]); } @Cron('*/2 * * * *') diff --git a/src/endpoints/pool/pool.service.ts b/src/endpoints/pool/pool.service.ts index 1705cc201c..c87204411a 100644 --- a/src/endpoints/pool/pool.service.ts +++ b/src/endpoints/pool/pool.service.ts @@ -74,6 +74,8 @@ export class PoolService { CacheInfo.TransactionPool.key, async () => await this.getTxPoolRaw(), CacheInfo.TransactionPool.ttl, + CacheInfo.TransactionPool.ttl, + true, // cacheNullable: null stands for a pool too large to be read, see getTxPoolRaw ); if (pool === null) { diff --git a/src/test/unit/services/cache.warmer.pool.spec.ts b/src/test/unit/services/cache.warmer.pool.spec.ts index a30bf439e0..abef9b9708 100644 --- a/src/test/unit/services/cache.warmer.pool.spec.ts +++ b/src/test/unit/services/cache.warmer.pool.spec.ts @@ -49,6 +49,21 @@ describe('CacheWarmerService transaction pool', () => { } }); + it('should request the total count and the pool together', async () => { + let releaseCount: () => void = () => { }; + poolService.getPoolCountRaw.mockImplementationOnce(async () => await new Promise(resolve => releaseCount = () => resolve(counts.total))); + + const warming = warmer.handleTxPoolInvalidations(); + await new Promise(resolve => setImmediate(resolve)); + + // the pool was requested while the total count is still pending + expect(poolService.getTxPoolRaw).toHaveBeenCalled(); + + releaseCount(); + await warming; + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, counts.total, ttl); + }); + it('should cache the pool as null, besides the total count, while it is too large', async () => { poolService.getTxPoolRaw.mockResolvedValue(null); diff --git a/src/test/unit/services/pool.spec.ts b/src/test/unit/services/pool.spec.ts index 3372822e2f..22eeaa5cea 100644 --- a/src/test/unit/services/pool.spec.ts +++ b/src/test/unit/services/pool.spec.ts @@ -122,7 +122,7 @@ describe('PoolService', () => { const poolCount = await service.getPoolCount(filter); expect(cacheService.getOrSet).toHaveBeenCalledTimes(1); - expect(cacheService.getOrSet).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, expect.any(Function), CacheInfo.TransactionPool.ttl); + expect(cacheService.getOrSet).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, expect.any(Function), CacheInfo.TransactionPool.ttl, CacheInfo.TransactionPool.ttl, true); const pool = await service.getPool(new QueryPagination({ from: 0, size: 100 }), filter); expect(pool.length).toBeGreaterThan(0); From 94425664140e85867fc0e29bb978b1614146dd36 Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Tue, 29 Sep 2026 12:02:25 +0300 Subject: [PATCH 05/20] answer pool too large with custom response on rest and websocket --- src/crons/websocket/pool.gateway.ts | 20 ++++- .../transaction.pool.too.large.error.ts | 7 ++ .../transaction.pool.too.large.exception.ts | 15 ---- .../entities/transaction.pool.too.large.ts | 9 ++ src/endpoints/pool/pool.controller.ts | 56 +++++++++---- src/endpoints/pool/pool.service.ts | 4 +- src/test/unit/services/pool.spec.ts | 14 +--- src/test/unit/services/pool.too.large.spec.ts | 83 +++++++++++++++++++ 8 files changed, 162 insertions(+), 46 deletions(-) create mode 100644 src/endpoints/pool/entities/transaction.pool.too.large.error.ts delete mode 100644 src/endpoints/pool/entities/transaction.pool.too.large.exception.ts create mode 100644 src/endpoints/pool/entities/transaction.pool.too.large.ts create mode 100644 src/test/unit/services/pool.too.large.spec.ts diff --git a/src/crons/websocket/pool.gateway.ts b/src/crons/websocket/pool.gateway.ts index 722466a9e2..d4142eb3d3 100644 --- a/src/crons/websocket/pool.gateway.ts +++ b/src/crons/websocket/pool.gateway.ts @@ -10,6 +10,7 @@ import { QueryPagination } from 'src/common/entities/query.pagination'; import { PoolSubscribePayload } from '../../endpoints/pool/entities/pool.subscribe'; import { RoomKeyGenerator } from './room.key.generator'; import { LockingGuardInterceptor } from 'src/utils/locking.guard.interceptor'; +import { TransactionPoolTooLargeError } from '../../endpoints/pool/entities/transaction.pool.too.large.error'; @UseFilters(WebsocketExceptionsFilter) @WebSocketGateway({ cors: { origin: '*' }, path: '/ws/subscription' }) @@ -70,16 +71,31 @@ export class PoolGateway { size: filter.size, }), poolFilter, - ), - this.poolService.getPoolCount(poolFilter), + ).catch(error => this.nullIfTooLarge(error)), + this.poolService.getPoolCount(poolFilter).catch(error => this.nullIfTooLarge(error)), ]); + if (pool === null) { + // the total count comes from the gateway, so it is still sent unless the room filters by type + this.server.to(roomName).emit("poolUpdate", { pool: [], poolCount, tooLarge: true }); + return; + } + this.server.to(roomName).emit("poolUpdate", { pool, poolCount }); } catch (error) { this.logger.error(error); } } + // a pool too large to be read is sent to the clients as such, instead of being logged as an error + private nullIfTooLarge(error: any): null { + if (error instanceof TransactionPoolTooLargeError) { + return null; + } + + throw error; + } + async pushPool(): Promise { const promises: Promise[] = []; diff --git a/src/endpoints/pool/entities/transaction.pool.too.large.error.ts b/src/endpoints/pool/entities/transaction.pool.too.large.error.ts new file mode 100644 index 0000000000..4593cc2fd3 --- /dev/null +++ b/src/endpoints/pool/entities/transaction.pool.too.large.error.ts @@ -0,0 +1,7 @@ +// thrown when the pool is too large to be read from the gateway, which limits the size of its response. +// the controller and the websocket gateway answer it with TransactionPoolTooLarge instead of an error +export class TransactionPoolTooLargeError extends Error { + constructor() { + super('The transaction pool is too large to be read'); + } +} diff --git a/src/endpoints/pool/entities/transaction.pool.too.large.exception.ts b/src/endpoints/pool/entities/transaction.pool.too.large.exception.ts deleted file mode 100644 index b679891c40..0000000000 --- a/src/endpoints/pool/entities/transaction.pool.too.large.exception.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { HttpException, HttpStatus } from "@nestjs/common"; - -// the pool is read from the gateway in a single response, which is limited in size. above that limit there -// is no pool to show, and clients can tell this case apart from other failures by the code in the body -export class TransactionPoolTooLargeException extends HttpException { - static readonly code = 'transaction_pool_too_large'; - - constructor() { - super({ - statusCode: HttpStatus.SERVICE_UNAVAILABLE, - code: TransactionPoolTooLargeException.code, - message: 'The transaction pool is too large to be displayed', - }, HttpStatus.SERVICE_UNAVAILABLE); - } -} diff --git a/src/endpoints/pool/entities/transaction.pool.too.large.ts b/src/endpoints/pool/entities/transaction.pool.too.large.ts new file mode 100644 index 0000000000..290be6ac67 --- /dev/null +++ b/src/endpoints/pool/entities/transaction.pool.too.large.ts @@ -0,0 +1,9 @@ +import { ApiProperty } from "@nestjs/swagger"; + +export class TransactionPoolTooLarge { + @ApiProperty({ type: Boolean, example: true, description: 'Set only when the transaction pool is too large to be displayed' }) + tooLarge: boolean = true; + + @ApiProperty({ type: String, example: 'The transaction pool is too large to be displayed' }) + message: string = 'The transaction pool is too large to be displayed'; +} diff --git a/src/endpoints/pool/pool.controller.ts b/src/endpoints/pool/pool.controller.ts index 81c6b1d926..ee357781c7 100644 --- a/src/endpoints/pool/pool.controller.ts +++ b/src/endpoints/pool/pool.controller.ts @@ -1,24 +1,42 @@ import { ParseAddressAndMetachainPipe, ParseAddressPipe, ParseEnumPipe, ParseIntPipe, ParseTransactionHashPipe, ParseArrayPipe } from "@multiversx/sdk-nestjs-common"; import { Controller, DefaultValuePipe, Get, NotFoundException, Param, Query } from "@nestjs/common"; -import { ApiExcludeEndpoint, ApiNotFoundResponse, ApiOkResponse, ApiOperation, ApiQuery, ApiServiceUnavailableResponse, ApiTags } from "@nestjs/swagger"; +import { ApiExcludeEndpoint, ApiExtraModels, ApiNotFoundResponse, ApiOkResponse, ApiOperation, ApiQuery, ApiTags, getSchemaPath } from "@nestjs/swagger"; import { PoolService } from "./pool.service"; import { QueryPagination } from "src/common/entities/query.pagination"; import { TransactionInPool } from "./entities/transaction.in.pool.dto"; import { TransactionType } from "../transactions/entities/transaction.type"; import { PoolFilter } from "./entities/pool.filter"; import { ParseArrayPipeOptions } from "@multiversx/sdk-nestjs-common/lib/pipes/entities/parse.array.options"; +import { TransactionPoolTooLarge } from "./entities/transaction.pool.too.large"; +import { TransactionPoolTooLargeError } from "./entities/transaction.pool.too.large.error"; @Controller() @ApiTags('pool') +@ApiExtraModels(TransactionInPool, TransactionPoolTooLarge) export class PoolController { constructor( private readonly poolService: PoolService, ) { } + // a pool too large to be read is not an error for the clients, which show that instead of its content + private async unlessTooLarge(getResult: () => Promise): Promise { + try { + return await getResult(); + } catch (error) { + if (error instanceof TransactionPoolTooLargeError) { + return new TransactionPoolTooLarge(); + } + + throw error; + } + } + @Get("/pool") @ApiOperation({ summary: 'Transactions pool', description: 'Returns the transactions that are currently in the memory pool.' }) - @ApiOkResponse({ type: TransactionInPool, isArray: true }) - @ApiServiceUnavailableResponse({ description: 'The transaction pool is too large to be displayed' }) + @ApiOkResponse({ + description: 'The transactions, or TransactionPoolTooLarge when the pool is too large to be displayed', + schema: { oneOf: [{ type: 'array', items: { $ref: getSchemaPath(TransactionInPool) } }, { $ref: getSchemaPath(TransactionPoolTooLarge) }] }, + }) @ApiQuery({ name: 'from', description: 'Number of items to skip for the result set', required: false }) @ApiQuery({ name: 'size', description: 'Number of items to retrieve', required: false }) @ApiQuery({ name: 'sender', description: 'Search in transaction pool by a specific sender', required: false }) @@ -37,21 +55,23 @@ export class PoolController { @Query('receiverShard', ParseIntPipe) receiverShard?: number, @Query('type', new ParseEnumPipe(TransactionType)) type?: TransactionType, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], - ): Promise { - return await this.poolService.getPool(new QueryPagination({ from, size }), new PoolFilter({ + ): Promise { + return await this.unlessTooLarge(() => this.poolService.getPool(new QueryPagination({ from, size }), new PoolFilter({ sender: sender, receiver: receiver, senderShard: senderShard, receiverShard: receiverShard, type: type, functions: functions, - })); + }))); } @Get("/pool/count") @ApiOperation({ summary: 'Transactions pool count', description: 'Returns the number of transactions that are currently in the memory pool.' }) - @ApiOkResponse({ type: Number }) - @ApiServiceUnavailableResponse({ description: 'The transaction pool is too large to be displayed' }) + @ApiOkResponse({ + description: 'The count, or TransactionPoolTooLarge when it needs the pool and the pool is too large to be read', + schema: { oneOf: [{ type: 'number' }, { $ref: getSchemaPath(TransactionPoolTooLarge) }] }, + }) @ApiQuery({ name: 'sender', description: 'Returns the number of transactions with a specific sender', required: false }) @ApiQuery({ name: 'receiver', description: 'Search in transaction pool by a specific receiver', required: false }) @ApiQuery({ name: 'senderShard', description: 'The shard of the sender', required: false }) @@ -63,14 +83,14 @@ export class PoolController { @Query('senderShard', ParseIntPipe) senderShard?: number, @Query('receiverShard', ParseIntPipe) receiverShard?: number, @Query('type', new ParseEnumPipe(TransactionType)) type?: TransactionType, - ): Promise { - return await this.poolService.getPoolCount(new PoolFilter({ + ): Promise { + return await this.unlessTooLarge(() => this.poolService.getPoolCount(new PoolFilter({ sender: sender, receiver: receiver, senderShard: senderShard, receiverShard: receiverShard, type: type, - })); + }))); } @Get("/pool/c") @@ -79,19 +99,21 @@ export class PoolController { @Query('sender', ParseAddressAndMetachainPipe) sender?: string, @Query('receiver', ParseAddressPipe) receiver?: string, @Query('type', new ParseEnumPipe(TransactionType)) type?: TransactionType, - ): Promise { - return await this.poolService.getPoolCount(new PoolFilter({ sender, receiver, type })); + ): Promise { + return await this.unlessTooLarge(() => this.poolService.getPoolCount(new PoolFilter({ sender, receiver, type }))); } @Get("/pool/:txhash") @ApiOperation({ summary: 'Transaction from pool', description: 'Returns a transaction from the memory pool.' }) - @ApiOkResponse({ type: TransactionInPool }) + @ApiOkResponse({ + description: 'The transaction, or TransactionPoolTooLarge when the pool is too large to be searched', + schema: { oneOf: [{ $ref: getSchemaPath(TransactionInPool) }, { $ref: getSchemaPath(TransactionPoolTooLarge) }] }, + }) @ApiNotFoundResponse({ description: 'Transaction not found' }) - @ApiServiceUnavailableResponse({ description: 'The transaction pool is too large to be displayed' }) async getTransactionFromPool( @Param('txhash', ParseTransactionHashPipe) txHash: string, - ): Promise { - const transaction = await this.poolService.getTransactionFromPool(txHash); + ): Promise { + const transaction = await this.unlessTooLarge(() => this.poolService.getTransactionFromPool(txHash)); if (transaction === undefined) { throw new NotFoundException('Transaction not found'); } diff --git a/src/endpoints/pool/pool.service.ts b/src/endpoints/pool/pool.service.ts index c87204411a..716949ccdd 100644 --- a/src/endpoints/pool/pool.service.ts +++ b/src/endpoints/pool/pool.service.ts @@ -12,7 +12,7 @@ import { TxInPoolFields } from "src/common/gateway/entities/tx.in.pool.fields"; import { TransactionActionService } from "../transactions/transaction-action/transaction.action.service"; import { Transaction } from "../transactions/entities/transaction"; import { ApiUtils } from "@multiversx/sdk-nestjs-http"; -import { TransactionPoolTooLargeException } from "./entities/transaction.pool.too.large.exception"; +import { TransactionPoolTooLargeError } from "./entities/transaction.pool.too.large.error"; @Injectable() export class PoolService { @@ -79,7 +79,7 @@ export class PoolService { ); if (pool === null) { - throw new TransactionPoolTooLargeException(); + throw new TransactionPoolTooLargeError(); } return this.applyFilters(pool, filter); diff --git a/src/test/unit/services/pool.spec.ts b/src/test/unit/services/pool.spec.ts index 22eeaa5cea..5f2798341c 100644 --- a/src/test/unit/services/pool.spec.ts +++ b/src/test/unit/services/pool.spec.ts @@ -8,7 +8,7 @@ import { PoolService } from "src/endpoints/pool/pool.service"; import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; import { ProtocolService } from "../../../common/protocol/protocol.service"; import { TransactionActionService } from "../../../endpoints/transactions/transaction-action/transaction.action.service"; -import { TransactionPoolTooLargeException } from "src/endpoints/pool/entities/transaction.pool.too.large.exception"; +import { TransactionPoolTooLargeError } from "src/endpoints/pool/entities/transaction.pool.too.large.error"; import { CacheInfo } from "src/utils/cache.info"; describe('PoolService', () => { @@ -154,9 +154,9 @@ describe('PoolService', () => { }); gatewayService.getTransactionPool = jest.fn(); - await expect(service.getPool(new QueryPagination(), new PoolFilter())).rejects.toBeInstanceOf(TransactionPoolTooLargeException); - await expect(service.getTransactionFromPool('e07af9835b6da5740d0f791cfe65491a562852c57d44af63fdc14be5d73f01da')).rejects.toBeInstanceOf(TransactionPoolTooLargeException); - await expect(service.getPoolCount(new PoolFilter({ type: TransactionType.Reward }))).rejects.toBeInstanceOf(TransactionPoolTooLargeException); + await expect(service.getPool(new QueryPagination(), new PoolFilter())).rejects.toBeInstanceOf(TransactionPoolTooLargeError); + await expect(service.getTransactionFromPool('e07af9835b6da5740d0f791cfe65491a562852c57d44af63fdc14be5d73f01da')).rejects.toBeInstanceOf(TransactionPoolTooLargeError); + await expect(service.getPoolCount(new PoolFilter({ type: TransactionType.Reward }))).rejects.toBeInstanceOf(TransactionPoolTooLargeError); expect(gatewayService.getTransactionPool).not.toHaveBeenCalled(); }); @@ -168,12 +168,6 @@ describe('PoolService', () => { expect(await service.getPoolCount(new PoolFilter())).toStrictEqual(42); }); - - it('should answer with a code the clients can recognize', () => { - const exception = new TransactionPoolTooLargeException(); - expect(exception.getStatus()).toStrictEqual(503); - expect(exception.getResponse()).toEqual(expect.objectContaining({ code: 'transaction_pool_too_large' })); - }); }); describe('getTransactionFromPool', () => { diff --git a/src/test/unit/services/pool.too.large.spec.ts b/src/test/unit/services/pool.too.large.spec.ts new file mode 100644 index 0000000000..343cb68984 --- /dev/null +++ b/src/test/unit/services/pool.too.large.spec.ts @@ -0,0 +1,83 @@ +import { NotFoundException } from "@nestjs/common"; +import { PoolGateway } from "src/crons/websocket/pool.gateway"; +import { PoolFilter } from "src/endpoints/pool/entities/pool.filter"; +import { TransactionPoolTooLarge } from "src/endpoints/pool/entities/transaction.pool.too.large"; +import { TransactionPoolTooLargeError } from "src/endpoints/pool/entities/transaction.pool.too.large.error"; +import { PoolController } from "src/endpoints/pool/pool.controller"; +import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; + +describe('Transaction pool too large', () => { + const tooLarge = { tooLarge: true, message: 'The transaction pool is too large to be displayed' }; + const txHash = 'e07af9835b6da5740d0f791cfe65491a562852c57d44af63fdc14be5d73f01da'; + + let poolService: any; + + beforeEach(() => { + poolService = { + getPool: jest.fn().mockRejectedValue(new TransactionPoolTooLargeError()), + getTransactionFromPool: jest.fn().mockRejectedValue(new TransactionPoolTooLargeError()), + // the total comes from the gateway, only counts that need the pool fail + getPoolCount: jest.fn().mockImplementation(async (filter: PoolFilter) => { + return filter.type ? await Promise.reject(new TransactionPoolTooLargeError()) : 42; + }), + }; + }); + + describe('PoolController', () => { + let controller: PoolController; + + beforeEach(() => { + controller = new PoolController(poolService); + }); + + it('should answer the pool, the transaction and the counts that need the pool with a custom response', async () => { + expect(await controller.getTransactionPool(0, 25)).toStrictEqual(new TransactionPoolTooLarge()); + expect(await controller.getTransactionPool(0, 25)).toEqual(tooLarge); + expect(await controller.getTransactionFromPool(txHash)).toEqual(tooLarge); + expect(await controller.getTransactionPoolCount(undefined, undefined, undefined, undefined, TransactionType.Reward)).toEqual(tooLarge); + }); + + it('should still answer the total count', async () => { + expect(await controller.getTransactionPoolCount()).toStrictEqual(42); + }); + + it('should keep answering other failures as errors', async () => { + poolService.getPool.mockRejectedValue(new Error('gateway unreachable')); + poolService.getTransactionFromPool.mockResolvedValue(undefined); + + await expect(controller.getTransactionPool(0, 25)).rejects.toThrow('gateway unreachable'); + await expect(controller.getTransactionFromPool(txHash)).rejects.toBeInstanceOf(NotFoundException); + }); + }); + + describe('PoolGateway', () => { + let gateway: PoolGateway; + let emit: jest.Mock; + + beforeEach(() => { + emit = jest.fn(); + gateway = new PoolGateway(poolService); + gateway.server = { to: jest.fn().mockReturnValue({ emit }) } as any; + }); + + it('should send the room an empty pool marked as too large, with the total count', async () => { + await gateway.pushPoolForRoom('pool-{"from":0,"size":25}'); + + expect(emit).toHaveBeenCalledWith('poolUpdate', { pool: [], poolCount: 42, tooLarge: true }); + }); + + it('should send no count to a room that filters by type, as that count needs the pool', async () => { + await gateway.pushPoolForRoom(`pool-{"from":0,"size":25,"type":"${TransactionType.Reward}"}`); + + expect(emit).toHaveBeenCalledWith('poolUpdate', { pool: [], poolCount: null, tooLarge: true }); + }); + + it('should send the pool as before when it can be read', async () => { + poolService.getPool.mockResolvedValue([{ txHash }]); + + await gateway.pushPoolForRoom('pool-{"from":0,"size":25}'); + + expect(emit).toHaveBeenCalledWith('poolUpdate', { pool: [{ txHash }], poolCount: 42 }); + }); + }); +}); From 3dcaea9ea35bf848bee2d1c217846460d294cec3 Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Tue, 29 Sep 2026 12:12:47 +0300 Subject: [PATCH 06/20] send pool status on websocket and warm pool counts with promise all --- .../cache.warmer/cache.warmer.service.ts | 30 ++++++++------- src/crons/websocket/pool.gateway.ts | 22 ++++------- .../pool/entities/pool.update.status.ts | 5 +++ .../transaction.pool.too.large.error.ts | 7 ---- src/endpoints/pool/pool.controller.ts | 37 +++++++++---------- src/endpoints/pool/pool.service.ts | 29 ++++++++++----- .../transactions/transaction.service.ts | 5 +++ .../unit/services/cache.warmer.pool.spec.ts | 31 +++++++++------- src/test/unit/services/pool.spec.ts | 18 ++++----- src/test/unit/services/pool.too.large.spec.ts | 37 +++++++++++++------ 10 files changed, 122 insertions(+), 99 deletions(-) create mode 100644 src/endpoints/pool/entities/pool.update.status.ts delete mode 100644 src/endpoints/pool/entities/transaction.pool.too.large.error.ts diff --git a/src/crons/cache.warmer/cache.warmer.service.ts b/src/crons/cache.warmer/cache.warmer.service.ts index c9d2c6e190..9b2999d4ad 100644 --- a/src/crons/cache.warmer/cache.warmer.service.ts +++ b/src/crons/cache.warmer/cache.warmer.service.ts @@ -154,24 +154,26 @@ export class CacheWarmerService { async handleTxPoolInvalidations() { const ttl = this.apiConfigService.getTransactionPoolCacheWarmerTtlInSeconds(); - const warmCount = async (type?: TransactionType) => { - const count = await this.poolService.getPoolCountRaw(type); - await this.invalidateKey(CacheInfo.TransactionPoolCount(type).key, count, ttl); - }; - - const warmPool = async () => { + // the gateway counts the pool itself, so the total is there even while the pool is too large to be read + const [count, pool] = await Promise.all([ + this.poolService.getPoolCountRaw(), + this.poolService.getTxPoolRaw(), + ]); + + const invalidations = [ + this.invalidateKey(CacheInfo.TransactionPoolCount().key, count, ttl), // null when the pool is too large, which is cached as well, so that requests answer that from the cache - const pool = await this.poolService.getTxPoolRaw(); - await this.invalidateKey(CacheInfo.TransactionPool.key, pool, ttl); + this.invalidateKey(CacheInfo.TransactionPool.key, pool, ttl), + ]; - // counted from the pool warmed above, so only once it is there - if (pool !== null) { - await Promise.all(Object.values(TransactionType).map(type => warmCount(type))); + if (pool !== null) { + for (const type of Object.values(TransactionType)) { + const typeCount = pool.filter(transaction => transaction.type === type).length; + invalidations.push(this.invalidateKey(CacheInfo.TransactionPoolCount(type).key, typeCount, ttl)); } - }; + } - // the gateway counts the pool itself, so the total is warmed even while the pool is too large to be read - await Promise.all([warmCount(), warmPool()]); + await Promise.all(invalidations); } @Cron('*/2 * * * *') diff --git a/src/crons/websocket/pool.gateway.ts b/src/crons/websocket/pool.gateway.ts index d4142eb3d3..fb9f955dd0 100644 --- a/src/crons/websocket/pool.gateway.ts +++ b/src/crons/websocket/pool.gateway.ts @@ -10,7 +10,7 @@ import { QueryPagination } from 'src/common/entities/query.pagination'; import { PoolSubscribePayload } from '../../endpoints/pool/entities/pool.subscribe'; import { RoomKeyGenerator } from './room.key.generator'; import { LockingGuardInterceptor } from 'src/utils/locking.guard.interceptor'; -import { TransactionPoolTooLargeError } from '../../endpoints/pool/entities/transaction.pool.too.large.error'; +import { PoolUpdateStatus } from '../../endpoints/pool/entities/pool.update.status'; @UseFilters(WebsocketExceptionsFilter) @WebSocketGateway({ cors: { origin: '*' }, path: '/ws/subscription' }) @@ -71,31 +71,23 @@ export class PoolGateway { size: filter.size, }), poolFilter, - ).catch(error => this.nullIfTooLarge(error)), - this.poolService.getPoolCount(poolFilter).catch(error => this.nullIfTooLarge(error)), + ), + this.poolService.getPoolCount(poolFilter), ]); + // the pool is null when it is too large to be read. the total count comes from the gateway, so it is + // still sent then, unless the room filters by type if (pool === null) { - // the total count comes from the gateway, so it is still sent unless the room filters by type - this.server.to(roomName).emit("poolUpdate", { pool: [], poolCount, tooLarge: true }); + this.server.to(roomName).emit("poolUpdate", { status: PoolUpdateStatus.tooLarge, pool: [], poolCount }); return; } - this.server.to(roomName).emit("poolUpdate", { pool, poolCount }); + this.server.to(roomName).emit("poolUpdate", { status: PoolUpdateStatus.success, pool, poolCount }); } catch (error) { this.logger.error(error); } } - // a pool too large to be read is sent to the clients as such, instead of being logged as an error - private nullIfTooLarge(error: any): null { - if (error instanceof TransactionPoolTooLargeError) { - return null; - } - - throw error; - } - async pushPool(): Promise { const promises: Promise[] = []; diff --git a/src/endpoints/pool/entities/pool.update.status.ts b/src/endpoints/pool/entities/pool.update.status.ts new file mode 100644 index 0000000000..6c8acb3fcc --- /dev/null +++ b/src/endpoints/pool/entities/pool.update.status.ts @@ -0,0 +1,5 @@ +// sent with every poolUpdate, so that clients know whether the pool it carries can be shown +export enum PoolUpdateStatus { + success = 'success', + tooLarge = 'tooLarge', +} diff --git a/src/endpoints/pool/entities/transaction.pool.too.large.error.ts b/src/endpoints/pool/entities/transaction.pool.too.large.error.ts deleted file mode 100644 index 4593cc2fd3..0000000000 --- a/src/endpoints/pool/entities/transaction.pool.too.large.error.ts +++ /dev/null @@ -1,7 +0,0 @@ -// thrown when the pool is too large to be read from the gateway, which limits the size of its response. -// the controller and the websocket gateway answer it with TransactionPoolTooLarge instead of an error -export class TransactionPoolTooLargeError extends Error { - constructor() { - super('The transaction pool is too large to be read'); - } -} diff --git a/src/endpoints/pool/pool.controller.ts b/src/endpoints/pool/pool.controller.ts index ee357781c7..3e4547e9ef 100644 --- a/src/endpoints/pool/pool.controller.ts +++ b/src/endpoints/pool/pool.controller.ts @@ -8,7 +8,6 @@ import { TransactionType } from "../transactions/entities/transaction.type"; import { PoolFilter } from "./entities/pool.filter"; import { ParseArrayPipeOptions } from "@multiversx/sdk-nestjs-common/lib/pipes/entities/parse.array.options"; import { TransactionPoolTooLarge } from "./entities/transaction.pool.too.large"; -import { TransactionPoolTooLargeError } from "./entities/transaction.pool.too.large.error"; @Controller() @ApiTags('pool') @@ -18,19 +17,6 @@ export class PoolController { private readonly poolService: PoolService, ) { } - // a pool too large to be read is not an error for the clients, which show that instead of its content - private async unlessTooLarge(getResult: () => Promise): Promise { - try { - return await getResult(); - } catch (error) { - if (error instanceof TransactionPoolTooLargeError) { - return new TransactionPoolTooLarge(); - } - - throw error; - } - } - @Get("/pool") @ApiOperation({ summary: 'Transactions pool', description: 'Returns the transactions that are currently in the memory pool.' }) @ApiOkResponse({ @@ -56,14 +42,17 @@ export class PoolController { @Query('type', new ParseEnumPipe(TransactionType)) type?: TransactionType, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], ): Promise { - return await this.unlessTooLarge(() => this.poolService.getPool(new QueryPagination({ from, size }), new PoolFilter({ + const pool = await this.poolService.getPool(new QueryPagination({ from, size }), new PoolFilter({ sender: sender, receiver: receiver, senderShard: senderShard, receiverShard: receiverShard, type: type, functions: functions, - }))); + })); + + // null when the pool is too large to be read + return pool ?? new TransactionPoolTooLarge(); } @Get("/pool/count") @@ -84,13 +73,16 @@ export class PoolController { @Query('receiverShard', ParseIntPipe) receiverShard?: number, @Query('type', new ParseEnumPipe(TransactionType)) type?: TransactionType, ): Promise { - return await this.unlessTooLarge(() => this.poolService.getPoolCount(new PoolFilter({ + const count = await this.poolService.getPoolCount(new PoolFilter({ sender: sender, receiver: receiver, senderShard: senderShard, receiverShard: receiverShard, type: type, - }))); + })); + + // null when the count needs the pool and the pool is too large to be read + return count ?? new TransactionPoolTooLarge(); } @Get("/pool/c") @@ -100,7 +92,8 @@ export class PoolController { @Query('receiver', ParseAddressPipe) receiver?: string, @Query('type', new ParseEnumPipe(TransactionType)) type?: TransactionType, ): Promise { - return await this.unlessTooLarge(() => this.poolService.getPoolCount(new PoolFilter({ sender, receiver, type }))); + const count = await this.poolService.getPoolCount(new PoolFilter({ sender, receiver, type })); + return count ?? new TransactionPoolTooLarge(); } @Get("/pool/:txhash") @@ -113,7 +106,11 @@ export class PoolController { async getTransactionFromPool( @Param('txhash', ParseTransactionHashPipe) txHash: string, ): Promise { - const transaction = await this.unlessTooLarge(() => this.poolService.getTransactionFromPool(txHash)); + const transaction = await this.poolService.getTransactionFromPool(txHash); + if (transaction === null) { + return new TransactionPoolTooLarge(); + } + if (transaction === undefined) { throw new NotFoundException('Transaction not found'); } diff --git a/src/endpoints/pool/pool.service.ts b/src/endpoints/pool/pool.service.ts index 716949ccdd..45b26b6d9e 100644 --- a/src/endpoints/pool/pool.service.ts +++ b/src/endpoints/pool/pool.service.ts @@ -12,7 +12,6 @@ import { TxInPoolFields } from "src/common/gateway/entities/tx.in.pool.fields"; import { TransactionActionService } from "../transactions/transaction-action/transaction.action.service"; import { Transaction } from "../transactions/entities/transaction"; import { ApiUtils } from "@multiversx/sdk-nestjs-http"; -import { TransactionPoolTooLargeError } from "./entities/transaction.pool.too.large.error"; @Injectable() export class PoolService { @@ -23,16 +22,22 @@ export class PoolService { private readonly transactionActionService: TransactionActionService, ) { } - async getTransactionFromPool(txHash: string): Promise { + // undefined when the transaction is not in the pool, null when the pool is too large to be read + async getTransactionFromPool(txHash: string): Promise { const pool = await this.getPoolWithFilters(); + if (pool === null) { + return null; + } + return pool.find(tx => tx.txHash === txHash); } - async getPoolCount(filter: PoolFilter): Promise { + // null when the count needs the pool and the pool is too large to be read + async getPoolCount(filter: PoolFilter): Promise { const { type, ...otherFilters } = filter; if (Object.values(otherFilters).some(value => value !== undefined)) { const pool = await this.getPoolWithFilters(filter); - return pool.length; + return pool === null ? null : pool.length; } // the total and the count for each type are cached on their own @@ -45,31 +50,37 @@ export class PoolService { // the gateway counts the pool itself, so the total needs neither the pool nor its size limit. counts by // type are taken from the pool - async getPoolCountRaw(type?: TransactionType): Promise { + async getPoolCountRaw(type?: TransactionType): Promise { if (type === undefined) { return await this.gatewayService.getTransactionPoolCount(); } const pool = await this.getPoolWithFilters(new PoolFilter({ type })); - return pool.length; + return pool === null ? null : pool.length; } + // null when the pool is too large to be read async getPool( queryPagination: QueryPagination, filter?: PoolFilter, - ): Promise { + ): Promise { if (!this.apiConfigService.isTransactionPoolEnabled()) { return []; } const { from, size } = queryPagination; const pool = await this.getPoolWithFilters(filter); + if (pool === null) { + return null; + } + return pool.slice(from, from + size); } + // null when the pool is too large to be read async getPoolWithFilters( filter?: PoolFilter, - ): Promise { + ): Promise { const pool = await this.cacheService.getOrSet( CacheInfo.TransactionPool.key, async () => await this.getTxPoolRaw(), @@ -79,7 +90,7 @@ export class PoolService { ); if (pool === null) { - throw new TransactionPoolTooLargeError(); + return null; } return this.applyFilters(pool, filter); diff --git a/src/endpoints/transactions/transaction.service.ts b/src/endpoints/transactions/transaction.service.ts index ab8577fc0e..81c72ae250 100644 --- a/src/endpoints/transactions/transaction.service.ts +++ b/src/endpoints/transactions/transaction.service.ts @@ -752,6 +752,11 @@ export class TransactionService { const poolTransactions = await this.poolService.getPoolWithFilters({ senderShard: shardId }); + // null when the pool is too large to be read, which leaves the prices unknown rather than at zero + if (poolTransactions === null) { + return null; + } + if (!poolTransactions || poolTransactions.length === 0) { return new PpuMetadata({ lastBlock: lastBlock, diff --git a/src/test/unit/services/cache.warmer.pool.spec.ts b/src/test/unit/services/cache.warmer.pool.spec.ts index abef9b9708..2e829565c0 100644 --- a/src/test/unit/services/cache.warmer.pool.spec.ts +++ b/src/test/unit/services/cache.warmer.pool.spec.ts @@ -5,12 +5,16 @@ import { CacheInfo } from "src/utils/cache.info"; describe('CacheWarmerService transaction pool', () => { const ttl = 10; - const pool = [{ txHash: 'a', type: TransactionType.Transaction }]; - const counts: Record = { - total: 12, - [TransactionType.Transaction]: 3, - [TransactionType.SmartContractResult]: 4, - [TransactionType.Reward]: 5, + const totalCount = 12; + const pool = [ + { txHash: 'a', type: TransactionType.Transaction }, + { txHash: 'b', type: TransactionType.Reward }, + { txHash: 'c', type: TransactionType.Reward }, + ]; + const typeCounts: Record = { + [TransactionType.Transaction]: 1, + [TransactionType.SmartContractResult]: 0, + [TransactionType.Reward]: 2, }; let warmer: CacheWarmerService; @@ -25,7 +29,7 @@ describe('CacheWarmerService transaction pool', () => { }); poolService = { - getPoolCountRaw: jest.fn().mockImplementation(async (type?: TransactionType) => await Promise.resolve(counts[type ?? 'total'])), + getPoolCountRaw: jest.fn().mockResolvedValue(totalCount), getTxPoolRaw: jest.fn().mockResolvedValue(pool), }; cachingService = { set: jest.fn() }; @@ -39,19 +43,20 @@ describe('CacheWarmerService transaction pool', () => { }); }); - it('should warm the pool, its total count and the count for every transaction type', async () => { + it('should warm the pool, its total count and the count for every transaction type, counted from that pool', async () => { await warmer.handleTxPoolInvalidations(); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, counts.total, ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, totalCount, ttl); expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, pool, ttl); for (const type of Object.values(TransactionType)) { - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(type).key, counts[type], ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(type).key, typeCounts[type], ttl); } + expect(poolService.getPoolCountRaw).toHaveBeenCalledTimes(1); }); it('should request the total count and the pool together', async () => { let releaseCount: () => void = () => { }; - poolService.getPoolCountRaw.mockImplementationOnce(async () => await new Promise(resolve => releaseCount = () => resolve(counts.total))); + poolService.getPoolCountRaw.mockImplementationOnce(async () => await new Promise(resolve => releaseCount = () => resolve(totalCount))); const warming = warmer.handleTxPoolInvalidations(); await new Promise(resolve => setImmediate(resolve)); @@ -61,7 +66,7 @@ describe('CacheWarmerService transaction pool', () => { releaseCount(); await warming; - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, counts.total, ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, totalCount, ttl); }); it('should cache the pool as null, besides the total count, while it is too large', async () => { @@ -70,7 +75,7 @@ describe('CacheWarmerService transaction pool', () => { await warmer.handleTxPoolInvalidations(); expect(cachingService.set).toHaveBeenCalledTimes(2); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, counts.total, ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, totalCount, ttl); expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, null, ttl); expect(poolService.getPoolCountRaw).toHaveBeenCalledTimes(1); }); diff --git a/src/test/unit/services/pool.spec.ts b/src/test/unit/services/pool.spec.ts index 5f2798341c..c4f83289ec 100644 --- a/src/test/unit/services/pool.spec.ts +++ b/src/test/unit/services/pool.spec.ts @@ -8,7 +8,6 @@ import { PoolService } from "src/endpoints/pool/pool.service"; import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; import { ProtocolService } from "../../../common/protocol/protocol.service"; import { TransactionActionService } from "../../../endpoints/transactions/transaction-action/transaction.action.service"; -import { TransactionPoolTooLargeError } from "src/endpoints/pool/entities/transaction.pool.too.large.error"; import { CacheInfo } from "src/utils/cache.info"; describe('PoolService', () => { @@ -78,7 +77,7 @@ describe('PoolService', () => { it('should work and return the pool', async () => { const pool = await service.getPool(new QueryPagination(), new PoolFilter()); expect(pool).toHaveLength(7); - expect(pool[0].type).toStrictEqual(TransactionType.Transaction); + expect(pool?.[0].type).toStrictEqual(TransactionType.Transaction); }); it('should work and return the pool with filters', async () => { @@ -95,7 +94,7 @@ describe('PoolService', () => { it('should work and return the pool with query pagination', async () => { const pool = await service.getPool(new QueryPagination({ from: 0, size: 2 }), new PoolFilter({ type: TransactionType.Reward })); expect(pool).toHaveLength(2); - expect(pool[0].type).toStrictEqual(TransactionType.Reward); + expect(pool?.[0].type).toStrictEqual(TransactionType.Reward); }); }); @@ -125,8 +124,8 @@ describe('PoolService', () => { expect(cacheService.getOrSet).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, expect.any(Function), CacheInfo.TransactionPool.ttl, CacheInfo.TransactionPool.ttl, true); const pool = await service.getPool(new QueryPagination({ from: 0, size: 100 }), filter); - expect(pool.length).toBeGreaterThan(0); - expect(poolCount).toStrictEqual(pool.length); + expect(pool?.length).toBeGreaterThan(0); + expect(poolCount).toStrictEqual(pool?.length); }); it('should work and return the pool count with filters', async () => { @@ -148,15 +147,16 @@ describe('PoolService', () => { expect(await service.getTxPoolRaw()).toBeNull(); }); - it('should answer from the cached null without downloading the pool again', async () => { + it('should answer null from the cached null, without downloading the pool again', async () => { cacheService.getOrSet = jest.fn().mockImplementation(async (key: string, createValueFunc: () => Promise) => { return key === CacheInfo.TransactionPool.key ? null : await createValueFunc(); }); gatewayService.getTransactionPool = jest.fn(); - await expect(service.getPool(new QueryPagination(), new PoolFilter())).rejects.toBeInstanceOf(TransactionPoolTooLargeError); - await expect(service.getTransactionFromPool('e07af9835b6da5740d0f791cfe65491a562852c57d44af63fdc14be5d73f01da')).rejects.toBeInstanceOf(TransactionPoolTooLargeError); - await expect(service.getPoolCount(new PoolFilter({ type: TransactionType.Reward }))).rejects.toBeInstanceOf(TransactionPoolTooLargeError); + expect(await service.getPool(new QueryPagination(), new PoolFilter())).toBeNull(); + expect(await service.getTransactionFromPool('e07af9835b6da5740d0f791cfe65491a562852c57d44af63fdc14be5d73f01da')).toBeNull(); + expect(await service.getPoolCount(new PoolFilter({ type: TransactionType.Reward }))).toBeNull(); + expect(await service.getPoolCount(new PoolFilter({ sender: 'erd1qqqqqqqqqqqqqpgqp699jngundfqw07d8jzkepucvpzush6k3wvqyc44rx' }))).toBeNull(); expect(gatewayService.getTransactionPool).not.toHaveBeenCalled(); }); diff --git a/src/test/unit/services/pool.too.large.spec.ts b/src/test/unit/services/pool.too.large.spec.ts index 343cb68984..a919439fbf 100644 --- a/src/test/unit/services/pool.too.large.spec.ts +++ b/src/test/unit/services/pool.too.large.spec.ts @@ -2,9 +2,10 @@ import { NotFoundException } from "@nestjs/common"; import { PoolGateway } from "src/crons/websocket/pool.gateway"; import { PoolFilter } from "src/endpoints/pool/entities/pool.filter"; import { TransactionPoolTooLarge } from "src/endpoints/pool/entities/transaction.pool.too.large"; -import { TransactionPoolTooLargeError } from "src/endpoints/pool/entities/transaction.pool.too.large.error"; +import { PoolUpdateStatus } from "src/endpoints/pool/entities/pool.update.status"; import { PoolController } from "src/endpoints/pool/pool.controller"; import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; +import { TransactionService } from "src/endpoints/transactions/transaction.service"; describe('Transaction pool too large', () => { const tooLarge = { tooLarge: true, message: 'The transaction pool is too large to be displayed' }; @@ -14,12 +15,11 @@ describe('Transaction pool too large', () => { beforeEach(() => { poolService = { - getPool: jest.fn().mockRejectedValue(new TransactionPoolTooLargeError()), - getTransactionFromPool: jest.fn().mockRejectedValue(new TransactionPoolTooLargeError()), - // the total comes from the gateway, only counts that need the pool fail - getPoolCount: jest.fn().mockImplementation(async (filter: PoolFilter) => { - return filter.type ? await Promise.reject(new TransactionPoolTooLargeError()) : 42; - }), + // null stands for a pool too large to be read + getPool: jest.fn().mockResolvedValue(null), + getTransactionFromPool: jest.fn().mockResolvedValue(null), + // the total comes from the gateway, only the counts that need the pool are null + getPoolCount: jest.fn().mockImplementation(async (filter: PoolFilter) => await Promise.resolve(filter.type ? null : 42)), }; }); @@ -50,6 +50,19 @@ describe('Transaction pool too large', () => { }); }); + describe('TransactionService price per unit', () => { + it('should leave the prices unknown, rather than at zero, while the pool is too large', async () => { + // only the dependencies of the price per unit, without the constructor + const transactionService: TransactionService = Object.assign(Object.create(TransactionService.prototype), { + blockService: { getBlocks: jest.fn().mockResolvedValue([{ nonce: 100 }]) }, + networkService: { getConstants: jest.fn().mockResolvedValue({ minGasLimit: 50000, gasPerDataByte: 1500, gasPriceModifier: '0.01' }) }, + poolService: { getPoolWithFilters: jest.fn().mockResolvedValue(null) }, + }); + + expect(await transactionService.getPpuByShardIdRaw(1)).toBeNull(); + }); + }); + describe('PoolGateway', () => { let gateway: PoolGateway; let emit: jest.Mock; @@ -60,24 +73,24 @@ describe('Transaction pool too large', () => { gateway.server = { to: jest.fn().mockReturnValue({ emit }) } as any; }); - it('should send the room an empty pool marked as too large, with the total count', async () => { + it('should send the room an empty pool with the too large status, and the total count', async () => { await gateway.pushPoolForRoom('pool-{"from":0,"size":25}'); - expect(emit).toHaveBeenCalledWith('poolUpdate', { pool: [], poolCount: 42, tooLarge: true }); + expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.tooLarge, pool: [], poolCount: 42 }); }); it('should send no count to a room that filters by type, as that count needs the pool', async () => { await gateway.pushPoolForRoom(`pool-{"from":0,"size":25,"type":"${TransactionType.Reward}"}`); - expect(emit).toHaveBeenCalledWith('poolUpdate', { pool: [], poolCount: null, tooLarge: true }); + expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.tooLarge, pool: [], poolCount: null }); }); - it('should send the pool as before when it can be read', async () => { + it('should send the pool with the success status when it can be read', async () => { poolService.getPool.mockResolvedValue([{ txHash }]); await gateway.pushPoolForRoom('pool-{"from":0,"size":25}'); - expect(emit).toHaveBeenCalledWith('poolUpdate', { pool: [{ txHash }], poolCount: 42 }); + expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.success, pool: [{ txHash }], poolCount: 42 }); }); }); }); From 5c203d3c79b42a0a212ad42012cdb6a13f71054b Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Tue, 29 Sep 2026 12:22:59 +0300 Subject: [PATCH 07/20] count pool by length and use gateway count only when pool is too large --- .../cache.warmer/cache.warmer.service.ts | 14 +++---- src/endpoints/pool/pool.service.ts | 16 +++++--- .../unit/services/cache.warmer.pool.spec.ts | 37 ++++++------------- src/test/unit/services/pool.spec.ts | 8 ++-- 4 files changed, 32 insertions(+), 43 deletions(-) diff --git a/src/crons/cache.warmer/cache.warmer.service.ts b/src/crons/cache.warmer/cache.warmer.service.ts index 9b2999d4ad..b9891b97f1 100644 --- a/src/crons/cache.warmer/cache.warmer.service.ts +++ b/src/crons/cache.warmer/cache.warmer.service.ts @@ -154,16 +154,16 @@ export class CacheWarmerService { async handleTxPoolInvalidations() { const ttl = this.apiConfigService.getTransactionPoolCacheWarmerTtlInSeconds(); - // the gateway counts the pool itself, so the total is there even while the pool is too large to be read - const [count, pool] = await Promise.all([ - this.poolService.getPoolCountRaw(), - this.poolService.getTxPoolRaw(), - ]); + // null when the pool is too large, which is cached as well, so that requests answer that from the cache + const pool = await this.poolService.getTxPoolRaw(); + + // counted from the pool, so that the total matches what the pool lists. only when the pool is too large to + // be read does the total come from the gateway, which counts the pool itself + const count = pool !== null ? pool.length : await this.gatewayService.getTransactionPoolCount(); const invalidations = [ - this.invalidateKey(CacheInfo.TransactionPoolCount().key, count, ttl), - // null when the pool is too large, which is cached as well, so that requests answer that from the cache this.invalidateKey(CacheInfo.TransactionPool.key, pool, ttl), + this.invalidateKey(CacheInfo.TransactionPoolCount().key, count, ttl), ]; if (pool !== null) { diff --git a/src/endpoints/pool/pool.service.ts b/src/endpoints/pool/pool.service.ts index 45b26b6d9e..74602e279f 100644 --- a/src/endpoints/pool/pool.service.ts +++ b/src/endpoints/pool/pool.service.ts @@ -48,15 +48,19 @@ export class PoolService { ); } - // the gateway counts the pool itself, so the total needs neither the pool nor its size limit. counts by - // type are taken from the pool + // counted from the pool, so that the count matches what the pool lists. only when the pool is too large to + // be read does the total come from the gateway, which counts the pool itself; counts by type are then null async getPoolCountRaw(type?: TransactionType): Promise { - if (type === undefined) { - return await this.gatewayService.getTransactionPoolCount(); + const pool = await this.getPoolWithFilters(new PoolFilter({ type })); + if (pool !== null) { + return pool.length; } - const pool = await this.getPoolWithFilters(new PoolFilter({ type })); - return pool === null ? null : pool.length; + if (type !== undefined) { + return null; + } + + return await this.gatewayService.getTransactionPoolCount(); } // null when the pool is too large to be read diff --git a/src/test/unit/services/cache.warmer.pool.spec.ts b/src/test/unit/services/cache.warmer.pool.spec.ts index 2e829565c0..7b6125ef76 100644 --- a/src/test/unit/services/cache.warmer.pool.spec.ts +++ b/src/test/unit/services/cache.warmer.pool.spec.ts @@ -5,7 +5,7 @@ import { CacheInfo } from "src/utils/cache.info"; describe('CacheWarmerService transaction pool', () => { const ttl = 10; - const totalCount = 12; + const gatewayCount = 12; const pool = [ { txHash: 'a', type: TransactionType.Transaction }, { txHash: 'b', type: TransactionType.Reward }, @@ -19,6 +19,7 @@ describe('CacheWarmerService transaction pool', () => { let warmer: CacheWarmerService; let poolService: any; + let gatewayService: any; let cachingService: any; beforeEach(() => { @@ -28,55 +29,39 @@ describe('CacheWarmerService transaction pool', () => { return LockResult.SUCCESS; }); - poolService = { - getPoolCountRaw: jest.fn().mockResolvedValue(totalCount), - getTxPoolRaw: jest.fn().mockResolvedValue(pool), - }; + poolService = { getTxPoolRaw: jest.fn().mockResolvedValue(pool) }; + gatewayService = { getTransactionPoolCount: jest.fn().mockResolvedValue(gatewayCount) }; cachingService = { set: jest.fn() }; // only the dependencies of the pool warming, without the constructor, which also schedules every cron warmer = Object.assign(Object.create(CacheWarmerService.prototype), { poolService, + gatewayService, cachingService, apiConfigService: { getTransactionPoolCacheWarmerTtlInSeconds: () => ttl }, clientProxy: { emit: jest.fn() }, }); }); - it('should warm the pool, its total count and the count for every transaction type, counted from that pool', async () => { + it('should warm the pool, and count the total and every transaction type from that pool', async () => { await warmer.handleTxPoolInvalidations(); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, totalCount, ttl); expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, pool, ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, pool.length, ttl); for (const type of Object.values(TransactionType)) { expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(type).key, typeCounts[type], ttl); } - expect(poolService.getPoolCountRaw).toHaveBeenCalledTimes(1); + expect(gatewayService.getTransactionPoolCount).not.toHaveBeenCalled(); }); - it('should request the total count and the pool together', async () => { - let releaseCount: () => void = () => { }; - poolService.getPoolCountRaw.mockImplementationOnce(async () => await new Promise(resolve => releaseCount = () => resolve(totalCount))); - - const warming = warmer.handleTxPoolInvalidations(); - await new Promise(resolve => setImmediate(resolve)); - - // the pool was requested while the total count is still pending - expect(poolService.getTxPoolRaw).toHaveBeenCalled(); - - releaseCount(); - await warming; - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, totalCount, ttl); - }); - - it('should cache the pool as null, besides the total count, while it is too large', async () => { + it('should cache the pool as null and take the total from the gateway while the pool is too large', async () => { poolService.getTxPoolRaw.mockResolvedValue(null); await warmer.handleTxPoolInvalidations(); expect(cachingService.set).toHaveBeenCalledTimes(2); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, totalCount, ttl); expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, null, ttl); - expect(poolService.getPoolCountRaw).toHaveBeenCalledTimes(1); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, gatewayCount, ttl); + expect(gatewayService.getTransactionPoolCount).toHaveBeenCalledTimes(1); }); }); diff --git a/src/test/unit/services/pool.spec.ts b/src/test/unit/services/pool.spec.ts index c4f83289ec..05039289af 100644 --- a/src/test/unit/services/pool.spec.ts +++ b/src/test/unit/services/pool.spec.ts @@ -99,13 +99,13 @@ describe('PoolService', () => { }); describe('getPoolCount', () => { - it('should return the total counted by the gateway', async () => { - gatewayService.getTransactionPoolCount = jest.fn().mockResolvedValue(42); + it('should count the total from the pool, so that it matches what the pool lists', async () => { + gatewayService.getTransactionPoolCount = jest.fn(); const poolCount = await service.getPoolCount(new PoolFilter()); - expect(poolCount).toStrictEqual(42); + expect(poolCount).toStrictEqual(7); expect(cacheService.getOrSet).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, expect.any(Function), CacheInfo.TransactionPoolCount().ttl); - expect(cacheService.getOrSet).not.toHaveBeenCalledWith(CacheInfo.TransactionPool.key, expect.anything(), expect.anything()); + expect(gatewayService.getTransactionPoolCount).not.toHaveBeenCalled(); }); it('should cache the count for a type apart from the total', async () => { From 5d0aad1d1882f7a208fc48889d6593d8132127ce Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Tue, 29 Sep 2026 12:30:50 +0300 Subject: [PATCH 08/20] send null pool on websocket and count types through gateway when pool is too large --- src/common/gateway/gateway.service.ts | 1 - .../cache.warmer/cache.warmer.service.ts | 24 ++++------- src/crons/websocket/pool.gateway.ts | 10 +---- .../pool/entities/pool.update.status.ts | 5 --- src/endpoints/pool/pool.controller.ts | 3 +- src/endpoints/pool/pool.service.ts | 17 +++----- .../transactions/transaction.service.ts | 3 +- .../unit/services/cache.warmer.pool.spec.ts | 42 +++++++++---------- src/test/unit/services/pool.spec.ts | 30 ++++++++++++- src/test/unit/services/pool.too.large.spec.ts | 27 ++++++------ 10 files changed, 78 insertions(+), 84 deletions(-) delete mode 100644 src/endpoints/pool/entities/pool.update.status.ts diff --git a/src/common/gateway/gateway.service.ts b/src/common/gateway/gateway.service.ts index 85de1034e1..601f64ee10 100644 --- a/src/common/gateway/gateway.service.ts +++ b/src/common/gateway/gateway.service.ts @@ -182,7 +182,6 @@ export class GatewayService { return new NftData(result.tokenData); } - // undefined when the pool is larger than the response size allowed for it in getRaw async getTransactionPool(): Promise { // eslint-disable-next-line require-await return await this.get(`transaction/pool?fields=*`, GatewayComponentRequest.transactionPool, async (error) => { diff --git a/src/crons/cache.warmer/cache.warmer.service.ts b/src/crons/cache.warmer/cache.warmer.service.ts index b9891b97f1..18c34b24b2 100644 --- a/src/crons/cache.warmer/cache.warmer.service.ts +++ b/src/crons/cache.warmer/cache.warmer.service.ts @@ -153,27 +153,17 @@ export class CacheWarmerService { @Lock({ name: 'Transaction pool invalidation', verbose: true }) async handleTxPoolInvalidations() { const ttl = this.apiConfigService.getTransactionPoolCacheWarmerTtlInSeconds(); - - // null when the pool is too large, which is cached as well, so that requests answer that from the cache const pool = await this.poolService.getTxPoolRaw(); + const types = [undefined, ...Object.values(TransactionType)]; - // counted from the pool, so that the total matches what the pool lists. only when the pool is too large to - // be read does the total come from the gateway, which counts the pool itself - const count = pool !== null ? pool.length : await this.gatewayService.getTransactionPoolCount(); + const counts = pool !== null + ? types.map(type => pool.filter(transaction => type === undefined || transaction.type === type).length) + : await Promise.all(types.map(type => this.poolService.getPoolCountFromGateway(type))); - const invalidations = [ + await Promise.all([ this.invalidateKey(CacheInfo.TransactionPool.key, pool, ttl), - this.invalidateKey(CacheInfo.TransactionPoolCount().key, count, ttl), - ]; - - if (pool !== null) { - for (const type of Object.values(TransactionType)) { - const typeCount = pool.filter(transaction => transaction.type === type).length; - invalidations.push(this.invalidateKey(CacheInfo.TransactionPoolCount(type).key, typeCount, ttl)); - } - } - - await Promise.all(invalidations); + ...types.map((type, index) => this.invalidateKey(CacheInfo.TransactionPoolCount(type).key, counts[index], ttl)), + ]); } @Cron('*/2 * * * *') diff --git a/src/crons/websocket/pool.gateway.ts b/src/crons/websocket/pool.gateway.ts index fb9f955dd0..722466a9e2 100644 --- a/src/crons/websocket/pool.gateway.ts +++ b/src/crons/websocket/pool.gateway.ts @@ -10,7 +10,6 @@ import { QueryPagination } from 'src/common/entities/query.pagination'; import { PoolSubscribePayload } from '../../endpoints/pool/entities/pool.subscribe'; import { RoomKeyGenerator } from './room.key.generator'; import { LockingGuardInterceptor } from 'src/utils/locking.guard.interceptor'; -import { PoolUpdateStatus } from '../../endpoints/pool/entities/pool.update.status'; @UseFilters(WebsocketExceptionsFilter) @WebSocketGateway({ cors: { origin: '*' }, path: '/ws/subscription' }) @@ -75,14 +74,7 @@ export class PoolGateway { this.poolService.getPoolCount(poolFilter), ]); - // the pool is null when it is too large to be read. the total count comes from the gateway, so it is - // still sent then, unless the room filters by type - if (pool === null) { - this.server.to(roomName).emit("poolUpdate", { status: PoolUpdateStatus.tooLarge, pool: [], poolCount }); - return; - } - - this.server.to(roomName).emit("poolUpdate", { status: PoolUpdateStatus.success, pool, poolCount }); + this.server.to(roomName).emit("poolUpdate", { pool, poolCount }); } catch (error) { this.logger.error(error); } diff --git a/src/endpoints/pool/entities/pool.update.status.ts b/src/endpoints/pool/entities/pool.update.status.ts deleted file mode 100644 index 6c8acb3fcc..0000000000 --- a/src/endpoints/pool/entities/pool.update.status.ts +++ /dev/null @@ -1,5 +0,0 @@ -// sent with every poolUpdate, so that clients know whether the pool it carries can be shown -export enum PoolUpdateStatus { - success = 'success', - tooLarge = 'tooLarge', -} diff --git a/src/endpoints/pool/pool.controller.ts b/src/endpoints/pool/pool.controller.ts index 3e4547e9ef..24a686ffa7 100644 --- a/src/endpoints/pool/pool.controller.ts +++ b/src/endpoints/pool/pool.controller.ts @@ -51,7 +51,6 @@ export class PoolController { functions: functions, })); - // null when the pool is too large to be read return pool ?? new TransactionPoolTooLarge(); } @@ -81,7 +80,6 @@ export class PoolController { type: type, })); - // null when the count needs the pool and the pool is too large to be read return count ?? new TransactionPoolTooLarge(); } @@ -93,6 +91,7 @@ export class PoolController { @Query('type', new ParseEnumPipe(TransactionType)) type?: TransactionType, ): Promise { const count = await this.poolService.getPoolCount(new PoolFilter({ sender, receiver, type })); + return count ?? new TransactionPoolTooLarge(); } diff --git a/src/endpoints/pool/pool.service.ts b/src/endpoints/pool/pool.service.ts index 74602e279f..a987f58096 100644 --- a/src/endpoints/pool/pool.service.ts +++ b/src/endpoints/pool/pool.service.ts @@ -22,7 +22,6 @@ export class PoolService { private readonly transactionActionService: TransactionActionService, ) { } - // undefined when the transaction is not in the pool, null when the pool is too large to be read async getTransactionFromPool(txHash: string): Promise { const pool = await this.getPoolWithFilters(); if (pool === null) { @@ -32,7 +31,6 @@ export class PoolService { return pool.find(tx => tx.txHash === txHash); } - // null when the count needs the pool and the pool is too large to be read async getPoolCount(filter: PoolFilter): Promise { const { type, ...otherFilters } = filter; if (Object.values(otherFilters).some(value => value !== undefined)) { @@ -40,7 +38,6 @@ export class PoolService { return pool === null ? null : pool.length; } - // the total and the count for each type are cached on their own return await this.cacheService.getOrSet( CacheInfo.TransactionPoolCount(type).key, async () => await this.getPoolCountRaw(type), @@ -48,22 +45,23 @@ export class PoolService { ); } - // counted from the pool, so that the count matches what the pool lists. only when the pool is too large to - // be read does the total come from the gateway, which counts the pool itself; counts by type are then null async getPoolCountRaw(type?: TransactionType): Promise { const pool = await this.getPoolWithFilters(new PoolFilter({ type })); if (pool !== null) { return pool.length; } - if (type !== undefined) { + return await this.getPoolCountFromGateway(type); + } + + async getPoolCountFromGateway(type?: TransactionType): Promise { + if (type !== undefined && type !== TransactionType.Transaction) { return null; } return await this.gatewayService.getTransactionPoolCount(); } - // null when the pool is too large to be read async getPool( queryPagination: QueryPagination, filter?: PoolFilter, @@ -81,7 +79,6 @@ export class PoolService { return pool.slice(from, from + size); } - // null when the pool is too large to be read async getPoolWithFilters( filter?: PoolFilter, ): Promise { @@ -90,7 +87,7 @@ export class PoolService { async () => await this.getTxPoolRaw(), CacheInfo.TransactionPool.ttl, CacheInfo.TransactionPool.ttl, - true, // cacheNullable: null stands for a pool too large to be read, see getTxPoolRaw + true, ); if (pool === null) { @@ -100,8 +97,6 @@ export class PoolService { return this.applyFilters(pool, filter); } - // null only when the pool is too large to be read. it is cached like the pool itself, so that until it - // expires the pool is not downloaded again, up to the size limit, on every request async getTxPoolRaw(): Promise { const pool = await this.gatewayService.getTransactionPool(); if (!pool) { diff --git a/src/endpoints/transactions/transaction.service.ts b/src/endpoints/transactions/transaction.service.ts index 81c72ae250..495e776548 100644 --- a/src/endpoints/transactions/transaction.service.ts +++ b/src/endpoints/transactions/transaction.service.ts @@ -752,12 +752,11 @@ export class TransactionService { const poolTransactions = await this.poolService.getPoolWithFilters({ senderShard: shardId }); - // null when the pool is too large to be read, which leaves the prices unknown rather than at zero if (poolTransactions === null) { return null; } - if (!poolTransactions || poolTransactions.length === 0) { + if (poolTransactions.length === 0) { return new PpuMetadata({ lastBlock: lastBlock, fast: 0, diff --git a/src/test/unit/services/cache.warmer.pool.spec.ts b/src/test/unit/services/cache.warmer.pool.spec.ts index 7b6125ef76..2e8935c84b 100644 --- a/src/test/unit/services/cache.warmer.pool.spec.ts +++ b/src/test/unit/services/cache.warmer.pool.spec.ts @@ -5,63 +5,63 @@ import { CacheInfo } from "src/utils/cache.info"; describe('CacheWarmerService transaction pool', () => { const ttl = 10; - const gatewayCount = 12; const pool = [ { txHash: 'a', type: TransactionType.Transaction }, { txHash: 'b', type: TransactionType.Reward }, { txHash: 'c', type: TransactionType.Reward }, ]; - const typeCounts: Record = { - [TransactionType.Transaction]: 1, - [TransactionType.SmartContractResult]: 0, - [TransactionType.Reward]: 2, + const gatewayCounts: Record = { + total: 12, + [TransactionType.Transaction]: 12, + [TransactionType.SmartContractResult]: null, + [TransactionType.Reward]: null, }; let warmer: CacheWarmerService; let poolService: any; - let gatewayService: any; let cachingService: any; beforeEach(() => { - // run the handler without the lock around it, which records metrics and swallows the errors jest.spyOn(Locker, 'lock').mockImplementation(async (_key: string, func: () => Promise) => { await func(); return LockResult.SUCCESS; }); - poolService = { getTxPoolRaw: jest.fn().mockResolvedValue(pool) }; - gatewayService = { getTransactionPoolCount: jest.fn().mockResolvedValue(gatewayCount) }; + poolService = { + getTxPoolRaw: jest.fn().mockResolvedValue(pool), + getPoolCountFromGateway: jest.fn().mockImplementation(async (type?: TransactionType) => await Promise.resolve(gatewayCounts[type ?? 'total'])), + }; cachingService = { set: jest.fn() }; - // only the dependencies of the pool warming, without the constructor, which also schedules every cron warmer = Object.assign(Object.create(CacheWarmerService.prototype), { poolService, - gatewayService, cachingService, apiConfigService: { getTransactionPoolCacheWarmerTtlInSeconds: () => ttl }, clientProxy: { emit: jest.fn() }, }); }); - it('should warm the pool, and count the total and every transaction type from that pool', async () => { + it('should warm the pool and count the total and every type from it', async () => { await warmer.handleTxPoolInvalidations(); expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, pool, ttl); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, pool.length, ttl); - for (const type of Object.values(TransactionType)) { - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(type).key, typeCounts[type], ttl); - } - expect(gatewayService.getTransactionPoolCount).not.toHaveBeenCalled(); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, 3, ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(TransactionType.Transaction).key, 1, ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(TransactionType.SmartContractResult).key, 0, ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(TransactionType.Reward).key, 2, ttl); + expect(poolService.getPoolCountFromGateway).not.toHaveBeenCalled(); }); - it('should cache the pool as null and take the total from the gateway while the pool is too large', async () => { + it('should warm the pool as null and count the total and every type through the gateway when it is too large', async () => { poolService.getTxPoolRaw.mockResolvedValue(null); await warmer.handleTxPoolInvalidations(); - expect(cachingService.set).toHaveBeenCalledTimes(2); expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, null, ttl); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, gatewayCount, ttl); - expect(gatewayService.getTransactionPoolCount).toHaveBeenCalledTimes(1); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, gatewayCounts.total, ttl); + for (const type of Object.values(TransactionType)) { + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(type).key, gatewayCounts[type], ttl); + } + expect(poolService.getPoolCountFromGateway).toHaveBeenCalledTimes(Object.values(TransactionType).length + 1); }); }); diff --git a/src/test/unit/services/pool.spec.ts b/src/test/unit/services/pool.spec.ts index 05039289af..8a09b03513 100644 --- a/src/test/unit/services/pool.spec.ts +++ b/src/test/unit/services/pool.spec.ts @@ -62,7 +62,6 @@ describe('PoolService', () => { gatewayService.getTransactionPool = jest.fn().mockResolvedValue(data); const txPoolRaw = await service.getTxPoolRaw(); - // the pool is served from the cache, every other value is computed cacheService.getOrSet = jest.fn().mockImplementation(async (key: string, createValueFunc: () => Promise) => { return key === CacheInfo.TransactionPool.key ? txPoolRaw : await createValueFunc(); }); @@ -170,6 +169,35 @@ describe('PoolService', () => { }); }); + describe('getPoolCountFromGateway', () => { + it('should count the total and the transactions through the gateway', async () => { + gatewayService.getTransactionPoolCount = jest.fn().mockResolvedValue(42); + + expect(await service.getPoolCountFromGateway()).toStrictEqual(42); + expect(await service.getPoolCountFromGateway(TransactionType.Transaction)).toStrictEqual(42); + }); + + it('should not count the types the gateway does not count', async () => { + gatewayService.getTransactionPoolCount = jest.fn(); + + expect(await service.getPoolCountFromGateway(TransactionType.SmartContractResult)).toBeNull(); + expect(await service.getPoolCountFromGateway(TransactionType.Reward)).toBeNull(); + expect(gatewayService.getTransactionPoolCount).not.toHaveBeenCalled(); + }); + + it('should count a type through the gateway only while the pool is too large', async () => { + gatewayService.getTransactionPoolCount = jest.fn().mockResolvedValue(42); + + expect(await service.getPoolCount(new PoolFilter({ type: TransactionType.Transaction }))).toStrictEqual(1); + + cacheService.getOrSet = jest.fn().mockImplementation(async (key: string, createValueFunc: () => Promise) => { + return key === CacheInfo.TransactionPool.key ? null : await createValueFunc(); + }); + + expect(await service.getPoolCount(new PoolFilter({ type: TransactionType.Transaction }))).toStrictEqual(42); + }); + }); + describe('getTransactionFromPool', () => { it('should work and return the transaction', async () => { const tx = await service.getTransactionFromPool("e07af9835b6da5740d0f791cfe65491a562852c57d44af63fdc14be5d73f01da"); diff --git a/src/test/unit/services/pool.too.large.spec.ts b/src/test/unit/services/pool.too.large.spec.ts index a919439fbf..7981ac1912 100644 --- a/src/test/unit/services/pool.too.large.spec.ts +++ b/src/test/unit/services/pool.too.large.spec.ts @@ -2,7 +2,6 @@ import { NotFoundException } from "@nestjs/common"; import { PoolGateway } from "src/crons/websocket/pool.gateway"; import { PoolFilter } from "src/endpoints/pool/entities/pool.filter"; import { TransactionPoolTooLarge } from "src/endpoints/pool/entities/transaction.pool.too.large"; -import { PoolUpdateStatus } from "src/endpoints/pool/entities/pool.update.status"; import { PoolController } from "src/endpoints/pool/pool.controller"; import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; import { TransactionService } from "src/endpoints/transactions/transaction.service"; @@ -15,11 +14,9 @@ describe('Transaction pool too large', () => { beforeEach(() => { poolService = { - // null stands for a pool too large to be read getPool: jest.fn().mockResolvedValue(null), getTransactionFromPool: jest.fn().mockResolvedValue(null), - // the total comes from the gateway, only the counts that need the pool are null - getPoolCount: jest.fn().mockImplementation(async (filter: PoolFilter) => await Promise.resolve(filter.type ? null : 42)), + getPoolCount: jest.fn().mockImplementation(async (filter: PoolFilter) => await Promise.resolve(filter.type === TransactionType.Reward ? null : 42)), }; }); @@ -30,15 +27,16 @@ describe('Transaction pool too large', () => { controller = new PoolController(poolService); }); - it('should answer the pool, the transaction and the counts that need the pool with a custom response', async () => { + it('should answer the pool and the transaction with a custom response', async () => { expect(await controller.getTransactionPool(0, 25)).toStrictEqual(new TransactionPoolTooLarge()); expect(await controller.getTransactionPool(0, 25)).toEqual(tooLarge); expect(await controller.getTransactionFromPool(txHash)).toEqual(tooLarge); - expect(await controller.getTransactionPoolCount(undefined, undefined, undefined, undefined, TransactionType.Reward)).toEqual(tooLarge); }); - it('should still answer the total count', async () => { + it('should answer the counts the gateway gives and a custom response for the others', async () => { expect(await controller.getTransactionPoolCount()).toStrictEqual(42); + expect(await controller.getTransactionPoolCount(undefined, undefined, undefined, undefined, TransactionType.Transaction)).toStrictEqual(42); + expect(await controller.getTransactionPoolCount(undefined, undefined, undefined, undefined, TransactionType.Reward)).toEqual(tooLarge); }); it('should keep answering other failures as errors', async () => { @@ -51,8 +49,7 @@ describe('Transaction pool too large', () => { }); describe('TransactionService price per unit', () => { - it('should leave the prices unknown, rather than at zero, while the pool is too large', async () => { - // only the dependencies of the price per unit, without the constructor + it('should leave the prices unknown while the pool is too large', async () => { const transactionService: TransactionService = Object.assign(Object.create(TransactionService.prototype), { blockService: { getBlocks: jest.fn().mockResolvedValue([{ nonce: 100 }]) }, networkService: { getConstants: jest.fn().mockResolvedValue({ minGasLimit: 50000, gasPerDataByte: 1500, gasPriceModifier: '0.01' }) }, @@ -73,24 +70,24 @@ describe('Transaction pool too large', () => { gateway.server = { to: jest.fn().mockReturnValue({ emit }) } as any; }); - it('should send the room an empty pool with the too large status, and the total count', async () => { + it('should send a null pool with the total count', async () => { await gateway.pushPoolForRoom('pool-{"from":0,"size":25}'); - expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.tooLarge, pool: [], poolCount: 42 }); + expect(emit).toHaveBeenCalledWith('poolUpdate', { pool: null, poolCount: 42 }); }); - it('should send no count to a room that filters by type, as that count needs the pool', async () => { + it('should send a null pool with no count to a room of a type the gateway does not count', async () => { await gateway.pushPoolForRoom(`pool-{"from":0,"size":25,"type":"${TransactionType.Reward}"}`); - expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.tooLarge, pool: [], poolCount: null }); + expect(emit).toHaveBeenCalledWith('poolUpdate', { pool: null, poolCount: null }); }); - it('should send the pool with the success status when it can be read', async () => { + it('should send the pool when it can be read', async () => { poolService.getPool.mockResolvedValue([{ txHash }]); await gateway.pushPoolForRoom('pool-{"from":0,"size":25}'); - expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.success, pool: [{ txHash }], poolCount: 42 }); + expect(emit).toHaveBeenCalledWith('poolUpdate', { pool: [{ txHash }], poolCount: 42 }); }); }); }); From ec134c407964b931058b7462fa97d40e00ad3b2e Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Tue, 29 Sep 2026 12:44:29 +0300 Subject: [PATCH 09/20] pass type to gateway pool count --- src/common/gateway/gateway.service.ts | 6 ++- .../cache.warmer/cache.warmer.service.ts | 2 +- src/endpoints/pool/pool.service.ts | 12 +----- .../unit/services/cache.warmer.pool.spec.ts | 20 +++++----- .../unit/services/gateway.pool.count.spec.ts | 29 +++++++++++++++ src/test/unit/services/pool.spec.ts | 37 +++---------------- src/test/unit/services/pool.too.large.spec.ts | 11 +++--- 7 files changed, 58 insertions(+), 59 deletions(-) create mode 100644 src/test/unit/services/gateway.pool.count.spec.ts diff --git a/src/common/gateway/gateway.service.ts b/src/common/gateway/gateway.service.ts index 601f64ee10..7b66e14f3d 100644 --- a/src/common/gateway/gateway.service.ts +++ b/src/common/gateway/gateway.service.ts @@ -20,6 +20,7 @@ import { ApiService, ApiSettings } from "@multiversx/sdk-nestjs-http"; import { GuardianResult } from "./entities/guardian.result"; import { TransactionProcessStatus } from "./entities/transaction.process.status"; import { TxPoolGatewayResponse } from "./entities/tx.pool.gateway.response"; +import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; @Injectable() export class GatewayService { @@ -189,8 +190,9 @@ export class GatewayService { }); } - async getTransactionPoolCount(): Promise { - const result = await this.get('transaction/pool/count', GatewayComponentRequest.transactionPool); + async getTransactionPoolCount(type?: TransactionType): Promise { + const url = type ? `transaction/pool/count?type=${type}` : 'transaction/pool/count'; + const result = await this.get(url, GatewayComponentRequest.transactionPool); return Object.values(result.txPoolCounts).reduce((total, count) => total + count, 0); } diff --git a/src/crons/cache.warmer/cache.warmer.service.ts b/src/crons/cache.warmer/cache.warmer.service.ts index 18c34b24b2..a511438eb1 100644 --- a/src/crons/cache.warmer/cache.warmer.service.ts +++ b/src/crons/cache.warmer/cache.warmer.service.ts @@ -158,7 +158,7 @@ export class CacheWarmerService { const counts = pool !== null ? types.map(type => pool.filter(transaction => type === undefined || transaction.type === type).length) - : await Promise.all(types.map(type => this.poolService.getPoolCountFromGateway(type))); + : await Promise.all(types.map(type => this.gatewayService.getTransactionPoolCount(type))); await Promise.all([ this.invalidateKey(CacheInfo.TransactionPool.key, pool, ttl), diff --git a/src/endpoints/pool/pool.service.ts b/src/endpoints/pool/pool.service.ts index a987f58096..ee7ad34462 100644 --- a/src/endpoints/pool/pool.service.ts +++ b/src/endpoints/pool/pool.service.ts @@ -45,21 +45,13 @@ export class PoolService { ); } - async getPoolCountRaw(type?: TransactionType): Promise { + async getPoolCountRaw(type?: TransactionType): Promise { const pool = await this.getPoolWithFilters(new PoolFilter({ type })); if (pool !== null) { return pool.length; } - return await this.getPoolCountFromGateway(type); - } - - async getPoolCountFromGateway(type?: TransactionType): Promise { - if (type !== undefined && type !== TransactionType.Transaction) { - return null; - } - - return await this.gatewayService.getTransactionPoolCount(); + return await this.gatewayService.getTransactionPoolCount(type); } async getPool( diff --git a/src/test/unit/services/cache.warmer.pool.spec.ts b/src/test/unit/services/cache.warmer.pool.spec.ts index 2e8935c84b..1029879a6c 100644 --- a/src/test/unit/services/cache.warmer.pool.spec.ts +++ b/src/test/unit/services/cache.warmer.pool.spec.ts @@ -10,15 +10,16 @@ describe('CacheWarmerService transaction pool', () => { { txHash: 'b', type: TransactionType.Reward }, { txHash: 'c', type: TransactionType.Reward }, ]; - const gatewayCounts: Record = { + const gatewayCounts: Record = { total: 12, - [TransactionType.Transaction]: 12, - [TransactionType.SmartContractResult]: null, - [TransactionType.Reward]: null, + [TransactionType.Transaction]: 7, + [TransactionType.SmartContractResult]: 3, + [TransactionType.Reward]: 2, }; let warmer: CacheWarmerService; let poolService: any; + let gatewayService: any; let cachingService: any; beforeEach(() => { @@ -27,14 +28,15 @@ describe('CacheWarmerService transaction pool', () => { return LockResult.SUCCESS; }); - poolService = { - getTxPoolRaw: jest.fn().mockResolvedValue(pool), - getPoolCountFromGateway: jest.fn().mockImplementation(async (type?: TransactionType) => await Promise.resolve(gatewayCounts[type ?? 'total'])), + poolService = { getTxPoolRaw: jest.fn().mockResolvedValue(pool) }; + gatewayService = { + getTransactionPoolCount: jest.fn().mockImplementation(async (type?: TransactionType) => await Promise.resolve(gatewayCounts[type ?? 'total'])), }; cachingService = { set: jest.fn() }; warmer = Object.assign(Object.create(CacheWarmerService.prototype), { poolService, + gatewayService, cachingService, apiConfigService: { getTransactionPoolCacheWarmerTtlInSeconds: () => ttl }, clientProxy: { emit: jest.fn() }, @@ -49,7 +51,7 @@ describe('CacheWarmerService transaction pool', () => { expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(TransactionType.Transaction).key, 1, ttl); expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(TransactionType.SmartContractResult).key, 0, ttl); expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(TransactionType.Reward).key, 2, ttl); - expect(poolService.getPoolCountFromGateway).not.toHaveBeenCalled(); + expect(gatewayService.getTransactionPoolCount).not.toHaveBeenCalled(); }); it('should warm the pool as null and count the total and every type through the gateway when it is too large', async () => { @@ -62,6 +64,6 @@ describe('CacheWarmerService transaction pool', () => { for (const type of Object.values(TransactionType)) { expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(type).key, gatewayCounts[type], ttl); } - expect(poolService.getPoolCountFromGateway).toHaveBeenCalledTimes(Object.values(TransactionType).length + 1); + expect(gatewayService.getTransactionPoolCount).toHaveBeenCalledTimes(Object.values(TransactionType).length + 1); }); }); diff --git a/src/test/unit/services/gateway.pool.count.spec.ts b/src/test/unit/services/gateway.pool.count.spec.ts new file mode 100644 index 0000000000..358379f059 --- /dev/null +++ b/src/test/unit/services/gateway.pool.count.spec.ts @@ -0,0 +1,29 @@ +import { GatewayService } from "src/common/gateway/gateway.service"; +import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; + +describe('GatewayService transaction pool count', () => { + let gatewayService: GatewayService; + let apiService: any; + + beforeEach(() => { + apiService = { + get: jest.fn().mockResolvedValue({ data: { data: { txPoolCounts: { '0': 1, '1': 2, '2': 3, '4294967295': 4 } } } }), + }; + + gatewayService = new GatewayService( + { getGatewayUrl: () => 'https://gateway', getSnapshotlessGatewayUrl: () => undefined } as any, + apiService, + ); + Object.assign(gatewayService, { eventEmitter: { emit: jest.fn() } }); + }); + + it('should sum the counts of every shard', async () => { + expect(await gatewayService.getTransactionPoolCount()).toStrictEqual(10); + expect(apiService.get).toHaveBeenCalledWith('https://gateway/transaction/pool/count', expect.anything(), undefined); + }); + + it('should ask for the count of a type', async () => { + expect(await gatewayService.getTransactionPoolCount(TransactionType.SmartContractResult)).toStrictEqual(10); + expect(apiService.get).toHaveBeenCalledWith('https://gateway/transaction/pool/count?type=SmartContractResult', expect.anything(), undefined); + }); +}); diff --git a/src/test/unit/services/pool.spec.ts b/src/test/unit/services/pool.spec.ts index 8a09b03513..dee6393f86 100644 --- a/src/test/unit/services/pool.spec.ts +++ b/src/test/unit/services/pool.spec.ts @@ -154,47 +154,20 @@ describe('PoolService', () => { expect(await service.getPool(new QueryPagination(), new PoolFilter())).toBeNull(); expect(await service.getTransactionFromPool('e07af9835b6da5740d0f791cfe65491a562852c57d44af63fdc14be5d73f01da')).toBeNull(); - expect(await service.getPoolCount(new PoolFilter({ type: TransactionType.Reward }))).toBeNull(); expect(await service.getPoolCount(new PoolFilter({ sender: 'erd1qqqqqqqqqqqqqpgqp699jngundfqw07d8jzkepucvpzush6k3wvqyc44rx' }))).toBeNull(); expect(gatewayService.getTransactionPool).not.toHaveBeenCalled(); }); - it('should still count the total through the gateway', async () => { - gatewayService.getTransactionPoolCount = jest.fn().mockResolvedValue(42); + it('should count the total and every type through the gateway', async () => { + gatewayService.getTransactionPoolCount = jest.fn().mockImplementation(async (type?: TransactionType) => await Promise.resolve(type ? 10 : 42)); cacheService.getOrSet = jest.fn().mockImplementation(async (key: string, createValueFunc: () => Promise) => { return key === CacheInfo.TransactionPool.key ? null : await createValueFunc(); }); expect(await service.getPoolCount(new PoolFilter())).toStrictEqual(42); - }); - }); - - describe('getPoolCountFromGateway', () => { - it('should count the total and the transactions through the gateway', async () => { - gatewayService.getTransactionPoolCount = jest.fn().mockResolvedValue(42); - - expect(await service.getPoolCountFromGateway()).toStrictEqual(42); - expect(await service.getPoolCountFromGateway(TransactionType.Transaction)).toStrictEqual(42); - }); - - it('should not count the types the gateway does not count', async () => { - gatewayService.getTransactionPoolCount = jest.fn(); - - expect(await service.getPoolCountFromGateway(TransactionType.SmartContractResult)).toBeNull(); - expect(await service.getPoolCountFromGateway(TransactionType.Reward)).toBeNull(); - expect(gatewayService.getTransactionPoolCount).not.toHaveBeenCalled(); - }); - - it('should count a type through the gateway only while the pool is too large', async () => { - gatewayService.getTransactionPoolCount = jest.fn().mockResolvedValue(42); - - expect(await service.getPoolCount(new PoolFilter({ type: TransactionType.Transaction }))).toStrictEqual(1); - - cacheService.getOrSet = jest.fn().mockImplementation(async (key: string, createValueFunc: () => Promise) => { - return key === CacheInfo.TransactionPool.key ? null : await createValueFunc(); - }); - - expect(await service.getPoolCount(new PoolFilter({ type: TransactionType.Transaction }))).toStrictEqual(42); + expect(await service.getPoolCount(new PoolFilter({ type: TransactionType.Reward }))).toStrictEqual(10); + expect(gatewayService.getTransactionPoolCount).toHaveBeenCalledWith(undefined); + expect(gatewayService.getTransactionPoolCount).toHaveBeenCalledWith(TransactionType.Reward); }); }); diff --git a/src/test/unit/services/pool.too.large.spec.ts b/src/test/unit/services/pool.too.large.spec.ts index 7981ac1912..cb26e26f2e 100644 --- a/src/test/unit/services/pool.too.large.spec.ts +++ b/src/test/unit/services/pool.too.large.spec.ts @@ -16,7 +16,7 @@ describe('Transaction pool too large', () => { poolService = { getPool: jest.fn().mockResolvedValue(null), getTransactionFromPool: jest.fn().mockResolvedValue(null), - getPoolCount: jest.fn().mockImplementation(async (filter: PoolFilter) => await Promise.resolve(filter.type === TransactionType.Reward ? null : 42)), + getPoolCount: jest.fn().mockImplementation(async (filter: PoolFilter) => await Promise.resolve(filter.sender ? null : 42)), }; }); @@ -35,8 +35,8 @@ describe('Transaction pool too large', () => { it('should answer the counts the gateway gives and a custom response for the others', async () => { expect(await controller.getTransactionPoolCount()).toStrictEqual(42); - expect(await controller.getTransactionPoolCount(undefined, undefined, undefined, undefined, TransactionType.Transaction)).toStrictEqual(42); - expect(await controller.getTransactionPoolCount(undefined, undefined, undefined, undefined, TransactionType.Reward)).toEqual(tooLarge); + expect(await controller.getTransactionPoolCount(undefined, undefined, undefined, undefined, TransactionType.Reward)).toStrictEqual(42); + expect(await controller.getTransactionPoolCount('erd1qqqqqqqqqqqqqpgqp699jngundfqw07d8jzkepucvpzush6k3wvqyc44rx')).toEqual(tooLarge); }); it('should keep answering other failures as errors', async () => { @@ -76,10 +76,11 @@ describe('Transaction pool too large', () => { expect(emit).toHaveBeenCalledWith('poolUpdate', { pool: null, poolCount: 42 }); }); - it('should send a null pool with no count to a room of a type the gateway does not count', async () => { + it('should send a null pool with the count of the type of the room', async () => { await gateway.pushPoolForRoom(`pool-{"from":0,"size":25,"type":"${TransactionType.Reward}"}`); - expect(emit).toHaveBeenCalledWith('poolUpdate', { pool: null, poolCount: null }); + expect(poolService.getPoolCount).toHaveBeenCalledWith(new PoolFilter({ type: TransactionType.Reward })); + expect(emit).toHaveBeenCalledWith('poolUpdate', { pool: null, poolCount: 42 }); }); it('should send the pool when it can be read', async () => { From 59e6fc64760dabc8c16189016152fabfb4811262 Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Tue, 29 Sep 2026 12:55:30 +0300 Subject: [PATCH 10/20] back to 503 for pool too large, status on pool websocket and cache info ttls in warmer --- .../cache.warmer/cache.warmer.service.ts | 5 +- src/crons/websocket/pool.gateway.ts | 4 +- .../pool/entities/pool.update.status.ts | 4 ++ .../transaction.pool.too.large.exception.ts | 13 +++++ .../entities/transaction.pool.too.large.ts | 9 ---- src/endpoints/pool/pool.controller.ts | 50 ++++++++++--------- .../unit/services/cache.warmer.pool.spec.ts | 18 +++---- src/test/unit/services/pool.too.large.spec.ts | 28 +++++++---- 8 files changed, 74 insertions(+), 57 deletions(-) create mode 100644 src/endpoints/pool/entities/pool.update.status.ts create mode 100644 src/endpoints/pool/entities/transaction.pool.too.large.exception.ts delete mode 100644 src/endpoints/pool/entities/transaction.pool.too.large.ts diff --git a/src/crons/cache.warmer/cache.warmer.service.ts b/src/crons/cache.warmer/cache.warmer.service.ts index a511438eb1..ceb56334f9 100644 --- a/src/crons/cache.warmer/cache.warmer.service.ts +++ b/src/crons/cache.warmer/cache.warmer.service.ts @@ -152,7 +152,6 @@ export class CacheWarmerService { @Lock({ name: 'Transaction pool invalidation', verbose: true }) async handleTxPoolInvalidations() { - const ttl = this.apiConfigService.getTransactionPoolCacheWarmerTtlInSeconds(); const pool = await this.poolService.getTxPoolRaw(); const types = [undefined, ...Object.values(TransactionType)]; @@ -161,8 +160,8 @@ export class CacheWarmerService { : await Promise.all(types.map(type => this.gatewayService.getTransactionPoolCount(type))); await Promise.all([ - this.invalidateKey(CacheInfo.TransactionPool.key, pool, ttl), - ...types.map((type, index) => this.invalidateKey(CacheInfo.TransactionPoolCount(type).key, counts[index], ttl)), + this.invalidateKey(CacheInfo.TransactionPool.key, pool, CacheInfo.TransactionPool.ttl), + ...types.map((type, index) => this.invalidateKey(CacheInfo.TransactionPoolCount(type).key, counts[index], CacheInfo.TransactionPoolCount(type).ttl)), ]); } diff --git a/src/crons/websocket/pool.gateway.ts b/src/crons/websocket/pool.gateway.ts index 722466a9e2..2be7e38510 100644 --- a/src/crons/websocket/pool.gateway.ts +++ b/src/crons/websocket/pool.gateway.ts @@ -10,6 +10,7 @@ import { QueryPagination } from 'src/common/entities/query.pagination'; import { PoolSubscribePayload } from '../../endpoints/pool/entities/pool.subscribe'; import { RoomKeyGenerator } from './room.key.generator'; import { LockingGuardInterceptor } from 'src/utils/locking.guard.interceptor'; +import { PoolUpdateStatus } from '../../endpoints/pool/entities/pool.update.status'; @UseFilters(WebsocketExceptionsFilter) @WebSocketGateway({ cors: { origin: '*' }, path: '/ws/subscription' }) @@ -74,7 +75,8 @@ export class PoolGateway { this.poolService.getPoolCount(poolFilter), ]); - this.server.to(roomName).emit("poolUpdate", { pool, poolCount }); + const status = pool === null ? PoolUpdateStatus.tooLarge : PoolUpdateStatus.success; + this.server.to(roomName).emit("poolUpdate", { status, pool, poolCount }); } catch (error) { this.logger.error(error); } diff --git a/src/endpoints/pool/entities/pool.update.status.ts b/src/endpoints/pool/entities/pool.update.status.ts new file mode 100644 index 0000000000..c33d828402 --- /dev/null +++ b/src/endpoints/pool/entities/pool.update.status.ts @@ -0,0 +1,4 @@ +export enum PoolUpdateStatus { + success = 'success', + tooLarge = 'tooLarge', +} diff --git a/src/endpoints/pool/entities/transaction.pool.too.large.exception.ts b/src/endpoints/pool/entities/transaction.pool.too.large.exception.ts new file mode 100644 index 0000000000..67342075ac --- /dev/null +++ b/src/endpoints/pool/entities/transaction.pool.too.large.exception.ts @@ -0,0 +1,13 @@ +import { HttpException, HttpStatus } from "@nestjs/common"; + +export class TransactionPoolTooLargeException extends HttpException { + static readonly code = 'transaction_pool_too_large'; + + constructor() { + super({ + statusCode: HttpStatus.SERVICE_UNAVAILABLE, + code: TransactionPoolTooLargeException.code, + message: 'The transaction pool is too large to be displayed', + }, HttpStatus.SERVICE_UNAVAILABLE); + } +} diff --git a/src/endpoints/pool/entities/transaction.pool.too.large.ts b/src/endpoints/pool/entities/transaction.pool.too.large.ts deleted file mode 100644 index 290be6ac67..0000000000 --- a/src/endpoints/pool/entities/transaction.pool.too.large.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { ApiProperty } from "@nestjs/swagger"; - -export class TransactionPoolTooLarge { - @ApiProperty({ type: Boolean, example: true, description: 'Set only when the transaction pool is too large to be displayed' }) - tooLarge: boolean = true; - - @ApiProperty({ type: String, example: 'The transaction pool is too large to be displayed' }) - message: string = 'The transaction pool is too large to be displayed'; -} diff --git a/src/endpoints/pool/pool.controller.ts b/src/endpoints/pool/pool.controller.ts index 24a686ffa7..099feeb5b1 100644 --- a/src/endpoints/pool/pool.controller.ts +++ b/src/endpoints/pool/pool.controller.ts @@ -1,17 +1,16 @@ import { ParseAddressAndMetachainPipe, ParseAddressPipe, ParseEnumPipe, ParseIntPipe, ParseTransactionHashPipe, ParseArrayPipe } from "@multiversx/sdk-nestjs-common"; import { Controller, DefaultValuePipe, Get, NotFoundException, Param, Query } from "@nestjs/common"; -import { ApiExcludeEndpoint, ApiExtraModels, ApiNotFoundResponse, ApiOkResponse, ApiOperation, ApiQuery, ApiTags, getSchemaPath } from "@nestjs/swagger"; +import { ApiExcludeEndpoint, ApiNotFoundResponse, ApiOkResponse, ApiOperation, ApiQuery, ApiServiceUnavailableResponse, ApiTags } from "@nestjs/swagger"; import { PoolService } from "./pool.service"; import { QueryPagination } from "src/common/entities/query.pagination"; import { TransactionInPool } from "./entities/transaction.in.pool.dto"; import { TransactionType } from "../transactions/entities/transaction.type"; import { PoolFilter } from "./entities/pool.filter"; import { ParseArrayPipeOptions } from "@multiversx/sdk-nestjs-common/lib/pipes/entities/parse.array.options"; -import { TransactionPoolTooLarge } from "./entities/transaction.pool.too.large"; +import { TransactionPoolTooLargeException } from "./entities/transaction.pool.too.large.exception"; @Controller() @ApiTags('pool') -@ApiExtraModels(TransactionInPool, TransactionPoolTooLarge) export class PoolController { constructor( private readonly poolService: PoolService, @@ -19,10 +18,8 @@ export class PoolController { @Get("/pool") @ApiOperation({ summary: 'Transactions pool', description: 'Returns the transactions that are currently in the memory pool.' }) - @ApiOkResponse({ - description: 'The transactions, or TransactionPoolTooLarge when the pool is too large to be displayed', - schema: { oneOf: [{ type: 'array', items: { $ref: getSchemaPath(TransactionInPool) } }, { $ref: getSchemaPath(TransactionPoolTooLarge) }] }, - }) + @ApiOkResponse({ type: TransactionInPool, isArray: true }) + @ApiServiceUnavailableResponse({ description: 'The transaction pool is too large to be displayed' }) @ApiQuery({ name: 'from', description: 'Number of items to skip for the result set', required: false }) @ApiQuery({ name: 'size', description: 'Number of items to retrieve', required: false }) @ApiQuery({ name: 'sender', description: 'Search in transaction pool by a specific sender', required: false }) @@ -41,7 +38,7 @@ export class PoolController { @Query('receiverShard', ParseIntPipe) receiverShard?: number, @Query('type', new ParseEnumPipe(TransactionType)) type?: TransactionType, @Query('function', new ParseArrayPipe(new ParseArrayPipeOptions({ allowEmptyString: true }))) functions?: string[], - ): Promise { + ): Promise { const pool = await this.poolService.getPool(new QueryPagination({ from, size }), new PoolFilter({ sender: sender, receiver: receiver, @@ -51,15 +48,17 @@ export class PoolController { functions: functions, })); - return pool ?? new TransactionPoolTooLarge(); + if (pool === null) { + throw new TransactionPoolTooLargeException(); + } + + return pool; } @Get("/pool/count") @ApiOperation({ summary: 'Transactions pool count', description: 'Returns the number of transactions that are currently in the memory pool.' }) - @ApiOkResponse({ - description: 'The count, or TransactionPoolTooLarge when it needs the pool and the pool is too large to be read', - schema: { oneOf: [{ type: 'number' }, { $ref: getSchemaPath(TransactionPoolTooLarge) }] }, - }) + @ApiOkResponse({ type: Number }) + @ApiServiceUnavailableResponse({ description: 'The transaction pool is too large to be displayed' }) @ApiQuery({ name: 'sender', description: 'Returns the number of transactions with a specific sender', required: false }) @ApiQuery({ name: 'receiver', description: 'Search in transaction pool by a specific receiver', required: false }) @ApiQuery({ name: 'senderShard', description: 'The shard of the sender', required: false }) @@ -71,7 +70,7 @@ export class PoolController { @Query('senderShard', ParseIntPipe) senderShard?: number, @Query('receiverShard', ParseIntPipe) receiverShard?: number, @Query('type', new ParseEnumPipe(TransactionType)) type?: TransactionType, - ): Promise { + ): Promise { const count = await this.poolService.getPoolCount(new PoolFilter({ sender: sender, receiver: receiver, @@ -80,7 +79,11 @@ export class PoolController { type: type, })); - return count ?? new TransactionPoolTooLarge(); + if (count === null) { + throw new TransactionPoolTooLargeException(); + } + + return count; } @Get("/pool/c") @@ -89,25 +92,26 @@ export class PoolController { @Query('sender', ParseAddressAndMetachainPipe) sender?: string, @Query('receiver', ParseAddressPipe) receiver?: string, @Query('type', new ParseEnumPipe(TransactionType)) type?: TransactionType, - ): Promise { + ): Promise { const count = await this.poolService.getPoolCount(new PoolFilter({ sender, receiver, type })); + if (count === null) { + throw new TransactionPoolTooLargeException(); + } - return count ?? new TransactionPoolTooLarge(); + return count; } @Get("/pool/:txhash") @ApiOperation({ summary: 'Transaction from pool', description: 'Returns a transaction from the memory pool.' }) - @ApiOkResponse({ - description: 'The transaction, or TransactionPoolTooLarge when the pool is too large to be searched', - schema: { oneOf: [{ $ref: getSchemaPath(TransactionInPool) }, { $ref: getSchemaPath(TransactionPoolTooLarge) }] }, - }) + @ApiOkResponse({ type: TransactionInPool }) @ApiNotFoundResponse({ description: 'Transaction not found' }) + @ApiServiceUnavailableResponse({ description: 'The transaction pool is too large to be displayed' }) async getTransactionFromPool( @Param('txhash', ParseTransactionHashPipe) txHash: string, - ): Promise { + ): Promise { const transaction = await this.poolService.getTransactionFromPool(txHash); if (transaction === null) { - return new TransactionPoolTooLarge(); + throw new TransactionPoolTooLargeException(); } if (transaction === undefined) { diff --git a/src/test/unit/services/cache.warmer.pool.spec.ts b/src/test/unit/services/cache.warmer.pool.spec.ts index 1029879a6c..e88c4bc995 100644 --- a/src/test/unit/services/cache.warmer.pool.spec.ts +++ b/src/test/unit/services/cache.warmer.pool.spec.ts @@ -4,7 +4,6 @@ import { TransactionType } from "src/endpoints/transactions/entities/transaction import { CacheInfo } from "src/utils/cache.info"; describe('CacheWarmerService transaction pool', () => { - const ttl = 10; const pool = [ { txHash: 'a', type: TransactionType.Transaction }, { txHash: 'b', type: TransactionType.Reward }, @@ -38,7 +37,6 @@ describe('CacheWarmerService transaction pool', () => { poolService, gatewayService, cachingService, - apiConfigService: { getTransactionPoolCacheWarmerTtlInSeconds: () => ttl }, clientProxy: { emit: jest.fn() }, }); }); @@ -46,11 +44,11 @@ describe('CacheWarmerService transaction pool', () => { it('should warm the pool and count the total and every type from it', async () => { await warmer.handleTxPoolInvalidations(); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, pool, ttl); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, 3, ttl); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(TransactionType.Transaction).key, 1, ttl); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(TransactionType.SmartContractResult).key, 0, ttl); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(TransactionType.Reward).key, 2, ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, pool, CacheInfo.TransactionPool.ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, 3, CacheInfo.TransactionPoolCount().ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(TransactionType.Transaction).key, 1, CacheInfo.TransactionPoolCount(TransactionType.Transaction).ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(TransactionType.SmartContractResult).key, 0, CacheInfo.TransactionPoolCount(TransactionType.SmartContractResult).ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(TransactionType.Reward).key, 2, CacheInfo.TransactionPoolCount(TransactionType.Reward).ttl); expect(gatewayService.getTransactionPoolCount).not.toHaveBeenCalled(); }); @@ -59,10 +57,10 @@ describe('CacheWarmerService transaction pool', () => { await warmer.handleTxPoolInvalidations(); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, null, ttl); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, gatewayCounts.total, ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, null, CacheInfo.TransactionPool.ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, gatewayCounts.total, CacheInfo.TransactionPoolCount().ttl); for (const type of Object.values(TransactionType)) { - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(type).key, gatewayCounts[type], ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(type).key, gatewayCounts[type], CacheInfo.TransactionPoolCount(type).ttl); } expect(gatewayService.getTransactionPoolCount).toHaveBeenCalledTimes(Object.values(TransactionType).length + 1); }); diff --git a/src/test/unit/services/pool.too.large.spec.ts b/src/test/unit/services/pool.too.large.spec.ts index cb26e26f2e..51e66ab5d7 100644 --- a/src/test/unit/services/pool.too.large.spec.ts +++ b/src/test/unit/services/pool.too.large.spec.ts @@ -1,13 +1,13 @@ import { NotFoundException } from "@nestjs/common"; import { PoolGateway } from "src/crons/websocket/pool.gateway"; import { PoolFilter } from "src/endpoints/pool/entities/pool.filter"; -import { TransactionPoolTooLarge } from "src/endpoints/pool/entities/transaction.pool.too.large"; +import { TransactionPoolTooLargeException } from "src/endpoints/pool/entities/transaction.pool.too.large.exception"; +import { PoolUpdateStatus } from "src/endpoints/pool/entities/pool.update.status"; import { PoolController } from "src/endpoints/pool/pool.controller"; import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; import { TransactionService } from "src/endpoints/transactions/transaction.service"; describe('Transaction pool too large', () => { - const tooLarge = { tooLarge: true, message: 'The transaction pool is too large to be displayed' }; const txHash = 'e07af9835b6da5740d0f791cfe65491a562852c57d44af63fdc14be5d73f01da'; let poolService: any; @@ -27,16 +27,22 @@ describe('Transaction pool too large', () => { controller = new PoolController(poolService); }); - it('should answer the pool and the transaction with a custom response', async () => { - expect(await controller.getTransactionPool(0, 25)).toStrictEqual(new TransactionPoolTooLarge()); - expect(await controller.getTransactionPool(0, 25)).toEqual(tooLarge); - expect(await controller.getTransactionFromPool(txHash)).toEqual(tooLarge); + it('should answer the pool and the transaction with the too large exception', async () => { + await expect(controller.getTransactionPool(0, 25)).rejects.toBeInstanceOf(TransactionPoolTooLargeException); + await expect(controller.getTransactionFromPool(txHash)).rejects.toBeInstanceOf(TransactionPoolTooLargeException); }); - it('should answer the counts the gateway gives and a custom response for the others', async () => { + it('should answer the counts the gateway gives and the too large exception for the others', async () => { expect(await controller.getTransactionPoolCount()).toStrictEqual(42); expect(await controller.getTransactionPoolCount(undefined, undefined, undefined, undefined, TransactionType.Reward)).toStrictEqual(42); - expect(await controller.getTransactionPoolCount('erd1qqqqqqqqqqqqqpgqp699jngundfqw07d8jzkepucvpzush6k3wvqyc44rx')).toEqual(tooLarge); + await expect(controller.getTransactionPoolCount('erd1qqqqqqqqqqqqqpgqp699jngundfqw07d8jzkepucvpzush6k3wvqyc44rx')).rejects.toBeInstanceOf(TransactionPoolTooLargeException); + }); + + it('should answer the too large exception as unavailable, with a code and a message', () => { + const exception = new TransactionPoolTooLargeException(); + + expect(exception.getStatus()).toStrictEqual(503); + expect(exception.getResponse()).toEqual({ statusCode: 503, code: 'transaction_pool_too_large', message: 'The transaction pool is too large to be displayed' }); }); it('should keep answering other failures as errors', async () => { @@ -73,14 +79,14 @@ describe('Transaction pool too large', () => { it('should send a null pool with the total count', async () => { await gateway.pushPoolForRoom('pool-{"from":0,"size":25}'); - expect(emit).toHaveBeenCalledWith('poolUpdate', { pool: null, poolCount: 42 }); + expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.tooLarge, pool: null, poolCount: 42 }); }); it('should send a null pool with the count of the type of the room', async () => { await gateway.pushPoolForRoom(`pool-{"from":0,"size":25,"type":"${TransactionType.Reward}"}`); expect(poolService.getPoolCount).toHaveBeenCalledWith(new PoolFilter({ type: TransactionType.Reward })); - expect(emit).toHaveBeenCalledWith('poolUpdate', { pool: null, poolCount: 42 }); + expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.tooLarge, pool: null, poolCount: 42 }); }); it('should send the pool when it can be read', async () => { @@ -88,7 +94,7 @@ describe('Transaction pool too large', () => { await gateway.pushPoolForRoom('pool-{"from":0,"size":25}'); - expect(emit).toHaveBeenCalledWith('poolUpdate', { pool: [{ txHash }], poolCount: 42 }); + expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.success, pool: [{ txHash }], poolCount: 42 }); }); }); }); From fb112969c08c8fcfb580f5f62c2d9befc8d58dea Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Tue, 29 Sep 2026 14:37:42 +0300 Subject: [PATCH 11/20] count total from gateway only when pool is too large, without type --- src/common/gateway/gateway.service.ts | 19 ++++++----- .../cache.warmer/cache.warmer.service.ts | 18 +++++------ src/endpoints/pool/pool.controller.ts | 1 + src/endpoints/pool/pool.service.ts | 26 ++++++--------- .../unit/services/cache.warmer.pool.spec.ts | 32 ++++--------------- .../unit/services/gateway.pool.count.spec.ts | 20 ++++++++---- src/test/unit/services/pool.spec.ts | 24 ++++---------- src/test/unit/services/pool.too.large.spec.ts | 9 +++--- src/utils/cache.info.ts | 11 +++---- 9 files changed, 66 insertions(+), 94 deletions(-) diff --git a/src/common/gateway/gateway.service.ts b/src/common/gateway/gateway.service.ts index 7b66e14f3d..a01d6ee37c 100644 --- a/src/common/gateway/gateway.service.ts +++ b/src/common/gateway/gateway.service.ts @@ -20,7 +20,6 @@ import { ApiService, ApiSettings } from "@multiversx/sdk-nestjs-http"; import { GuardianResult } from "./entities/guardian.result"; import { TransactionProcessStatus } from "./entities/transaction.process.status"; import { TxPoolGatewayResponse } from "./entities/tx.pool.gateway.response"; -import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; @Injectable() export class GatewayService { @@ -184,15 +183,19 @@ export class GatewayService { } async getTransactionPool(): Promise { - // eslint-disable-next-line require-await - return await this.get(`transaction/pool?fields=*`, GatewayComponentRequest.transactionPool, async (error) => { - return error?.message?.startsWith('maxContentLength size of') === true; - }); + try { + return await this.get(`transaction/pool?fields=*`, GatewayComponentRequest.transactionPool); + } catch (error: any) { + if (error?.message?.startsWith('maxContentLength size of')) { + return undefined; + } + + throw error; + } } - async getTransactionPoolCount(type?: TransactionType): Promise { - const url = type ? `transaction/pool/count?type=${type}` : 'transaction/pool/count'; - const result = await this.get(url, GatewayComponentRequest.transactionPool); + async getTransactionPoolCount(): Promise { + const result = await this.get('transaction/pool/count', GatewayComponentRequest.transactionPool); return Object.values(result.txPoolCounts).reduce((total, count) => total + count, 0); } diff --git a/src/crons/cache.warmer/cache.warmer.service.ts b/src/crons/cache.warmer/cache.warmer.service.ts index ceb56334f9..582a59939d 100644 --- a/src/crons/cache.warmer/cache.warmer.service.ts +++ b/src/crons/cache.warmer/cache.warmer.service.ts @@ -30,7 +30,6 @@ import { TokenDetailed } from "src/endpoints/tokens/entities/token.detailed"; import { DataApiService } from "src/common/data-api/data-api.service"; import { BlockService } from "src/endpoints/blocks/block.service"; import { PoolService } from "src/endpoints/pool/pool.service"; -import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; import * as JsonDiff from "json-diff"; import { QueryPagination } from "src/common/entities/query.pagination"; import { StakeService } from "src/endpoints/stake/stake.service"; @@ -153,16 +152,17 @@ export class CacheWarmerService { @Lock({ name: 'Transaction pool invalidation', verbose: true }) async handleTxPoolInvalidations() { const pool = await this.poolService.getTxPoolRaw(); - const types = [undefined, ...Object.values(TransactionType)]; - const counts = pool !== null - ? types.map(type => pool.filter(transaction => type === undefined || transaction.type === type).length) - : await Promise.all(types.map(type => this.gatewayService.getTransactionPoolCount(type))); - - await Promise.all([ + const invalidations = [ this.invalidateKey(CacheInfo.TransactionPool.key, pool, CacheInfo.TransactionPool.ttl), - ...types.map((type, index) => this.invalidateKey(CacheInfo.TransactionPoolCount(type).key, counts[index], CacheInfo.TransactionPoolCount(type).ttl)), - ]); + ]; + + if (pool === null) { + const count = await this.gatewayService.getTransactionPoolCount(); + invalidations.push(this.invalidateKey(CacheInfo.TransactionPoolCount.key, count, CacheInfo.TransactionPoolCount.ttl)); + } + + await Promise.all(invalidations); } @Cron('*/2 * * * *') diff --git a/src/endpoints/pool/pool.controller.ts b/src/endpoints/pool/pool.controller.ts index 099feeb5b1..ae4c81d4a3 100644 --- a/src/endpoints/pool/pool.controller.ts +++ b/src/endpoints/pool/pool.controller.ts @@ -94,6 +94,7 @@ export class PoolController { @Query('type', new ParseEnumPipe(TransactionType)) type?: TransactionType, ): Promise { const count = await this.poolService.getPoolCount(new PoolFilter({ sender, receiver, type })); + if (count === null) { throw new TransactionPoolTooLargeException(); } diff --git a/src/endpoints/pool/pool.service.ts b/src/endpoints/pool/pool.service.ts index ee7ad34462..6ec6707233 100644 --- a/src/endpoints/pool/pool.service.ts +++ b/src/endpoints/pool/pool.service.ts @@ -32,26 +32,20 @@ export class PoolService { } async getPoolCount(filter: PoolFilter): Promise { - const { type, ...otherFilters } = filter; - if (Object.values(otherFilters).some(value => value !== undefined)) { - const pool = await this.getPoolWithFilters(filter); - return pool === null ? null : pool.length; - } - - return await this.cacheService.getOrSet( - CacheInfo.TransactionPoolCount(type).key, - async () => await this.getPoolCountRaw(type), - CacheInfo.TransactionPoolCount(type).ttl, - ); - } - - async getPoolCountRaw(type?: TransactionType): Promise { - const pool = await this.getPoolWithFilters(new PoolFilter({ type })); + const pool = await this.getPoolWithFilters(filter); if (pool !== null) { return pool.length; } - return await this.gatewayService.getTransactionPoolCount(type); + if (Object.values(filter).some(value => value !== undefined)) { + return null; + } + + return await this.cacheService.getOrSet( + CacheInfo.TransactionPoolCount.key, + async () => await this.gatewayService.getTransactionPoolCount(), + CacheInfo.TransactionPoolCount.ttl, + ); } async getPool( diff --git a/src/test/unit/services/cache.warmer.pool.spec.ts b/src/test/unit/services/cache.warmer.pool.spec.ts index e88c4bc995..05ec88f1b2 100644 --- a/src/test/unit/services/cache.warmer.pool.spec.ts +++ b/src/test/unit/services/cache.warmer.pool.spec.ts @@ -4,17 +4,7 @@ import { TransactionType } from "src/endpoints/transactions/entities/transaction import { CacheInfo } from "src/utils/cache.info"; describe('CacheWarmerService transaction pool', () => { - const pool = [ - { txHash: 'a', type: TransactionType.Transaction }, - { txHash: 'b', type: TransactionType.Reward }, - { txHash: 'c', type: TransactionType.Reward }, - ]; - const gatewayCounts: Record = { - total: 12, - [TransactionType.Transaction]: 7, - [TransactionType.SmartContractResult]: 3, - [TransactionType.Reward]: 2, - }; + const pool = [{ txHash: 'a', type: TransactionType.Transaction }]; let warmer: CacheWarmerService; let poolService: any; @@ -28,9 +18,7 @@ describe('CacheWarmerService transaction pool', () => { }); poolService = { getTxPoolRaw: jest.fn().mockResolvedValue(pool) }; - gatewayService = { - getTransactionPoolCount: jest.fn().mockImplementation(async (type?: TransactionType) => await Promise.resolve(gatewayCounts[type ?? 'total'])), - }; + gatewayService = { getTransactionPoolCount: jest.fn().mockResolvedValue(12) }; cachingService = { set: jest.fn() }; warmer = Object.assign(Object.create(CacheWarmerService.prototype), { @@ -41,27 +29,21 @@ describe('CacheWarmerService transaction pool', () => { }); }); - it('should warm the pool and count the total and every type from it', async () => { + it('should warm only the pool when it can be read', async () => { await warmer.handleTxPoolInvalidations(); + expect(cachingService.set).toHaveBeenCalledTimes(1); expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, pool, CacheInfo.TransactionPool.ttl); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, 3, CacheInfo.TransactionPoolCount().ttl); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(TransactionType.Transaction).key, 1, CacheInfo.TransactionPoolCount(TransactionType.Transaction).ttl); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(TransactionType.SmartContractResult).key, 0, CacheInfo.TransactionPoolCount(TransactionType.SmartContractResult).ttl); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(TransactionType.Reward).key, 2, CacheInfo.TransactionPoolCount(TransactionType.Reward).ttl); expect(gatewayService.getTransactionPoolCount).not.toHaveBeenCalled(); }); - it('should warm the pool as null and count the total and every type through the gateway when it is too large', async () => { + it('should warm the pool as null and refresh the count from the gateway when the pool is too large', async () => { poolService.getTxPoolRaw.mockResolvedValue(null); await warmer.handleTxPoolInvalidations(); + expect(cachingService.set).toHaveBeenCalledTimes(2); expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, null, CacheInfo.TransactionPool.ttl); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, gatewayCounts.total, CacheInfo.TransactionPoolCount().ttl); - for (const type of Object.values(TransactionType)) { - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(type).key, gatewayCounts[type], CacheInfo.TransactionPoolCount(type).ttl); - } - expect(gatewayService.getTransactionPoolCount).toHaveBeenCalledTimes(Object.values(TransactionType).length + 1); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount.key, 12, CacheInfo.TransactionPoolCount.ttl); }); }); diff --git a/src/test/unit/services/gateway.pool.count.spec.ts b/src/test/unit/services/gateway.pool.count.spec.ts index 358379f059..d71d603685 100644 --- a/src/test/unit/services/gateway.pool.count.spec.ts +++ b/src/test/unit/services/gateway.pool.count.spec.ts @@ -1,7 +1,6 @@ import { GatewayService } from "src/common/gateway/gateway.service"; -import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; -describe('GatewayService transaction pool count', () => { +describe('GatewayService transaction pool', () => { let gatewayService: GatewayService; let apiService: any; @@ -17,13 +16,20 @@ describe('GatewayService transaction pool count', () => { Object.assign(gatewayService, { eventEmitter: { emit: jest.fn() } }); }); + it('should read a pool over the response size limit as undefined', async () => { + apiService.get.mockRejectedValue({ message: 'maxContentLength size of 2097152 exceeded' }); + + expect(await gatewayService.getTransactionPool()).toBeUndefined(); + }); + + it('should keep throwing other failures of the pool', async () => { + apiService.get.mockRejectedValue({ message: 'connect ECONNREFUSED' }); + + await expect(gatewayService.getTransactionPool()).rejects.toEqual({ message: 'connect ECONNREFUSED' }); + }); + it('should sum the counts of every shard', async () => { expect(await gatewayService.getTransactionPoolCount()).toStrictEqual(10); expect(apiService.get).toHaveBeenCalledWith('https://gateway/transaction/pool/count', expect.anything(), undefined); }); - - it('should ask for the count of a type', async () => { - expect(await gatewayService.getTransactionPoolCount(TransactionType.SmartContractResult)).toStrictEqual(10); - expect(apiService.get).toHaveBeenCalledWith('https://gateway/transaction/pool/count?type=SmartContractResult', expect.anything(), undefined); - }); }); diff --git a/src/test/unit/services/pool.spec.ts b/src/test/unit/services/pool.spec.ts index dee6393f86..f394501787 100644 --- a/src/test/unit/services/pool.spec.ts +++ b/src/test/unit/services/pool.spec.ts @@ -101,21 +101,11 @@ describe('PoolService', () => { it('should count the total from the pool, so that it matches what the pool lists', async () => { gatewayService.getTransactionPoolCount = jest.fn(); - const poolCount = await service.getPoolCount(new PoolFilter()); - expect(poolCount).toStrictEqual(7); - expect(cacheService.getOrSet).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount().key, expect.any(Function), CacheInfo.TransactionPoolCount().ttl); + expect(await service.getPoolCount(new PoolFilter())).toStrictEqual(7); expect(gatewayService.getTransactionPoolCount).not.toHaveBeenCalled(); }); - it('should cache the count for a type apart from the total', async () => { - const poolCount = await service.getPoolCount(new PoolFilter({ type: TransactionType.Reward })); - expect(poolCount).toStrictEqual(5); - expect(cacheService.getOrSet).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount(TransactionType.Reward).key, expect.any(Function), CacheInfo.TransactionPoolCount(TransactionType.Reward).ttl); - expect(CacheInfo.TransactionPoolCount(TransactionType.Reward).key).not.toStrictEqual(CacheInfo.TransactionPoolCount().key); - expect(gatewayService.getTransactionPoolCount).not.toHaveBeenCalled(); - }); - - it('should count the pool, without caching the count, for other filters', async () => { + it('should count the pool, without caching the count, for filters', async () => { const filter = new PoolFilter({ type: TransactionType.Transaction, senderShard: 0 }); const poolCount = await service.getPoolCount(filter); @@ -158,16 +148,16 @@ describe('PoolService', () => { expect(gatewayService.getTransactionPool).not.toHaveBeenCalled(); }); - it('should count the total and every type through the gateway', async () => { - gatewayService.getTransactionPoolCount = jest.fn().mockImplementation(async (type?: TransactionType) => await Promise.resolve(type ? 10 : 42)); + it('should count only the total, through the gateway and cached', async () => { + gatewayService.getTransactionPoolCount = jest.fn().mockResolvedValue(42); cacheService.getOrSet = jest.fn().mockImplementation(async (key: string, createValueFunc: () => Promise) => { return key === CacheInfo.TransactionPool.key ? null : await createValueFunc(); }); expect(await service.getPoolCount(new PoolFilter())).toStrictEqual(42); - expect(await service.getPoolCount(new PoolFilter({ type: TransactionType.Reward }))).toStrictEqual(10); - expect(gatewayService.getTransactionPoolCount).toHaveBeenCalledWith(undefined); - expect(gatewayService.getTransactionPoolCount).toHaveBeenCalledWith(TransactionType.Reward); + expect(await service.getPoolCount(new PoolFilter({ type: TransactionType.Reward }))).toBeNull(); + expect(cacheService.getOrSet).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount.key, expect.any(Function), CacheInfo.TransactionPoolCount.ttl); + expect(gatewayService.getTransactionPoolCount).toHaveBeenCalledTimes(1); }); }); diff --git a/src/test/unit/services/pool.too.large.spec.ts b/src/test/unit/services/pool.too.large.spec.ts index 51e66ab5d7..1038d7a677 100644 --- a/src/test/unit/services/pool.too.large.spec.ts +++ b/src/test/unit/services/pool.too.large.spec.ts @@ -16,7 +16,7 @@ describe('Transaction pool too large', () => { poolService = { getPool: jest.fn().mockResolvedValue(null), getTransactionFromPool: jest.fn().mockResolvedValue(null), - getPoolCount: jest.fn().mockImplementation(async (filter: PoolFilter) => await Promise.resolve(filter.sender ? null : 42)), + getPoolCount: jest.fn().mockImplementation(async (filter: PoolFilter) => await Promise.resolve(filter.sender || filter.type ? null : 42)), }; }); @@ -34,7 +34,7 @@ describe('Transaction pool too large', () => { it('should answer the counts the gateway gives and the too large exception for the others', async () => { expect(await controller.getTransactionPoolCount()).toStrictEqual(42); - expect(await controller.getTransactionPoolCount(undefined, undefined, undefined, undefined, TransactionType.Reward)).toStrictEqual(42); + await expect(controller.getTransactionPoolCount(undefined, undefined, undefined, undefined, TransactionType.Reward)).rejects.toBeInstanceOf(TransactionPoolTooLargeException); await expect(controller.getTransactionPoolCount('erd1qqqqqqqqqqqqqpgqp699jngundfqw07d8jzkepucvpzush6k3wvqyc44rx')).rejects.toBeInstanceOf(TransactionPoolTooLargeException); }); @@ -82,11 +82,10 @@ describe('Transaction pool too large', () => { expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.tooLarge, pool: null, poolCount: 42 }); }); - it('should send a null pool with the count of the type of the room', async () => { + it('should send a null pool with no count to a room filtered by type', async () => { await gateway.pushPoolForRoom(`pool-{"from":0,"size":25,"type":"${TransactionType.Reward}"}`); - expect(poolService.getPoolCount).toHaveBeenCalledWith(new PoolFilter({ type: TransactionType.Reward })); - expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.tooLarge, pool: null, poolCount: 42 }); + expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.tooLarge, pool: null, poolCount: null }); }); it('should send the pool when it can be read', async () => { diff --git a/src/utils/cache.info.ts b/src/utils/cache.info.ts index af6ec15098..cbcd880de2 100644 --- a/src/utils/cache.info.ts +++ b/src/utils/cache.info.ts @@ -2,7 +2,6 @@ import { Constants } from "@multiversx/sdk-nestjs-common"; import { QueryPagination } from "src/common/entities/query.pagination"; import { BlockFilter } from "src/endpoints/blocks/entities/block.filter"; import { TpsInterval } from "src/endpoints/tps/entities/tps.interval"; -import { TransactionType } from "src/endpoints/transactions/entities/transaction.type"; export class CacheInfo { key: string = ""; @@ -58,12 +57,10 @@ export class CacheInfo { ttl: Constants.oneSecond(), }; - static TransactionPoolCount(type?: TransactionType): CacheInfo { - return { - key: type ? `txpool:count:${type}` : 'txpool:count', - ttl: Constants.oneSecond(), - }; - } + static TransactionPoolCount: CacheInfo = { + key: 'txpool:count', + ttl: Constants.oneSecond(), + }; static ApplicationMostUsed: CacheInfo = { key: 'applicationMostUsed', From f3a207be1fc8b34b0f77e9b806c08d5c7ce8bf7b Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Tue, 29 Sep 2026 14:39:33 +0300 Subject: [PATCH 12/20] use loose null checks for pool --- src/crons/cache.warmer/cache.warmer.service.ts | 2 +- src/crons/websocket/pool.gateway.ts | 2 +- src/endpoints/pool/pool.controller.ts | 6 +++--- src/endpoints/pool/pool.service.ts | 8 ++++---- src/endpoints/transactions/transaction.service.ts | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/crons/cache.warmer/cache.warmer.service.ts b/src/crons/cache.warmer/cache.warmer.service.ts index 582a59939d..3b00f240d4 100644 --- a/src/crons/cache.warmer/cache.warmer.service.ts +++ b/src/crons/cache.warmer/cache.warmer.service.ts @@ -157,7 +157,7 @@ export class CacheWarmerService { this.invalidateKey(CacheInfo.TransactionPool.key, pool, CacheInfo.TransactionPool.ttl), ]; - if (pool === null) { + if (pool == null) { const count = await this.gatewayService.getTransactionPoolCount(); invalidations.push(this.invalidateKey(CacheInfo.TransactionPoolCount.key, count, CacheInfo.TransactionPoolCount.ttl)); } diff --git a/src/crons/websocket/pool.gateway.ts b/src/crons/websocket/pool.gateway.ts index 2be7e38510..0ae1b5f806 100644 --- a/src/crons/websocket/pool.gateway.ts +++ b/src/crons/websocket/pool.gateway.ts @@ -75,7 +75,7 @@ export class PoolGateway { this.poolService.getPoolCount(poolFilter), ]); - const status = pool === null ? PoolUpdateStatus.tooLarge : PoolUpdateStatus.success; + const status = pool == null ? PoolUpdateStatus.tooLarge : PoolUpdateStatus.success; this.server.to(roomName).emit("poolUpdate", { status, pool, poolCount }); } catch (error) { this.logger.error(error); diff --git a/src/endpoints/pool/pool.controller.ts b/src/endpoints/pool/pool.controller.ts index ae4c81d4a3..a4e9bf6c71 100644 --- a/src/endpoints/pool/pool.controller.ts +++ b/src/endpoints/pool/pool.controller.ts @@ -48,7 +48,7 @@ export class PoolController { functions: functions, })); - if (pool === null) { + if (pool == null) { throw new TransactionPoolTooLargeException(); } @@ -79,7 +79,7 @@ export class PoolController { type: type, })); - if (count === null) { + if (count == null) { throw new TransactionPoolTooLargeException(); } @@ -95,7 +95,7 @@ export class PoolController { ): Promise { const count = await this.poolService.getPoolCount(new PoolFilter({ sender, receiver, type })); - if (count === null) { + if (count == null) { throw new TransactionPoolTooLargeException(); } diff --git a/src/endpoints/pool/pool.service.ts b/src/endpoints/pool/pool.service.ts index 6ec6707233..61f0c1ef35 100644 --- a/src/endpoints/pool/pool.service.ts +++ b/src/endpoints/pool/pool.service.ts @@ -24,7 +24,7 @@ export class PoolService { async getTransactionFromPool(txHash: string): Promise { const pool = await this.getPoolWithFilters(); - if (pool === null) { + if (pool == null) { return null; } @@ -33,7 +33,7 @@ export class PoolService { async getPoolCount(filter: PoolFilter): Promise { const pool = await this.getPoolWithFilters(filter); - if (pool !== null) { + if (pool != null) { return pool.length; } @@ -58,7 +58,7 @@ export class PoolService { const { from, size } = queryPagination; const pool = await this.getPoolWithFilters(filter); - if (pool === null) { + if (pool == null) { return null; } @@ -76,7 +76,7 @@ export class PoolService { true, ); - if (pool === null) { + if (pool == null) { return null; } diff --git a/src/endpoints/transactions/transaction.service.ts b/src/endpoints/transactions/transaction.service.ts index 495e776548..c87d98a7ec 100644 --- a/src/endpoints/transactions/transaction.service.ts +++ b/src/endpoints/transactions/transaction.service.ts @@ -752,7 +752,7 @@ export class TransactionService { const poolTransactions = await this.poolService.getPoolWithFilters({ senderShard: shardId }); - if (poolTransactions === null) { + if (poolTransactions == null) { return null; } From 58307a9b625c3c193777616b6ae169f6b2626577 Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Tue, 29 Sep 2026 14:42:24 +0300 Subject: [PATCH 13/20] return total pool count for type filter when pool is too large --- src/endpoints/pool/pool.service.ts | 2 +- src/test/unit/services/pool.spec.ts | 7 ++++--- src/test/unit/services/pool.too.large.spec.ts | 8 ++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/endpoints/pool/pool.service.ts b/src/endpoints/pool/pool.service.ts index 61f0c1ef35..a7467438ae 100644 --- a/src/endpoints/pool/pool.service.ts +++ b/src/endpoints/pool/pool.service.ts @@ -37,7 +37,7 @@ export class PoolService { return pool.length; } - if (Object.values(filter).some(value => value !== undefined)) { + if (Object.entries(filter).some(([key, value]) => key !== 'type' && value !== undefined)) { return null; } diff --git a/src/test/unit/services/pool.spec.ts b/src/test/unit/services/pool.spec.ts index f394501787..b0e652d001 100644 --- a/src/test/unit/services/pool.spec.ts +++ b/src/test/unit/services/pool.spec.ts @@ -148,16 +148,17 @@ describe('PoolService', () => { expect(gatewayService.getTransactionPool).not.toHaveBeenCalled(); }); - it('should count only the total, through the gateway and cached', async () => { + it('should count the total through the gateway, cached, also for a type', async () => { gatewayService.getTransactionPoolCount = jest.fn().mockResolvedValue(42); cacheService.getOrSet = jest.fn().mockImplementation(async (key: string, createValueFunc: () => Promise) => { return key === CacheInfo.TransactionPool.key ? null : await createValueFunc(); }); expect(await service.getPoolCount(new PoolFilter())).toStrictEqual(42); - expect(await service.getPoolCount(new PoolFilter({ type: TransactionType.Reward }))).toBeNull(); + expect(await service.getPoolCount(new PoolFilter({ type: TransactionType.Reward }))).toStrictEqual(42); + expect(await service.getPoolCount(new PoolFilter({ type: TransactionType.Reward, sender: 'erd1qqqqqqqqqqqqqpgqp699jngundfqw07d8jzkepucvpzush6k3wvqyc44rx' }))).toBeNull(); expect(cacheService.getOrSet).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount.key, expect.any(Function), CacheInfo.TransactionPoolCount.ttl); - expect(gatewayService.getTransactionPoolCount).toHaveBeenCalledTimes(1); + expect(gatewayService.getTransactionPoolCount).toHaveBeenCalledTimes(2); }); }); diff --git a/src/test/unit/services/pool.too.large.spec.ts b/src/test/unit/services/pool.too.large.spec.ts index 1038d7a677..4bacecbd82 100644 --- a/src/test/unit/services/pool.too.large.spec.ts +++ b/src/test/unit/services/pool.too.large.spec.ts @@ -16,7 +16,7 @@ describe('Transaction pool too large', () => { poolService = { getPool: jest.fn().mockResolvedValue(null), getTransactionFromPool: jest.fn().mockResolvedValue(null), - getPoolCount: jest.fn().mockImplementation(async (filter: PoolFilter) => await Promise.resolve(filter.sender || filter.type ? null : 42)), + getPoolCount: jest.fn().mockImplementation(async (filter: PoolFilter) => await Promise.resolve(filter.sender ? null : 42)), }; }); @@ -34,7 +34,7 @@ describe('Transaction pool too large', () => { it('should answer the counts the gateway gives and the too large exception for the others', async () => { expect(await controller.getTransactionPoolCount()).toStrictEqual(42); - await expect(controller.getTransactionPoolCount(undefined, undefined, undefined, undefined, TransactionType.Reward)).rejects.toBeInstanceOf(TransactionPoolTooLargeException); + expect(await controller.getTransactionPoolCount(undefined, undefined, undefined, undefined, TransactionType.Reward)).toStrictEqual(42); await expect(controller.getTransactionPoolCount('erd1qqqqqqqqqqqqqpgqp699jngundfqw07d8jzkepucvpzush6k3wvqyc44rx')).rejects.toBeInstanceOf(TransactionPoolTooLargeException); }); @@ -82,10 +82,10 @@ describe('Transaction pool too large', () => { expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.tooLarge, pool: null, poolCount: 42 }); }); - it('should send a null pool with no count to a room filtered by type', async () => { + it('should send a null pool with the total count to a room filtered by type', async () => { await gateway.pushPoolForRoom(`pool-{"from":0,"size":25,"type":"${TransactionType.Reward}"}`); - expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.tooLarge, pool: null, poolCount: null }); + expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.tooLarge, pool: null, poolCount: 42 }); }); it('should send the pool when it can be read', async () => { From 68166cba932b0377cfbf40667a2e00fecd67492b Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Tue, 29 Sep 2026 14:44:10 +0300 Subject: [PATCH 14/20] return total pool count for any filter when pool is too large --- src/endpoints/pool/pool.controller.ts | 17 ++--------------- src/endpoints/pool/pool.service.ts | 6 +----- src/test/unit/services/pool.spec.ts | 7 +++---- src/test/unit/services/pool.too.large.spec.ts | 7 +++---- 4 files changed, 9 insertions(+), 28 deletions(-) diff --git a/src/endpoints/pool/pool.controller.ts b/src/endpoints/pool/pool.controller.ts index a4e9bf6c71..80d9872958 100644 --- a/src/endpoints/pool/pool.controller.ts +++ b/src/endpoints/pool/pool.controller.ts @@ -58,7 +58,6 @@ export class PoolController { @Get("/pool/count") @ApiOperation({ summary: 'Transactions pool count', description: 'Returns the number of transactions that are currently in the memory pool.' }) @ApiOkResponse({ type: Number }) - @ApiServiceUnavailableResponse({ description: 'The transaction pool is too large to be displayed' }) @ApiQuery({ name: 'sender', description: 'Returns the number of transactions with a specific sender', required: false }) @ApiQuery({ name: 'receiver', description: 'Search in transaction pool by a specific receiver', required: false }) @ApiQuery({ name: 'senderShard', description: 'The shard of the sender', required: false }) @@ -71,19 +70,13 @@ export class PoolController { @Query('receiverShard', ParseIntPipe) receiverShard?: number, @Query('type', new ParseEnumPipe(TransactionType)) type?: TransactionType, ): Promise { - const count = await this.poolService.getPoolCount(new PoolFilter({ + return await this.poolService.getPoolCount(new PoolFilter({ sender: sender, receiver: receiver, senderShard: senderShard, receiverShard: receiverShard, type: type, })); - - if (count == null) { - throw new TransactionPoolTooLargeException(); - } - - return count; } @Get("/pool/c") @@ -93,13 +86,7 @@ export class PoolController { @Query('receiver', ParseAddressPipe) receiver?: string, @Query('type', new ParseEnumPipe(TransactionType)) type?: TransactionType, ): Promise { - const count = await this.poolService.getPoolCount(new PoolFilter({ sender, receiver, type })); - - if (count == null) { - throw new TransactionPoolTooLargeException(); - } - - return count; + return await this.poolService.getPoolCount(new PoolFilter({ sender, receiver, type })); } @Get("/pool/:txhash") diff --git a/src/endpoints/pool/pool.service.ts b/src/endpoints/pool/pool.service.ts index a7467438ae..7df2f2142f 100644 --- a/src/endpoints/pool/pool.service.ts +++ b/src/endpoints/pool/pool.service.ts @@ -31,16 +31,12 @@ export class PoolService { return pool.find(tx => tx.txHash === txHash); } - async getPoolCount(filter: PoolFilter): Promise { + async getPoolCount(filter: PoolFilter): Promise { const pool = await this.getPoolWithFilters(filter); if (pool != null) { return pool.length; } - if (Object.entries(filter).some(([key, value]) => key !== 'type' && value !== undefined)) { - return null; - } - return await this.cacheService.getOrSet( CacheInfo.TransactionPoolCount.key, async () => await this.gatewayService.getTransactionPoolCount(), diff --git a/src/test/unit/services/pool.spec.ts b/src/test/unit/services/pool.spec.ts index b0e652d001..a34db26be6 100644 --- a/src/test/unit/services/pool.spec.ts +++ b/src/test/unit/services/pool.spec.ts @@ -144,11 +144,10 @@ describe('PoolService', () => { expect(await service.getPool(new QueryPagination(), new PoolFilter())).toBeNull(); expect(await service.getTransactionFromPool('e07af9835b6da5740d0f791cfe65491a562852c57d44af63fdc14be5d73f01da')).toBeNull(); - expect(await service.getPoolCount(new PoolFilter({ sender: 'erd1qqqqqqqqqqqqqpgqp699jngundfqw07d8jzkepucvpzush6k3wvqyc44rx' }))).toBeNull(); expect(gatewayService.getTransactionPool).not.toHaveBeenCalled(); }); - it('should count the total through the gateway, cached, also for a type', async () => { + it('should count the total through the gateway, cached, whatever the filters', async () => { gatewayService.getTransactionPoolCount = jest.fn().mockResolvedValue(42); cacheService.getOrSet = jest.fn().mockImplementation(async (key: string, createValueFunc: () => Promise) => { return key === CacheInfo.TransactionPool.key ? null : await createValueFunc(); @@ -156,9 +155,9 @@ describe('PoolService', () => { expect(await service.getPoolCount(new PoolFilter())).toStrictEqual(42); expect(await service.getPoolCount(new PoolFilter({ type: TransactionType.Reward }))).toStrictEqual(42); - expect(await service.getPoolCount(new PoolFilter({ type: TransactionType.Reward, sender: 'erd1qqqqqqqqqqqqqpgqp699jngundfqw07d8jzkepucvpzush6k3wvqyc44rx' }))).toBeNull(); + expect(await service.getPoolCount(new PoolFilter({ type: TransactionType.Reward, sender: 'erd1qqqqqqqqqqqqqpgqp699jngundfqw07d8jzkepucvpzush6k3wvqyc44rx' }))).toStrictEqual(42); expect(cacheService.getOrSet).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount.key, expect.any(Function), CacheInfo.TransactionPoolCount.ttl); - expect(gatewayService.getTransactionPoolCount).toHaveBeenCalledTimes(2); + expect(gatewayService.getTransactionPoolCount).toHaveBeenCalledTimes(3); }); }); diff --git a/src/test/unit/services/pool.too.large.spec.ts b/src/test/unit/services/pool.too.large.spec.ts index 4bacecbd82..6fa6948d41 100644 --- a/src/test/unit/services/pool.too.large.spec.ts +++ b/src/test/unit/services/pool.too.large.spec.ts @@ -1,6 +1,5 @@ import { NotFoundException } from "@nestjs/common"; import { PoolGateway } from "src/crons/websocket/pool.gateway"; -import { PoolFilter } from "src/endpoints/pool/entities/pool.filter"; import { TransactionPoolTooLargeException } from "src/endpoints/pool/entities/transaction.pool.too.large.exception"; import { PoolUpdateStatus } from "src/endpoints/pool/entities/pool.update.status"; import { PoolController } from "src/endpoints/pool/pool.controller"; @@ -16,7 +15,7 @@ describe('Transaction pool too large', () => { poolService = { getPool: jest.fn().mockResolvedValue(null), getTransactionFromPool: jest.fn().mockResolvedValue(null), - getPoolCount: jest.fn().mockImplementation(async (filter: PoolFilter) => await Promise.resolve(filter.sender ? null : 42)), + getPoolCount: jest.fn().mockResolvedValue(42), }; }); @@ -32,10 +31,10 @@ describe('Transaction pool too large', () => { await expect(controller.getTransactionFromPool(txHash)).rejects.toBeInstanceOf(TransactionPoolTooLargeException); }); - it('should answer the counts the gateway gives and the too large exception for the others', async () => { + it('should answer the count, whatever the filters', async () => { expect(await controller.getTransactionPoolCount()).toStrictEqual(42); expect(await controller.getTransactionPoolCount(undefined, undefined, undefined, undefined, TransactionType.Reward)).toStrictEqual(42); - await expect(controller.getTransactionPoolCount('erd1qqqqqqqqqqqqqpgqp699jngundfqw07d8jzkepucvpzush6k3wvqyc44rx')).rejects.toBeInstanceOf(TransactionPoolTooLargeException); + expect(await controller.getTransactionPoolCount('erd1qqqqqqqqqqqqqpgqp699jngundfqw07d8jzkepucvpzush6k3wvqyc44rx')).toStrictEqual(42); }); it('should answer the too large exception as unavailable, with a code and a message', () => { From 84a7933ae13e6867eb7c7422f1fdf4f194f76208 Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Tue, 29 Sep 2026 14:46:39 +0300 Subject: [PATCH 15/20] use tx pool warmer ttl from config in cache warmer --- src/crons/cache.warmer/cache.warmer.service.ts | 5 +++-- src/test/unit/services/cache.warmer.pool.spec.ts | 8 +++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/crons/cache.warmer/cache.warmer.service.ts b/src/crons/cache.warmer/cache.warmer.service.ts index 3b00f240d4..2a123873f9 100644 --- a/src/crons/cache.warmer/cache.warmer.service.ts +++ b/src/crons/cache.warmer/cache.warmer.service.ts @@ -151,15 +151,16 @@ export class CacheWarmerService { @Lock({ name: 'Transaction pool invalidation', verbose: true }) async handleTxPoolInvalidations() { + const ttl = this.apiConfigService.getTransactionPoolCacheWarmerTtlInSeconds(); const pool = await this.poolService.getTxPoolRaw(); const invalidations = [ - this.invalidateKey(CacheInfo.TransactionPool.key, pool, CacheInfo.TransactionPool.ttl), + this.invalidateKey(CacheInfo.TransactionPool.key, pool, ttl), ]; if (pool == null) { const count = await this.gatewayService.getTransactionPoolCount(); - invalidations.push(this.invalidateKey(CacheInfo.TransactionPoolCount.key, count, CacheInfo.TransactionPoolCount.ttl)); + invalidations.push(this.invalidateKey(CacheInfo.TransactionPoolCount.key, count, ttl)); } await Promise.all(invalidations); diff --git a/src/test/unit/services/cache.warmer.pool.spec.ts b/src/test/unit/services/cache.warmer.pool.spec.ts index 05ec88f1b2..c16847a413 100644 --- a/src/test/unit/services/cache.warmer.pool.spec.ts +++ b/src/test/unit/services/cache.warmer.pool.spec.ts @@ -4,6 +4,7 @@ import { TransactionType } from "src/endpoints/transactions/entities/transaction import { CacheInfo } from "src/utils/cache.info"; describe('CacheWarmerService transaction pool', () => { + const ttl = 10; const pool = [{ txHash: 'a', type: TransactionType.Transaction }]; let warmer: CacheWarmerService; @@ -25,6 +26,7 @@ describe('CacheWarmerService transaction pool', () => { poolService, gatewayService, cachingService, + apiConfigService: { getTransactionPoolCacheWarmerTtlInSeconds: () => ttl }, clientProxy: { emit: jest.fn() }, }); }); @@ -33,7 +35,7 @@ describe('CacheWarmerService transaction pool', () => { await warmer.handleTxPoolInvalidations(); expect(cachingService.set).toHaveBeenCalledTimes(1); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, pool, CacheInfo.TransactionPool.ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, pool, ttl); expect(gatewayService.getTransactionPoolCount).not.toHaveBeenCalled(); }); @@ -43,7 +45,7 @@ describe('CacheWarmerService transaction pool', () => { await warmer.handleTxPoolInvalidations(); expect(cachingService.set).toHaveBeenCalledTimes(2); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, null, CacheInfo.TransactionPool.ttl); - expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount.key, 12, CacheInfo.TransactionPoolCount.ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPool.key, null, ttl); + expect(cachingService.set).toHaveBeenCalledWith(CacheInfo.TransactionPoolCount.key, 12, ttl); }); }); From 144591bb1a1964abc31e34893b16825bfc1d8df6 Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Tue, 29 Sep 2026 14:51:42 +0300 Subject: [PATCH 16/20] return null from gateway when pool is too large --- src/common/gateway/gateway.service.ts | 4 ++-- src/endpoints/pool/pool.service.ts | 2 +- src/test/unit/services/gateway.pool.count.spec.ts | 4 ++-- src/test/unit/services/pool.spec.ts | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/common/gateway/gateway.service.ts b/src/common/gateway/gateway.service.ts index a01d6ee37c..80e5bd2fa4 100644 --- a/src/common/gateway/gateway.service.ts +++ b/src/common/gateway/gateway.service.ts @@ -182,12 +182,12 @@ export class GatewayService { return new NftData(result.tokenData); } - async getTransactionPool(): Promise { + async getTransactionPool(): Promise { try { return await this.get(`transaction/pool?fields=*`, GatewayComponentRequest.transactionPool); } catch (error: any) { if (error?.message?.startsWith('maxContentLength size of')) { - return undefined; + return null; } throw error; diff --git a/src/endpoints/pool/pool.service.ts b/src/endpoints/pool/pool.service.ts index 7df2f2142f..ca8876807c 100644 --- a/src/endpoints/pool/pool.service.ts +++ b/src/endpoints/pool/pool.service.ts @@ -81,7 +81,7 @@ export class PoolService { async getTxPoolRaw(): Promise { const pool = await this.gatewayService.getTransactionPool(); - if (!pool) { + if (pool == null) { return null; } diff --git a/src/test/unit/services/gateway.pool.count.spec.ts b/src/test/unit/services/gateway.pool.count.spec.ts index d71d603685..5d94938f6a 100644 --- a/src/test/unit/services/gateway.pool.count.spec.ts +++ b/src/test/unit/services/gateway.pool.count.spec.ts @@ -16,10 +16,10 @@ describe('GatewayService transaction pool', () => { Object.assign(gatewayService, { eventEmitter: { emit: jest.fn() } }); }); - it('should read a pool over the response size limit as undefined', async () => { + it('should read a pool over the response size limit as null', async () => { apiService.get.mockRejectedValue({ message: 'maxContentLength size of 2097152 exceeded' }); - expect(await gatewayService.getTransactionPool()).toBeUndefined(); + expect(await gatewayService.getTransactionPool()).toBeNull(); }); it('should keep throwing other failures of the pool', async () => { diff --git a/src/test/unit/services/pool.spec.ts b/src/test/unit/services/pool.spec.ts index a34db26be6..b0076c8a70 100644 --- a/src/test/unit/services/pool.spec.ts +++ b/src/test/unit/services/pool.spec.ts @@ -131,7 +131,7 @@ describe('PoolService', () => { describe('pool too large', () => { it('should read a pool too large for the gateway response limit as null', async () => { - gatewayService.getTransactionPool = jest.fn().mockResolvedValue(undefined); + gatewayService.getTransactionPool = jest.fn().mockResolvedValue(null); expect(await service.getTxPoolRaw()).toBeNull(); }); From d100e98120c753366aa210e6319cb630655b494f Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Wed, 30 Sep 2026 12:08:50 +0300 Subject: [PATCH 17/20] send internal server error status on pool websocket --- src/crons/websocket/pool.gateway.ts | 1 + src/endpoints/pool/entities/pool.update.status.ts | 1 + src/test/unit/services/pool.too.large.spec.ts | 8 ++++++++ 3 files changed, 10 insertions(+) diff --git a/src/crons/websocket/pool.gateway.ts b/src/crons/websocket/pool.gateway.ts index 0ae1b5f806..4bab065764 100644 --- a/src/crons/websocket/pool.gateway.ts +++ b/src/crons/websocket/pool.gateway.ts @@ -79,6 +79,7 @@ export class PoolGateway { this.server.to(roomName).emit("poolUpdate", { status, pool, poolCount }); } catch (error) { this.logger.error(error); + this.server.to(roomName).emit("poolUpdate", { status: PoolUpdateStatus.internalServerError, pool: null, poolCount: null }); } } diff --git a/src/endpoints/pool/entities/pool.update.status.ts b/src/endpoints/pool/entities/pool.update.status.ts index c33d828402..8b56289f44 100644 --- a/src/endpoints/pool/entities/pool.update.status.ts +++ b/src/endpoints/pool/entities/pool.update.status.ts @@ -1,4 +1,5 @@ export enum PoolUpdateStatus { success = 'success', tooLarge = 'tooLarge', + internalServerError = 'internalServerError', } diff --git a/src/test/unit/services/pool.too.large.spec.ts b/src/test/unit/services/pool.too.large.spec.ts index 6fa6948d41..5448d15694 100644 --- a/src/test/unit/services/pool.too.large.spec.ts +++ b/src/test/unit/services/pool.too.large.spec.ts @@ -94,5 +94,13 @@ describe('Transaction pool too large', () => { expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.success, pool: [{ txHash }], poolCount: 42 }); }); + + it('should send the internal server error status when the pool fails for another reason', async () => { + poolService.getPool.mockRejectedValue(new Error('gateway unreachable')); + + await gateway.pushPoolForRoom('pool-{"from":0,"size":25}'); + + expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.internalServerError, pool: null, poolCount: null }); + }); }); }); From 29ab1dfeab0eb3d258214212f2d3c2b2828997f3 Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Wed, 30 Sep 2026 12:11:01 +0300 Subject: [PATCH 18/20] count pool through gateway whenever pool fails --- src/crons/websocket/pool.gateway.ts | 12 ++++++++++-- src/endpoints/pool/pool.service.ts | 2 +- src/test/unit/services/pool.spec.ts | 12 ++++++++++++ src/test/unit/services/pool.too.large.spec.ts | 11 ++++++++++- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/crons/websocket/pool.gateway.ts b/src/crons/websocket/pool.gateway.ts index 4bab065764..e8974d8539 100644 --- a/src/crons/websocket/pool.gateway.ts +++ b/src/crons/websocket/pool.gateway.ts @@ -71,11 +71,19 @@ export class PoolGateway { size: filter.size, }), poolFilter, - ), + ).catch((error) => { + this.logger.error(error); + return undefined; + }), this.poolService.getPoolCount(poolFilter), ]); - const status = pool == null ? PoolUpdateStatus.tooLarge : PoolUpdateStatus.success; + if (pool === undefined) { + this.server.to(roomName).emit("poolUpdate", { status: PoolUpdateStatus.internalServerError, pool: null, poolCount }); + return; + } + + const status = pool === null ? PoolUpdateStatus.tooLarge : PoolUpdateStatus.success; this.server.to(roomName).emit("poolUpdate", { status, pool, poolCount }); } catch (error) { this.logger.error(error); diff --git a/src/endpoints/pool/pool.service.ts b/src/endpoints/pool/pool.service.ts index ca8876807c..84ffd8c001 100644 --- a/src/endpoints/pool/pool.service.ts +++ b/src/endpoints/pool/pool.service.ts @@ -32,7 +32,7 @@ export class PoolService { } async getPoolCount(filter: PoolFilter): Promise { - const pool = await this.getPoolWithFilters(filter); + const pool = await this.getPoolWithFilters(filter).catch(() => null); if (pool != null) { return pool.length; } diff --git a/src/test/unit/services/pool.spec.ts b/src/test/unit/services/pool.spec.ts index b0076c8a70..36a9586d49 100644 --- a/src/test/unit/services/pool.spec.ts +++ b/src/test/unit/services/pool.spec.ts @@ -161,6 +161,18 @@ describe('PoolService', () => { }); }); + describe('pool failing for another reason', () => { + it('should still count the total through the gateway', async () => { + gatewayService.getTransactionPoolCount = jest.fn().mockResolvedValue(42); + cacheService.getOrSet = jest.fn().mockImplementation(async (key: string, createValueFunc: () => Promise) => { + return key === CacheInfo.TransactionPool.key ? await Promise.reject(new Error('gateway unreachable')) : await createValueFunc(); + }); + + expect(await service.getPoolCount(new PoolFilter({ type: TransactionType.Reward }))).toStrictEqual(42); + await expect(service.getPool(new QueryPagination(), new PoolFilter())).rejects.toThrow('gateway unreachable'); + }); + }); + describe('getTransactionFromPool', () => { it('should work and return the transaction', async () => { const tx = await service.getTransactionFromPool("e07af9835b6da5740d0f791cfe65491a562852c57d44af63fdc14be5d73f01da"); diff --git a/src/test/unit/services/pool.too.large.spec.ts b/src/test/unit/services/pool.too.large.spec.ts index 5448d15694..fb987a8cdc 100644 --- a/src/test/unit/services/pool.too.large.spec.ts +++ b/src/test/unit/services/pool.too.large.spec.ts @@ -95,11 +95,20 @@ describe('Transaction pool too large', () => { expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.success, pool: [{ txHash }], poolCount: 42 }); }); - it('should send the internal server error status when the pool fails for another reason', async () => { + it('should send the internal server error status with the count when the pool fails for another reason', async () => { poolService.getPool.mockRejectedValue(new Error('gateway unreachable')); await gateway.pushPoolForRoom('pool-{"from":0,"size":25}'); + expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.internalServerError, pool: null, poolCount: 42 }); + }); + + it('should send the internal server error status without a count when the count fails as well', async () => { + poolService.getPool.mockRejectedValue(new Error('gateway unreachable')); + poolService.getPoolCount.mockRejectedValue(new Error('gateway unreachable')); + + await gateway.pushPoolForRoom('pool-{"from":0,"size":25}'); + expect(emit).toHaveBeenCalledWith('poolUpdate', { status: PoolUpdateStatus.internalServerError, pool: null, poolCount: null }); }); }); From 24bf10f64d890316cbd0478009fabd7d3a5313b6 Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Wed, 30 Sep 2026 12:15:08 +0300 Subject: [PATCH 19/20] use null pool with status for any pool error on websocket --- src/crons/websocket/pool.gateway.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/crons/websocket/pool.gateway.ts b/src/crons/websocket/pool.gateway.ts index e8974d8539..3b2cb29d30 100644 --- a/src/crons/websocket/pool.gateway.ts +++ b/src/crons/websocket/pool.gateway.ts @@ -64,6 +64,8 @@ export class PoolGateway { type: filter.type, }); + let status = PoolUpdateStatus.success; + const [pool, poolCount] = await Promise.all([ this.poolService.getPool( new QueryPagination({ @@ -73,17 +75,16 @@ export class PoolGateway { poolFilter, ).catch((error) => { this.logger.error(error); - return undefined; + status = PoolUpdateStatus.internalServerError; + return null; }), this.poolService.getPoolCount(poolFilter), ]); - if (pool === undefined) { - this.server.to(roomName).emit("poolUpdate", { status: PoolUpdateStatus.internalServerError, pool: null, poolCount }); - return; + if (pool == null && status === PoolUpdateStatus.success) { + status = PoolUpdateStatus.tooLarge; } - const status = pool === null ? PoolUpdateStatus.tooLarge : PoolUpdateStatus.success; this.server.to(roomName).emit("poolUpdate", { status, pool, poolCount }); } catch (error) { this.logger.error(error); From f4332d3796bcaed69e87947a57222b898a64413f Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Wed, 30 Sep 2026 12:23:38 +0300 Subject: [PATCH 20/20] keep price per unit as before and fail pool with filters when pool is too large --- src/endpoints/pool/pool.service.ts | 30 +++++++++++-------- .../transactions/transaction.service.ts | 6 +--- src/test/unit/services/pool.spec.ts | 7 +++++ src/test/unit/services/pool.too.large.spec.ts | 5 ++-- 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/src/endpoints/pool/pool.service.ts b/src/endpoints/pool/pool.service.ts index 84ffd8c001..3e667ebbdb 100644 --- a/src/endpoints/pool/pool.service.ts +++ b/src/endpoints/pool/pool.service.ts @@ -12,6 +12,7 @@ import { TxInPoolFields } from "src/common/gateway/entities/tx.in.pool.fields"; import { TransactionActionService } from "../transactions/transaction-action/transaction.action.service"; import { Transaction } from "../transactions/entities/transaction"; import { ApiUtils } from "@multiversx/sdk-nestjs-http"; +import { TransactionPoolTooLargeException } from "./entities/transaction.pool.too.large.exception"; @Injectable() export class PoolService { @@ -23,7 +24,7 @@ export class PoolService { ) { } async getTransactionFromPool(txHash: string): Promise { - const pool = await this.getPoolWithFilters(); + const pool = await this.getTxPool(); if (pool == null) { return null; } @@ -32,9 +33,9 @@ export class PoolService { } async getPoolCount(filter: PoolFilter): Promise { - const pool = await this.getPoolWithFilters(filter).catch(() => null); + const pool = await this.getTxPool().catch(() => null); if (pool != null) { - return pool.length; + return this.applyFilters(pool, filter).length; } return await this.cacheService.getOrSet( @@ -53,30 +54,33 @@ export class PoolService { } const { from, size } = queryPagination; - const pool = await this.getPoolWithFilters(filter); + const pool = await this.getTxPool(); if (pool == null) { return null; } - return pool.slice(from, from + size); + return this.applyFilters(pool, filter).slice(from, from + size); } async getPoolWithFilters( filter?: PoolFilter, - ): Promise { - const pool = await this.cacheService.getOrSet( + ): Promise { + const pool = await this.getTxPool(); + if (pool == null) { + throw new TransactionPoolTooLargeException(); + } + + return this.applyFilters(pool, filter); + } + + private async getTxPool(): Promise { + return await this.cacheService.getOrSet( CacheInfo.TransactionPool.key, async () => await this.getTxPoolRaw(), CacheInfo.TransactionPool.ttl, CacheInfo.TransactionPool.ttl, true, ); - - if (pool == null) { - return null; - } - - return this.applyFilters(pool, filter); } async getTxPoolRaw(): Promise { diff --git a/src/endpoints/transactions/transaction.service.ts b/src/endpoints/transactions/transaction.service.ts index c87d98a7ec..ab8577fc0e 100644 --- a/src/endpoints/transactions/transaction.service.ts +++ b/src/endpoints/transactions/transaction.service.ts @@ -752,11 +752,7 @@ export class TransactionService { const poolTransactions = await this.poolService.getPoolWithFilters({ senderShard: shardId }); - if (poolTransactions == null) { - return null; - } - - if (poolTransactions.length === 0) { + if (!poolTransactions || poolTransactions.length === 0) { return new PpuMetadata({ lastBlock: lastBlock, fast: 0, diff --git a/src/test/unit/services/pool.spec.ts b/src/test/unit/services/pool.spec.ts index 36a9586d49..43187ebe78 100644 --- a/src/test/unit/services/pool.spec.ts +++ b/src/test/unit/services/pool.spec.ts @@ -9,6 +9,7 @@ import { TransactionType } from "src/endpoints/transactions/entities/transaction import { ProtocolService } from "../../../common/protocol/protocol.service"; import { TransactionActionService } from "../../../endpoints/transactions/transaction-action/transaction.action.service"; import { CacheInfo } from "src/utils/cache.info"; +import { TransactionPoolTooLargeException } from "src/endpoints/pool/entities/transaction.pool.too.large.exception"; describe('PoolService', () => { let service: PoolService; @@ -147,6 +148,12 @@ describe('PoolService', () => { expect(gatewayService.getTransactionPool).not.toHaveBeenCalled(); }); + it('should fail the pool with filters, like any other failure, while the pool is too large', async () => { + cacheService.getOrSet = jest.fn().mockResolvedValue(null); + + await expect(service.getPoolWithFilters({ senderShard: 0 })).rejects.toBeInstanceOf(TransactionPoolTooLargeException); + }); + it('should count the total through the gateway, cached, whatever the filters', async () => { gatewayService.getTransactionPoolCount = jest.fn().mockResolvedValue(42); cacheService.getOrSet = jest.fn().mockImplementation(async (key: string, createValueFunc: () => Promise) => { diff --git a/src/test/unit/services/pool.too.large.spec.ts b/src/test/unit/services/pool.too.large.spec.ts index fb987a8cdc..df8e9eb656 100644 --- a/src/test/unit/services/pool.too.large.spec.ts +++ b/src/test/unit/services/pool.too.large.spec.ts @@ -54,11 +54,12 @@ describe('Transaction pool too large', () => { }); describe('TransactionService price per unit', () => { - it('should leave the prices unknown while the pool is too large', async () => { + it('should handle a pool too large like any other failure of the pool', async () => { const transactionService: TransactionService = Object.assign(Object.create(TransactionService.prototype), { blockService: { getBlocks: jest.fn().mockResolvedValue([{ nonce: 100 }]) }, networkService: { getConstants: jest.fn().mockResolvedValue({ minGasLimit: 50000, gasPerDataByte: 1500, gasPriceModifier: '0.01' }) }, - poolService: { getPoolWithFilters: jest.fn().mockResolvedValue(null) }, + poolService: { getPoolWithFilters: jest.fn().mockRejectedValue(new TransactionPoolTooLargeException()) }, + logger: { error: jest.fn() }, }); expect(await transactionService.getPpuByShardIdRaw(1)).toBeNull();